mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-03-28 17:32:48 +00:00
Update docs
This commit is contained in:
Sean Whalen
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>Overview: module code — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -91,8 +85,20 @@
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">Document</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
|
||||
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span><span class="p">,</span> <span class="n">Search</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
<span class="n">Index</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Text</span><span class="p">,</span>
|
||||
<span class="n">Boolean</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
@@ -100,10 +106,13 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ElasticsearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a><span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ElasticsearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an Elasticsearch error occurs"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -164,24 +173,21 @@
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span> <span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span> <span class="n">kwargs</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -211,24 +217,25 @@
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -273,14 +280,18 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_type</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">_details</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -290,13 +301,12 @@
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
@@ -308,27 +318,45 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">policy_type</span><span class="p">,</span> <span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">)</span>
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
||||
|
||||
@@ -343,25 +371,25 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'use_ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"use_ssl"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'ca_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"ca_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'http_auth'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">username</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">password</span><span class="p">)</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'api_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create Elasticsearch indexes</span>
|
||||
|
||||
@@ -376,17 +404,18 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -415,33 +444,34 @@
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span>
|
||||
<span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span>
|
||||
<span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to Elasticsearch</span>
|
||||
|
||||
@@ -462,10 +492,8 @@
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -474,8 +502,7 @@
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
||||
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
||||
@@ -497,18 +524,20 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report_id</span><span class="p">,</span>
|
||||
<span class="n">org_name</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">begin_date_human</span><span class="p">,</span>
|
||||
<span class="n">end_date_human</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">],</span>
|
||||
@@ -516,7 +545,7 @@
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">],</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -539,28 +568,33 @@
|
||||
<span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">],</span>
|
||||
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">],</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span><span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
|
||||
<span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -569,41 +603,46 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to Elasticsearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -648,14 +687,12 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span>
|
||||
<span class="n">from_</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="p">,</span>
|
||||
<span class="n">arrival_date_human</span>
|
||||
<span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
@@ -665,25 +702,25 @@
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">]</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
@@ -699,12 +736,11 @@
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span>
|
||||
<span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
@@ -717,26 +753,32 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed SMTP TLS report to Elasticsearch</span>
|
||||
|
||||
@@ -754,10 +796,8 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving smtp tls report to Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -786,15 +826,19 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in Elasticsearch"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in Elasticsearch"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -802,8 +846,9 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSReportDoc</span><span class="p">(</span>
|
||||
<span class="n">org_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
||||
@@ -811,10 +856,10 @@
|
||||
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"contact_info"</span><span class="p">],</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s1">'policies'</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
@@ -827,7 +872,7 @@
|
||||
<span class="n">succesful_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"successful_session_count"</span><span class="p">],</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">failure_detail</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]:</span>
|
||||
@@ -840,11 +885,11 @@
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span><span class="p">]</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_reason_code"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"ip_address"</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
|
||||
@@ -860,12 +905,11 @@
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span>
|
||||
<span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
@@ -875,8 +919,8 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
|
||||
<span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -90,8 +84,21 @@
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="n">Q</span><span class="p">,</span> <span class="n">connections</span><span class="p">,</span> <span class="n">Object</span><span class="p">,</span> <span class="n">Document</span><span class="p">,</span> <span class="n">Index</span><span class="p">,</span> <span class="n">Nested</span><span class="p">,</span> \
|
||||
<span class="n">InnerDoc</span><span class="p">,</span> <span class="n">Integer</span><span class="p">,</span> <span class="n">Text</span><span class="p">,</span> <span class="n">Boolean</span><span class="p">,</span> <span class="n">Ip</span><span class="p">,</span> <span class="n">Date</span><span class="p">,</span> <span class="n">Search</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
<span class="n">Index</span><span class="p">,</span>
|
||||
<span class="n">Nested</span><span class="p">,</span>
|
||||
<span class="n">InnerDoc</span><span class="p">,</span>
|
||||
<span class="n">Integer</span><span class="p">,</span>
|
||||
<span class="n">Text</span><span class="p">,</span>
|
||||
<span class="n">Boolean</span><span class="p">,</span>
|
||||
<span class="n">Ip</span><span class="p">,</span>
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
@@ -99,10 +106,13 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="OpenSearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a><span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="OpenSearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an OpenSearch error occurs"""</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -163,24 +173,21 @@
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span> <span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span> <span class="n">kwargs</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
@@ -210,24 +217,25 @@
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span><span class="p">))</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
@@ -272,13 +280,17 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_type</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_helo</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -287,12 +299,11 @@
|
||||
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
@@ -304,27 +315,45 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">policy_type</span><span class="p">,</span> <span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">)</span>
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Sets the OpenSearch hosts to use</span>
|
||||
|
||||
@@ -339,25 +368,25 @@
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">"timeout"</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'use_ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"use_ssl"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'ca_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"ca_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'http_auth'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">username</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">password</span><span class="p">)</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'api_key'</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create OpenSearch indexes</span>
|
||||
|
||||
@@ -372,17 +401,18 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Creating OpenSearch index: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
|
||||
<span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -411,33 +441,34 @@
|
||||
<span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">"long"</span><span class="p">:</span>
|
||||
<span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span>
|
||||
<span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span>
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"properties"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"published_policy.fo"</span><span class="p">:</span> <span class="p">{</span>
|
||||
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"text"</span><span class="p">,</span>
|
||||
<span class="s2">"fields"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"keyword"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"type"</span><span class="p">:</span> <span class="s2">"keyword"</span><span class="p">,</span> <span class="s2">"ignore_above"</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="p">}</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span>
|
||||
<span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to OpenSearch</span>
|
||||
|
||||
@@ -458,10 +489,8 @@
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -470,8 +499,7 @@
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
|
||||
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
|
||||
@@ -493,18 +521,20 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">report_id</span><span class="p">,</span>
|
||||
<span class="n">org_name</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="p">,</span>
|
||||
<span class="n">begin_date_human</span><span class="p">,</span>
|
||||
<span class="n">end_date_human</span><span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"adkim"</span><span class="p">],</span>
|
||||
@@ -512,7 +542,7 @@
|
||||
<span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"p"</span><span class="p">],</span>
|
||||
<span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"sp"</span><span class="p">],</span>
|
||||
<span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"pct"</span><span class="p">],</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">]</span>
|
||||
<span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">][</span><span class="s2">"fo"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
@@ -535,28 +565,33 @@
|
||||
<span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">],</span>
|
||||
<span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">],</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">"pass"</span><span class="p">,</span>
|
||||
<span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">],</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">]</span>
|
||||
<span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_to"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"policy_override_reasons"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span><span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span>
|
||||
<span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
|
||||
<span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"type"</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">"comment"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"selector"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">])</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"domain"</span><span class="p">],</span>
|
||||
<span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"scope"</span><span class="p">],</span>
|
||||
<span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">"result"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -564,41 +599,46 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to OpenSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> index_prefix (str): The prefix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
|
||||
<span class="sd"> number_of_replicas (int): The number of replicas to use in the</span>
|
||||
<span class="sd"> index</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving forensic report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -643,12 +683,12 @@
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span>
|
||||
<span class="p">))</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
|
||||
@@ -658,25 +698,25 @@
|
||||
<span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
|
||||
<span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"subject"</span><span class="p">],</span>
|
||||
<span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">],</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">]</span>
|
||||
<span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"body"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"reply_to"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
|
||||
<span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"cc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"bcc"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span>
|
||||
<span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"display_name"</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">"address"</span><span class="p">])</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">][</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">])</span>
|
||||
<span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"filename"</span><span class="p">],</span>
|
||||
<span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"mail_content_type"</span><span class="p">],</span>
|
||||
<span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
|
||||
<span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"feedback_type"</span><span class="p">],</span>
|
||||
@@ -692,12 +732,11 @@
|
||||
<span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">],</span>
|
||||
<span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">],</span>
|
||||
<span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span>
|
||||
<span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_mechanisms"</span><span class="p">],</span>
|
||||
<span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"auth_failure"</span><span class="p">],</span>
|
||||
<span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"dkim_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_rcpt_to"</span><span class="p">],</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span>
|
||||
<span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
@@ -710,26 +749,32 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="s2">"Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed SMTP TLS report to OpenSearch</span>
|
||||
|
||||
@@ -747,10 +792,8 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Saving aggregate report to OpenSearch"</span><span class="p">)</span>
|
||||
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"org_name"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span>
|
||||
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -779,15 +822,19 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch's search for existing report </span><span class="se">\</span>
|
||||
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in OpenSearch"</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from "</span>
|
||||
<span class="sa">f</span><span class="s2">" </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to "</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already "</span>
|
||||
<span class="s2">"exists in OpenSearch"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
@@ -795,8 +842,9 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
|
||||
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
|
||||
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureReportDoc</span><span class="p">(</span>
|
||||
<span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"organization_name"</span><span class="p">],</span>
|
||||
@@ -804,10 +852,10 @@
|
||||
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"date_begin"</span><span class="p">],</span>
|
||||
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"date_end"</span><span class="p">],</span>
|
||||
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"contact_info"</span><span class="p">],</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">]</span>
|
||||
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">"report_id"</span><span class="p">],</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s1">'policies'</span><span class="p">]:</span>
|
||||
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policies"</span><span class="p">]:</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"policy_strings"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
@@ -818,7 +866,7 @@
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_domain"</span><span class="p">],</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">"policy_type"</span><span class="p">],</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_details"</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"failure_details"</span><span class="p">]</span>
|
||||
@@ -826,11 +874,11 @@
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"receiving_mx_hostname"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
||||
<span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"receiving_mx_hostname"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"additional_information_uri"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
|
||||
<span class="s2">"additional_information_uri"</span><span class="p">]</span>
|
||||
<span class="s2">"additional_information_uri"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"failure_reason_code"</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">"failure_reason_code"</span><span class="p">]</span>
|
||||
<span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
|
||||
@@ -841,7 +889,7 @@
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">"failed_session_count"</span><span class="p">],</span>
|
||||
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
|
||||
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
|
||||
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
|
||||
|
||||
@@ -851,8 +899,8 @@
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
|
||||
<span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -100,18 +94,24 @@
|
||||
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SplunkError"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a><span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="SplunkError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a Splunk API error occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a><span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""A client for a Splunk HTTP Events Collector (HEC)"""</span>
|
||||
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span>
|
||||
<span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Initializes the HECClient</span>
|
||||
|
||||
@@ -125,8 +125,9 @@
|
||||
<span class="sd"> data before giving up</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">.</span><span class="n">netloc</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="n">url</span><span class="o">.</span><span class="n">netloc</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"Splunk "</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getfqdn</span><span class="p">()</span>
|
||||
@@ -134,15 +135,16 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span>
|
||||
<span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="s2">"parsedmarc/</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">),</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">)</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -166,36 +168,26 @@
|
||||
<span class="k">for</span> <span class="n">metadata</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="n">metadata</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"report_metadata"</span><span class="p">][</span><span class="n">metadata</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"published_policy"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">"policy_published"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"ip_address"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_ip_address"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"ip_address"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"country"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"reverse_dns"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span>
|
||||
<span class="s2">"base_domain"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"reverse_dns"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"base_domain"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"source_name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"source"</span><span class="p">][</span><span class="s2">"name"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"message_count"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"count"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span>
|
||||
<span class="s2">"disposition"</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"disposition"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"policy_evaluated"</span><span class="p">][</span><span class="s2">"disposition"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_aligned"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"passed_dmarc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"alignment"</span><span class="p">][</span><span class="s2">"dmarc"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span>
|
||||
<span class="s2">"header_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span>
|
||||
<span class="s2">"envelope_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"header_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"header_from"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"envelope_from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"identifiers"</span><span class="p">][</span><span class="s2">"envelope_from"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"dkim"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span>
|
||||
<span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"dkim_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"dkim"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="s2">"spf"</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">]:</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span>
|
||||
<span class="s2">"spf"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
||||
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:aggregate"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">new_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -203,15 +195,17 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
|
||||
@@ -230,8 +224,7 @@
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:forensic"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -239,15 +232,17 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -267,8 +262,7 @@
|
||||
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"smtp:tls"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">])</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
@@ -276,13 +270,14 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Skipping certificate verification for Splunk HEC"</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div></div>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
</div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -126,22 +120,30 @@
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.maps</span>
|
||||
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s1">'\s*\(.*\)\s*'</span><span class="p">)</span>
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="EmailParserError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a><span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="EmailParserError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error parsing the email occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="DownloadError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a><span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="DownloadError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a>
|
||||
<span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error occurs when downloading a file"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="decode_base64"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="decode_base64">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Decodes a base64 string, with padding being optional</span>
|
||||
|
||||
@@ -155,11 +157,14 @@
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"ascii"</span><span class="p">)</span>
|
||||
<span class="n">missing_padding</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span>
|
||||
<span class="k">if</span> <span class="n">missing_padding</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">+=</span> <span class="sa">b</span><span class="s1">'='</span> <span class="o">*</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="n">missing_padding</span><span class="p">)</span>
|
||||
<span class="n">data</span> <span class="o">+=</span> <span class="sa">b</span><span class="s2">"="</span> <span class="o">*</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="n">missing_padding</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Gets the base domain name for the given domain</span>
|
||||
|
||||
@@ -178,7 +183,10 @@
|
||||
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="query_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a><span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="query_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Queries DNS</span>
|
||||
|
||||
@@ -204,31 +212,45 @@
|
||||
<span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
|
||||
<span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"1.1.1.1"</span><span class="p">,</span> <span class="s2">"1.0.0.1"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1111"</span><span class="p">,</span> <span class="s2">"2606:4700:4700::1001"</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"1.1.1.1"</span><span class="p">,</span>
|
||||
<span class="s2">"1.0.0.1"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1111"</span><span class="p">,</span>
|
||||
<span class="s2">"2606:4700:4700::1001"</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
|
||||
<span class="k">if</span> <span class="n">record_type</span> <span class="o">==</span> <span class="s2">"TXT"</span><span class="p">:</span>
|
||||
<span class="n">resource_records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">strings</span><span class="p">,</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">)))</span>
|
||||
<span class="n">resource_records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">strings</span><span class="p">,</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">_resource_record</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">resource_record</span><span class="p">[</span><span class="mi">0</span><span class="p">][:</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">resource_record</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">resource_record</span> <span class="ow">in</span> <span class="n">resource_records</span> <span class="k">if</span> <span class="n">resource_record</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">resource_record</span> <span class="ow">in</span> <span class="n">resource_records</span>
|
||||
<span class="k">if</span> <span class="n">resource_record</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="p">[</span><span class="n">r</span><span class="o">.</span><span class="n">decode</span><span class="p">()</span> <span class="k">for</span> <span class="n">r</span> <span class="ow">in</span> <span class="n">_resource_record</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">),</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">)))</span>
|
||||
<span class="n">records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span>
|
||||
<span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'"'</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">),</span>
|
||||
<span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">),</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
|
||||
<span class="n">cache</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">records</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a><span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
|
||||
|
||||
@@ -245,9 +267,9 @@
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span><span class="n">address</span><span class="p">,</span> <span class="s2">"PTR"</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span>
|
||||
<span class="n">address</span><span class="p">,</span> <span class="s2">"PTR"</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
|
||||
<span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"get_reverse_dns(</span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2">) exception: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
@@ -256,7 +278,10 @@
|
||||
<span class="k">return</span> <span class="n">hostname</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``datetime`` object</span>
|
||||
|
||||
@@ -269,7 +294,10 @@
|
||||
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
|
||||
|
||||
@@ -282,7 +310,10 @@
|
||||
<span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a Python ``datetime`` object</span>
|
||||
|
||||
@@ -301,7 +332,10 @@
|
||||
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
|
||||
|
||||
@@ -315,7 +349,10 @@
|
||||
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the ISO code for the country associated</span>
|
||||
<span class="sd"> with the given IPv4 or IPv6 address</span>
|
||||
@@ -344,9 +381,11 @@
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
<span class="s2">"included copy of the IPDB IP to Country "</span>
|
||||
<span class="s2">"Lite database."</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
<span class="s2">"included copy of the IPDB IP to Country "</span>
|
||||
<span class="s2">"Lite database."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">db_paths</span><span class="p">:</span>
|
||||
@@ -355,12 +394,12 @@
|
||||
<span class="k">break</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span>
|
||||
<span class="s2">"dbip-country-lite.mmdb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span> <span class="s2">"dbip-country-lite.mmdb"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">path</span>
|
||||
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
|
||||
@@ -376,12 +415,17 @@
|
||||
<span class="k">return</span> <span class="n">country</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span><span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
<span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the service name of a given base domain name from reverse DNS.</span>
|
||||
|
||||
@@ -397,28 +441,27 @@
|
||||
<span class="sd"> If the service is unknown, the name will be</span>
|
||||
<span class="sd"> the supplied reverse_dns_base_domain and the type will be None</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="n">reader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span>
|
||||
<span class="n">key</span> <span class="o">=</span> <span class="n">row</span><span class="p">[</span><span class="s2">"base_reverse_dns"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
|
||||
<span class="n">name</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"type"</span><span class="p">])</span>
|
||||
<span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"name"</span><span class="p">],</span> <span class="nb">type</span><span class="o">=</span><span class="n">row</span><span class="p">[</span><span class="s2">"type"</span><span class="p">])</span>
|
||||
|
||||
<span class="n">base_domain</span> <span class="o">=</span> <span class="n">base_domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"https://raw.githubusercontent.com/domainaware"</span>
|
||||
<span class="s2">"/parsedmarc/master/parsedmarc/"</span>
|
||||
<span class="s2">"resources/maps/base_reverse_dns_map.csv"</span><span class="p">)</span>
|
||||
<span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s2">"https://raw.githubusercontent.com/domainaware"</span>
|
||||
<span class="s2">"/parsedmarc/master/parsedmarc/"</span>
|
||||
<span class="s2">"resources/maps/base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">reverse_dns_map</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns_map</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">csv_file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span>
|
||||
<span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch "</span>
|
||||
<span class="sa">f</span><span class="s2">"reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch "</span> <span class="sa">f</span><span class="s2">"reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
@@ -426,8 +469,9 @@
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Failed to fetch reverse DNS map: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading included reverse DNS map..."</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span>
|
||||
<span class="s2">"base_reverse_dns_map.csv"</span><span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span> <span class="s2">"base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
@@ -440,15 +484,21 @@
|
||||
<span class="k">return</span> <span class="n">service</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns reverse DNS and country information for the given IP address</span>
|
||||
|
||||
@@ -480,9 +530,9 @@
|
||||
<span class="k">if</span> <span class="n">offline</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">country</span> <span class="o">=</span> <span class="n">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">)</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"country"</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"reverse_dns"</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
|
||||
@@ -498,7 +548,8 @@
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
||||
<span class="n">url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">reverse_dns_map</span><span class="p">)</span>
|
||||
<span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">reverse_dns_map</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"base_domain"</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">"type"</span><span class="p">]</span>
|
||||
<span class="n">info</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">"name"</span><span class="p">]</span>
|
||||
@@ -512,6 +563,7 @@
|
||||
<span class="k">return</span> <span class="n">info</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@@ -525,13 +577,19 @@
|
||||
<span class="n">local</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">"display_name"</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"address"</span><span class="p">,</span> <span class="n">address</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"local"</span><span class="p">,</span> <span class="n">local</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">domain</span><span class="p">)])</span>
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[</span>
|
||||
<span class="p">(</span><span class="s2">"display_name"</span><span class="p">,</span> <span class="n">display_name</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"address"</span><span class="p">,</span> <span class="n">address</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"local"</span><span class="p">,</span> <span class="n">local</span><span class="p">),</span>
|
||||
<span class="p">(</span><span class="s2">"domain"</span><span class="p">,</span> <span class="n">domain</span><span class="p">),</span>
|
||||
<span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_filename_safe_string"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a><span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="get_filename_safe_string">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a string to a string that is safe for a filename</span>
|
||||
|
||||
@@ -541,8 +599,7 @@
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> str: A string safe for a filename</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'</span><span class="se">\\</span><span class="s1">'</span><span class="p">,</span> <span class="s1">'/'</span><span class="p">,</span> <span class="s1">':'</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">,</span> <span class="s1">'*'</span><span class="p">,</span> <span class="s1">'?'</span><span class="p">,</span> <span class="s1">'|'</span><span class="p">,</span> <span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span>
|
||||
<span class="s1">'</span><span class="se">\r</span><span class="s1">'</span><span class="p">]</span>
|
||||
<span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"</span><span class="se">\\</span><span class="s2">"</span><span class="p">,</span> <span class="s2">"/"</span><span class="p">,</span> <span class="s2">":"</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">,</span> <span class="s2">"*"</span><span class="p">,</span> <span class="s2">"?"</span><span class="p">,</span> <span class="s2">"|"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\r</span><span class="s2">"</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">string</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||
<span class="k">for</span> <span class="n">char</span> <span class="ow">in</span> <span class="n">invalid_filename_chars</span><span class="p">:</span>
|
||||
@@ -554,7 +611,10 @@
|
||||
<span class="k">return</span> <span class="n">string</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_mbox"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a><span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_mbox">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an MBOX mailbox file</span>
|
||||
|
||||
@@ -575,7 +635,10 @@
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an Outlook msg OLE/MSG file</span>
|
||||
|
||||
@@ -586,10 +649,14 @@
|
||||
<span class="sd"> bool: A flag that indicates if the file is an Outlook MSG file</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span> <span class="ow">and</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
|
||||
<span class="sa">b</span><span class="s2">"</span><span class="se">\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1</span><span class="s2">"</span><span class="p">)</span></div>
|
||||
<span class="sa">b</span><span class="s2">"</span><span class="se">\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1</span><span class="s2">"</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
|
||||
<span class="sd"> standard RFC 822 format</span>
|
||||
@@ -608,14 +675,16 @@
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">"sample.msg"</span><span class="p">,</span> <span class="s2">"wb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
|
||||
<span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">"msgconvert"</span><span class="p">,</span> <span class="s2">"sample.msg"</span><span class="p">],</span>
|
||||
<span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span><span class="p">)</span>
|
||||
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span>
|
||||
<span class="p">[</span><span class="s2">"msgconvert"</span><span class="p">,</span> <span class="s2">"sample.msg"</span><span class="p">],</span> <span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">eml_path</span> <span class="o">=</span> <span class="s2">"sample.eml"</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
|
||||
<span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">EmailParserError</span><span class="p">(</span>
|
||||
<span class="s2">"Failed to convert Outlook MSG: msgconvert utility not found"</span><span class="p">)</span>
|
||||
<span class="s2">"Failed to convert Outlook MSG: msgconvert utility not found"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">finally</span><span class="p">:</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">orig_dir</span><span class="p">)</span>
|
||||
<span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
|
||||
@@ -623,7 +692,10 @@
|
||||
<span class="k">return</span> <span class="n">rfc822</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_email"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a><span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_email">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a>
|
||||
<span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> A simplified email parser</span>
|
||||
|
||||
@@ -650,8 +722,7 @@
|
||||
<span class="k">if</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span>
|
||||
<span class="s2">" "</span><span class="p">)</span>
|
||||
<span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]:</span>
|
||||
@@ -667,33 +738,36 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"date"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"reply_to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"reply_to"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"cc"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"cc"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"bcc"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]))</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"bcc"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"delivered_to"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">])</span>
|
||||
<span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"delivered_to"</span><span class="p">])</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"attachments"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
@@ -710,9 +784,7 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">attachment</span><span class="p">[</span><span class="s2">"sha256"</span><span class="p">]</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
||||
<span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
|
||||
<span class="k">if</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"attachments"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="s2">"payload"</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
|
||||
@@ -722,12 +794,14 @@
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"filename_safe_subject"</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_filename_safe_string</span><span class="p">(</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">])</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"body"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">parsed_email</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -103,6 +103,12 @@ port = 514
|
||||
host = logger
|
||||
port = 12201
|
||||
mode = tcp
|
||||
|
||||
[webhook]
|
||||
aggregate_url = https://aggregate_url.example.com
|
||||
forensic_url = https://forensic_url.example.com
|
||||
smtp_tls_url = https://smtp_tls_url.example.com
|
||||
timeout = 60
|
||||
```
|
||||
|
||||
The full set of configuration options are:
|
||||
@@ -130,6 +136,8 @@ The full set of configuration options are:
|
||||
- `reverse_dns_map_url` - Overrides the default download URL for the reverse DNS map
|
||||
- `nameservers` - str: A comma separated list of
|
||||
DNS resolvers (Default: `[Cloudflare's public resolvers]`)
|
||||
- `dns_test_address` - str: a dummy address used for DNS pre-flight checks
|
||||
(Default: 1.1.1.1)
|
||||
- `dns_timeout` - float: DNS timeout period
|
||||
- `debug` - bool: Print debugging messages
|
||||
- `silent` - bool: Only print errors (Default: `True`)
|
||||
@@ -355,6 +363,16 @@ The full set of configuration options are:
|
||||
- `port` - int: The port to use
|
||||
- `mode` - str: The GELF transport type to use. Valid modes: `tcp`, `udp`, `tls`
|
||||
|
||||
- `maildir`
|
||||
- `reports_folder` - str: Full path for mailbox maidir location (Default: `INBOX`)
|
||||
- `maildir_create` - bool: Create maildir if not present (Default: False)
|
||||
|
||||
- `webhook` - Post the individual reports to a webhook url with the report as the JSON body
|
||||
- `aggregate_url` - str: URL of the webhook which should receive the aggregate reports
|
||||
- `forensic_url` - str: URL of the webhook which should receive the forensic reports
|
||||
- `smtp_tls_url` - str: URL of the webhook which should receive the smtp_tls reports
|
||||
- `timeout` - int: Interval in which the webhook call should timeout
|
||||
|
||||
:::{warning}
|
||||
It is **strongly recommended** to **not** use the `nameservers`
|
||||
setting. By default, `parsedmarc` uses
|
||||
|
||||
@@ -1,20 +1,9 @@
|
||||
/*
|
||||
* _sphinx_javascript_frameworks_compat.js
|
||||
* ~~~~~~~~~~
|
||||
*
|
||||
* Compatability shim for jQuery and underscores.js.
|
||||
*
|
||||
* WILL BE REMOVED IN Sphinx 6.0
|
||||
* xref RemovedInSphinx60Warning
|
||||
/* Compatability shim for jQuery and underscores.js.
|
||||
*
|
||||
* Copyright Sphinx contributors
|
||||
* Released under the two clause BSD licence
|
||||
*/
|
||||
|
||||
/**
|
||||
* select a different prefix for underscore
|
||||
*/
|
||||
$u = _.noConflict();
|
||||
|
||||
|
||||
/**
|
||||
* small helper function to urldecode strings
|
||||
*
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* basic.css
|
||||
* ~~~~~~~~~
|
||||
*
|
||||
* Sphinx stylesheet -- basic theme.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
|
||||
/* -- main layout ----------------------------------------------------------- */
|
||||
@@ -115,15 +108,11 @@ img {
|
||||
/* -- search page ----------------------------------------------------------- */
|
||||
|
||||
ul.search {
|
||||
margin: 10px 0 0 20px;
|
||||
padding: 0;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
ul.search li {
|
||||
padding: 5px 0 5px 20px;
|
||||
background-image: url(file.png);
|
||||
background-repeat: no-repeat;
|
||||
background-position: 0 7px;
|
||||
padding: 5px 0;
|
||||
}
|
||||
|
||||
ul.search li a {
|
||||
@@ -236,17 +225,11 @@ div.body p, div.body dd, div.body li, div.body blockquote {
|
||||
a.headerlink {
|
||||
visibility: hidden;
|
||||
}
|
||||
a.brackets:before,
|
||||
span.brackets > a:before{
|
||||
content: "[";
|
||||
}
|
||||
|
||||
a.brackets:after,
|
||||
span.brackets > a:after {
|
||||
content: "]";
|
||||
a:visited {
|
||||
color: #551A8B;
|
||||
}
|
||||
|
||||
|
||||
h1:hover > a.headerlink,
|
||||
h2:hover > a.headerlink,
|
||||
h3:hover > a.headerlink,
|
||||
@@ -334,11 +317,17 @@ aside.sidebar {
|
||||
p.sidebar-title {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
nav.contents,
|
||||
aside.topic,
|
||||
div.admonition, div.topic, blockquote {
|
||||
clear: left;
|
||||
}
|
||||
|
||||
/* -- topics ---------------------------------------------------------------- */
|
||||
|
||||
nav.contents,
|
||||
aside.topic,
|
||||
div.topic {
|
||||
border: 1px solid #ccc;
|
||||
padding: 7px;
|
||||
@@ -377,6 +366,8 @@ div.body p.centered {
|
||||
|
||||
div.sidebar > :last-child,
|
||||
aside.sidebar > :last-child,
|
||||
nav.contents > :last-child,
|
||||
aside.topic > :last-child,
|
||||
div.topic > :last-child,
|
||||
div.admonition > :last-child {
|
||||
margin-bottom: 0;
|
||||
@@ -384,6 +375,8 @@ div.admonition > :last-child {
|
||||
|
||||
div.sidebar::after,
|
||||
aside.sidebar::after,
|
||||
nav.contents::after,
|
||||
aside.topic::after,
|
||||
div.topic::after,
|
||||
div.admonition::after,
|
||||
blockquote::after {
|
||||
@@ -608,19 +601,27 @@ ol.simple p,
|
||||
ul.simple p {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
dl.footnote > dt,
|
||||
dl.citation > dt {
|
||||
float: left;
|
||||
margin-right: 0.5em;
|
||||
}
|
||||
|
||||
dl.footnote > dd,
|
||||
dl.citation > dd {
|
||||
aside.footnote > span,
|
||||
div.citation > span {
|
||||
float: left;
|
||||
}
|
||||
aside.footnote > span:last-of-type,
|
||||
div.citation > span:last-of-type {
|
||||
padding-right: 0.5em;
|
||||
}
|
||||
aside.footnote > p {
|
||||
margin-left: 2em;
|
||||
}
|
||||
div.citation > p {
|
||||
margin-left: 4em;
|
||||
}
|
||||
aside.footnote > p:last-of-type,
|
||||
div.citation > p:last-of-type {
|
||||
margin-bottom: 0em;
|
||||
}
|
||||
|
||||
dl.footnote > dd:after,
|
||||
dl.citation > dd:after {
|
||||
aside.footnote > p:last-of-type:after,
|
||||
div.citation > p:last-of-type:after {
|
||||
content: "";
|
||||
clear: both;
|
||||
}
|
||||
@@ -636,10 +637,6 @@ dl.field-list > dt {
|
||||
padding-left: 0.5em;
|
||||
padding-right: 5px;
|
||||
}
|
||||
dl.field-list > dt:after {
|
||||
content: ":";
|
||||
}
|
||||
|
||||
|
||||
dl.field-list > dd {
|
||||
padding-left: 0.5em;
|
||||
@@ -666,6 +663,16 @@ dd {
|
||||
margin-left: 30px;
|
||||
}
|
||||
|
||||
.sig dd {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
.sig dl {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
dl > dd:last-child,
|
||||
dl > dd:last-child > :last-child {
|
||||
margin-bottom: 0;
|
||||
@@ -734,6 +741,14 @@ abbr, acronym {
|
||||
cursor: help;
|
||||
}
|
||||
|
||||
.translated {
|
||||
background-color: rgba(207, 255, 207, 0.2)
|
||||
}
|
||||
|
||||
.untranslated {
|
||||
background-color: rgba(255, 207, 207, 0.2)
|
||||
}
|
||||
|
||||
/* -- code displays --------------------------------------------------------- */
|
||||
|
||||
pre {
|
||||
|
||||
@@ -1 +1 @@
|
||||
.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
|
||||
.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* doctools.js
|
||||
* ~~~~~~~~~~~
|
||||
*
|
||||
* Base JavaScript utilities for all Sphinx HTML documentation.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
var DOCUMENTATION_OPTIONS = {
|
||||
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
|
||||
VERSION: '8.15.0',
|
||||
const DOCUMENTATION_OPTIONS = {
|
||||
VERSION: '8.15.1',
|
||||
LANGUAGE: 'en',
|
||||
COLLAPSE_INDEX: false,
|
||||
BUILDER: 'html',
|
||||
|
||||
224
_static/js/versions.js
Normal file
224
_static/js/versions.js
Normal file
@@ -0,0 +1,224 @@
|
||||
const themeFlyoutDisplay = "hidden";
|
||||
const themeVersionSelector = "True";
|
||||
const themeLanguageSelector = "True";
|
||||
|
||||
if (themeFlyoutDisplay === "attached") {
|
||||
function renderLanguages(config) {
|
||||
if (!config.projects.translations.length) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const languagesHTML = `
|
||||
<dl>
|
||||
<dt>Languages</dt>
|
||||
${config.projects.translations
|
||||
.map(
|
||||
(translation) => `
|
||||
<dd ${translation.slug == config.projects.current.slug ? 'class="rtd-current-item"' : ""}>
|
||||
<a href="${translation.urls.documentation}">${translation.language.code}</a>
|
||||
</dd>
|
||||
`,
|
||||
)
|
||||
.join("\n")}
|
||||
</dl>
|
||||
`;
|
||||
return languagesHTML;
|
||||
}
|
||||
|
||||
function renderVersions(config) {
|
||||
if (!config.versions.active.length) {
|
||||
return "";
|
||||
}
|
||||
const versionsHTML = `
|
||||
<dl>
|
||||
<dt>Versions</dt>
|
||||
${config.versions.active
|
||||
.map(
|
||||
(version) => `
|
||||
<dd ${version.slug === config.versions.current.slug ? 'class="rtd-current-item"' : ""}>
|
||||
<a href="${version.urls.documentation}">${version.slug}</a>
|
||||
</dd>
|
||||
`,
|
||||
)
|
||||
.join("\n")}
|
||||
</dl>
|
||||
`;
|
||||
return versionsHTML;
|
||||
}
|
||||
|
||||
function renderDownloads(config) {
|
||||
if (!Object.keys(config.versions.current.downloads).length) {
|
||||
return "";
|
||||
}
|
||||
const downloadsNameDisplay = {
|
||||
pdf: "PDF",
|
||||
epub: "Epub",
|
||||
htmlzip: "HTML",
|
||||
};
|
||||
|
||||
const downloadsHTML = `
|
||||
<dl>
|
||||
<dt>Downloads</dt>
|
||||
${Object.entries(config.versions.current.downloads)
|
||||
.map(
|
||||
([name, url]) => `
|
||||
<dd>
|
||||
<a href="${url}">${downloadsNameDisplay[name]}</a>
|
||||
</dd>
|
||||
`,
|
||||
)
|
||||
.join("\n")}
|
||||
</dl>
|
||||
`;
|
||||
return downloadsHTML;
|
||||
}
|
||||
|
||||
document.addEventListener("readthedocs-addons-data-ready", function (event) {
|
||||
const config = event.detail.data();
|
||||
|
||||
const flyout = `
|
||||
<div class="rst-versions" data-toggle="rst-versions" role="note">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
<span class="fa fa-book"> Read the Docs</span>
|
||||
v: ${config.versions.current.slug}
|
||||
<span class="fa fa-caret-down"></span>
|
||||
</span>
|
||||
<div class="rst-other-versions">
|
||||
<div class="injected">
|
||||
${renderLanguages(config)}
|
||||
${renderVersions(config)}
|
||||
${renderDownloads(config)}
|
||||
<dl>
|
||||
<dt>On Read the Docs</dt>
|
||||
<dd>
|
||||
<a href="${config.projects.current.urls.home}">Project Home</a>
|
||||
</dd>
|
||||
<dd>
|
||||
<a href="${config.projects.current.urls.builds}">Builds</a>
|
||||
</dd>
|
||||
<dd>
|
||||
<a href="${config.projects.current.urls.downloads}">Downloads</a>
|
||||
</dd>
|
||||
</dl>
|
||||
<dl>
|
||||
<dt>Search</dt>
|
||||
<dd>
|
||||
<form id="flyout-search-form">
|
||||
<input
|
||||
class="wy-form"
|
||||
type="text"
|
||||
name="q"
|
||||
aria-label="Search docs"
|
||||
placeholder="Search docs"
|
||||
/>
|
||||
</form>
|
||||
</dd>
|
||||
</dl>
|
||||
<hr />
|
||||
<small>
|
||||
<span>Hosted by <a href="https://about.readthedocs.org/?utm_source=&utm_content=flyout">Read the Docs</a></span>
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Inject the generated flyout into the body HTML element.
|
||||
document.body.insertAdjacentHTML("beforeend", flyout);
|
||||
|
||||
// Trigger the Read the Docs Addons Search modal when clicking on the "Search docs" input from inside the flyout.
|
||||
document
|
||||
.querySelector("#flyout-search-form")
|
||||
.addEventListener("focusin", () => {
|
||||
const event = new CustomEvent("readthedocs-search-show");
|
||||
document.dispatchEvent(event);
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
if (themeLanguageSelector || themeVersionSelector) {
|
||||
function onSelectorSwitch(event) {
|
||||
const option = event.target.selectedIndex;
|
||||
const item = event.target.options[option];
|
||||
window.location.href = item.dataset.url;
|
||||
}
|
||||
|
||||
document.addEventListener("readthedocs-addons-data-ready", function (event) {
|
||||
const config = event.detail.data();
|
||||
|
||||
const versionSwitch = document.querySelector(
|
||||
"div.switch-menus > div.version-switch",
|
||||
);
|
||||
if (themeVersionSelector) {
|
||||
let versions = config.versions.active;
|
||||
if (config.versions.current.hidden || config.versions.current.type === "external") {
|
||||
versions.unshift(config.versions.current);
|
||||
}
|
||||
const versionSelect = `
|
||||
<select>
|
||||
${versions
|
||||
.map(
|
||||
(version) => `
|
||||
<option
|
||||
value="${version.slug}"
|
||||
${config.versions.current.slug === version.slug ? 'selected="selected"' : ""}
|
||||
data-url="${version.urls.documentation}">
|
||||
${version.slug}
|
||||
</option>`,
|
||||
)
|
||||
.join("\n")}
|
||||
</select>
|
||||
`;
|
||||
|
||||
versionSwitch.innerHTML = versionSelect;
|
||||
versionSwitch.firstElementChild.addEventListener("change", onSelectorSwitch);
|
||||
}
|
||||
|
||||
const languageSwitch = document.querySelector(
|
||||
"div.switch-menus > div.language-switch",
|
||||
);
|
||||
|
||||
if (themeLanguageSelector) {
|
||||
if (config.projects.translations.length) {
|
||||
// Add the current language to the options on the selector
|
||||
let languages = config.projects.translations.concat(
|
||||
config.projects.current,
|
||||
);
|
||||
languages = languages.sort((a, b) =>
|
||||
a.language.name.localeCompare(b.language.name),
|
||||
);
|
||||
|
||||
const languageSelect = `
|
||||
<select>
|
||||
${languages
|
||||
.map(
|
||||
(language) => `
|
||||
<option
|
||||
value="${language.language.code}"
|
||||
${config.projects.current.slug === language.slug ? 'selected="selected"' : ""}
|
||||
data-url="${language.urls.documentation}">
|
||||
${language.language.name}
|
||||
</option>`,
|
||||
)
|
||||
.join("\n")}
|
||||
</select>
|
||||
`;
|
||||
|
||||
languageSwitch.innerHTML = languageSelect;
|
||||
languageSwitch.firstElementChild.addEventListener("change", onSelectorSwitch);
|
||||
}
|
||||
else {
|
||||
languageSwitch.remove();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener("readthedocs-addons-data-ready", function (event) {
|
||||
// Trigger the Read the Docs Addons Search modal when clicking on "Search docs" input from the topnav.
|
||||
document
|
||||
.querySelector("[role='search'] input")
|
||||
.addEventListener("focusin", () => {
|
||||
const event = new CustomEvent("readthedocs-search-show");
|
||||
document.dispatchEvent(event);
|
||||
});
|
||||
});
|
||||
@@ -1,19 +1,12 @@
|
||||
/*
|
||||
* language_data.js
|
||||
* ~~~~~~~~~~~~~~~~
|
||||
*
|
||||
* This script contains the language-specific data used by searchtools.js,
|
||||
* namely the list of stopwords, stemmer, scorer and splitter.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
|
||||
var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"];
|
||||
|
||||
|
||||
/* Non-minified version is copied as a separate JS file, is available */
|
||||
/* Non-minified version is copied as a separate JS file, if available */
|
||||
|
||||
/**
|
||||
* Porter Stemmer
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* searchtools.js
|
||||
* ~~~~~~~~~~~~~~~~
|
||||
*
|
||||
* Sphinx JavaScript utilities for the full-text search.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
@@ -20,7 +13,7 @@ if (typeof Scorer === "undefined") {
|
||||
// and returns the new score.
|
||||
/*
|
||||
score: result => {
|
||||
const [docname, title, anchor, descr, score, filename] = result
|
||||
const [docname, title, anchor, descr, score, filename, kind] = result
|
||||
return score
|
||||
},
|
||||
*/
|
||||
@@ -47,6 +40,14 @@ if (typeof Scorer === "undefined") {
|
||||
};
|
||||
}
|
||||
|
||||
// Global search result kind enum, used by themes to style search results.
|
||||
class SearchResultKind {
|
||||
static get index() { return "index"; }
|
||||
static get object() { return "object"; }
|
||||
static get text() { return "text"; }
|
||||
static get title() { return "title"; }
|
||||
}
|
||||
|
||||
const _removeChildren = (element) => {
|
||||
while (element && element.lastChild) element.removeChild(element.lastChild);
|
||||
};
|
||||
@@ -57,16 +58,20 @@ const _removeChildren = (element) => {
|
||||
const _escapeRegExp = (string) =>
|
||||
string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
|
||||
|
||||
const _displayItem = (item, searchTerms) => {
|
||||
const _displayItem = (item, searchTerms, highlightTerms) => {
|
||||
const docBuilder = DOCUMENTATION_OPTIONS.BUILDER;
|
||||
const docUrlRoot = DOCUMENTATION_OPTIONS.URL_ROOT;
|
||||
const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX;
|
||||
const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX;
|
||||
const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY;
|
||||
const contentRoot = document.documentElement.dataset.content_root;
|
||||
|
||||
const [docName, title, anchor, descr, score, _filename] = item;
|
||||
const [docName, title, anchor, descr, score, _filename, kind] = item;
|
||||
|
||||
let listItem = document.createElement("li");
|
||||
// Add a class representing the item's type:
|
||||
// can be used by a theme's CSS selector for styling
|
||||
// See SearchResultKind for the class names.
|
||||
listItem.classList.add(`kind-${kind}`);
|
||||
let requestUrl;
|
||||
let linkUrl;
|
||||
if (docBuilder === "dirhtml") {
|
||||
@@ -75,28 +80,35 @@ const _displayItem = (item, searchTerms) => {
|
||||
if (dirname.match(/\/index\/$/))
|
||||
dirname = dirname.substring(0, dirname.length - 6);
|
||||
else if (dirname === "index/") dirname = "";
|
||||
requestUrl = docUrlRoot + dirname;
|
||||
requestUrl = contentRoot + dirname;
|
||||
linkUrl = requestUrl;
|
||||
} else {
|
||||
// normal html builders
|
||||
requestUrl = docUrlRoot + docName + docFileSuffix;
|
||||
requestUrl = contentRoot + docName + docFileSuffix;
|
||||
linkUrl = docName + docLinkSuffix;
|
||||
}
|
||||
let linkEl = listItem.appendChild(document.createElement("a"));
|
||||
linkEl.href = linkUrl + anchor;
|
||||
linkEl.dataset.score = score;
|
||||
linkEl.innerHTML = title;
|
||||
if (descr)
|
||||
if (descr) {
|
||||
listItem.appendChild(document.createElement("span")).innerHTML =
|
||||
" (" + descr + ")";
|
||||
// highlight search terms in the description
|
||||
if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
|
||||
highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
|
||||
}
|
||||
else if (showSearchSummary)
|
||||
fetch(requestUrl)
|
||||
.then((responseData) => responseData.text())
|
||||
.then((data) => {
|
||||
if (data)
|
||||
listItem.appendChild(
|
||||
Search.makeSearchSummary(data, searchTerms)
|
||||
Search.makeSearchSummary(data, searchTerms, anchor)
|
||||
);
|
||||
// highlight search terms in the summary
|
||||
if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
|
||||
highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
|
||||
});
|
||||
Search.output.appendChild(listItem);
|
||||
};
|
||||
@@ -108,27 +120,46 @@ const _finishSearch = (resultCount) => {
|
||||
"Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories."
|
||||
);
|
||||
else
|
||||
Search.status.innerText = _(
|
||||
`Search finished, found ${resultCount} page(s) matching the search query.`
|
||||
);
|
||||
Search.status.innerText = Documentation.ngettext(
|
||||
"Search finished, found one page matching the search query.",
|
||||
"Search finished, found ${resultCount} pages matching the search query.",
|
||||
resultCount,
|
||||
).replace('${resultCount}', resultCount);
|
||||
};
|
||||
const _displayNextItem = (
|
||||
results,
|
||||
resultCount,
|
||||
searchTerms
|
||||
searchTerms,
|
||||
highlightTerms,
|
||||
) => {
|
||||
// results left, load the summary and display it
|
||||
// this is intended to be dynamic (don't sub resultsCount)
|
||||
if (results.length) {
|
||||
_displayItem(results.pop(), searchTerms);
|
||||
_displayItem(results.pop(), searchTerms, highlightTerms);
|
||||
setTimeout(
|
||||
() => _displayNextItem(results, resultCount, searchTerms),
|
||||
() => _displayNextItem(results, resultCount, searchTerms, highlightTerms),
|
||||
5
|
||||
);
|
||||
}
|
||||
// search finished, update title and status message
|
||||
else _finishSearch(resultCount);
|
||||
};
|
||||
// Helper function used by query() to order search results.
|
||||
// Each input is an array of [docname, title, anchor, descr, score, filename, kind].
|
||||
// Order the results by score (in opposite order of appearance, since the
|
||||
// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically.
|
||||
const _orderResultsByScoreThenName = (a, b) => {
|
||||
const leftScore = a[4];
|
||||
const rightScore = b[4];
|
||||
if (leftScore === rightScore) {
|
||||
// same score: sort alphabetically
|
||||
const leftTitle = a[1].toLowerCase();
|
||||
const rightTitle = b[1].toLowerCase();
|
||||
if (leftTitle === rightTitle) return 0;
|
||||
return leftTitle > rightTitle ? -1 : 1; // inverted is intentional
|
||||
}
|
||||
return leftScore > rightScore ? 1 : -1;
|
||||
};
|
||||
|
||||
/**
|
||||
* Default splitQuery function. Can be overridden in ``sphinx.search`` with a
|
||||
@@ -152,13 +183,26 @@ const Search = {
|
||||
_queued_query: null,
|
||||
_pulse_status: -1,
|
||||
|
||||
htmlToText: (htmlString) => {
|
||||
htmlToText: (htmlString, anchor) => {
|
||||
const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html');
|
||||
htmlElement.querySelectorAll(".headerlink").forEach((el) => { el.remove() });
|
||||
for (const removalQuery of [".headerlink", "script", "style"]) {
|
||||
htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() });
|
||||
}
|
||||
if (anchor) {
|
||||
const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`);
|
||||
if (anchorContent) return anchorContent.textContent;
|
||||
|
||||
console.warn(
|
||||
`Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.`
|
||||
);
|
||||
}
|
||||
|
||||
// if anchor not specified or not found, fall back to main content
|
||||
const docContent = htmlElement.querySelector('[role="main"]');
|
||||
if (docContent !== undefined) return docContent.textContent;
|
||||
if (docContent) return docContent.textContent;
|
||||
|
||||
console.warn(
|
||||
"Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template."
|
||||
"Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template."
|
||||
);
|
||||
return "";
|
||||
},
|
||||
@@ -211,6 +255,7 @@ const Search = {
|
||||
searchSummary.classList.add("search-summary");
|
||||
searchSummary.innerText = "";
|
||||
const searchList = document.createElement("ul");
|
||||
searchList.setAttribute("role", "list");
|
||||
searchList.classList.add("search");
|
||||
|
||||
const out = document.getElementById("search-results");
|
||||
@@ -231,16 +276,7 @@ const Search = {
|
||||
else Search.deferQuery(query);
|
||||
},
|
||||
|
||||
/**
|
||||
* execute search (requires search index to be loaded)
|
||||
*/
|
||||
query: (query) => {
|
||||
const filenames = Search._index.filenames;
|
||||
const docNames = Search._index.docnames;
|
||||
const titles = Search._index.titles;
|
||||
const allTitles = Search._index.alltitles;
|
||||
const indexEntries = Search._index.indexentries;
|
||||
|
||||
_parseQuery: (query) => {
|
||||
// stem the search terms and add them to the correct list
|
||||
const stemmer = new Stemmer();
|
||||
const searchTerms = new Set();
|
||||
@@ -276,22 +312,40 @@ const Search = {
|
||||
// console.info("required: ", [...searchTerms]);
|
||||
// console.info("excluded: ", [...excludedTerms]);
|
||||
|
||||
// array of [docname, title, anchor, descr, score, filename]
|
||||
let results = [];
|
||||
return [query, searchTerms, excludedTerms, highlightTerms, objectTerms];
|
||||
},
|
||||
|
||||
/**
|
||||
* execute search (requires search index to be loaded)
|
||||
*/
|
||||
_performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => {
|
||||
const filenames = Search._index.filenames;
|
||||
const docNames = Search._index.docnames;
|
||||
const titles = Search._index.titles;
|
||||
const allTitles = Search._index.alltitles;
|
||||
const indexEntries = Search._index.indexentries;
|
||||
|
||||
// Collect multiple result groups to be sorted separately and then ordered.
|
||||
// Each is an array of [docname, title, anchor, descr, score, filename, kind].
|
||||
const normalResults = [];
|
||||
const nonMainIndexResults = [];
|
||||
|
||||
_removeChildren(document.getElementById("search-progress"));
|
||||
|
||||
const queryLower = query.toLowerCase();
|
||||
const queryLower = query.toLowerCase().trim();
|
||||
for (const [title, foundTitles] of Object.entries(allTitles)) {
|
||||
if (title.toLowerCase().includes(queryLower) && (queryLower.length >= title.length/2)) {
|
||||
if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) {
|
||||
for (const [file, id] of foundTitles) {
|
||||
let score = Math.round(100 * queryLower.length / title.length)
|
||||
results.push([
|
||||
const score = Math.round(Scorer.title * queryLower.length / title.length);
|
||||
const boost = titles[file] === title ? 1 : 0; // add a boost for document titles
|
||||
normalResults.push([
|
||||
docNames[file],
|
||||
titles[file] !== title ? `${titles[file]} > ${title}` : title,
|
||||
id !== null ? "#" + id : "",
|
||||
null,
|
||||
score,
|
||||
score + boost,
|
||||
filenames[file],
|
||||
SearchResultKind.title,
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -300,46 +354,48 @@ const Search = {
|
||||
// search for explicit entries in index directives
|
||||
for (const [entry, foundEntries] of Object.entries(indexEntries)) {
|
||||
if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) {
|
||||
for (const [file, id] of foundEntries) {
|
||||
let score = Math.round(100 * queryLower.length / entry.length)
|
||||
results.push([
|
||||
for (const [file, id, isMain] of foundEntries) {
|
||||
const score = Math.round(100 * queryLower.length / entry.length);
|
||||
const result = [
|
||||
docNames[file],
|
||||
titles[file],
|
||||
id ? "#" + id : "",
|
||||
null,
|
||||
score,
|
||||
filenames[file],
|
||||
]);
|
||||
SearchResultKind.index,
|
||||
];
|
||||
if (isMain) {
|
||||
normalResults.push(result);
|
||||
} else {
|
||||
nonMainIndexResults.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookup as object
|
||||
objectTerms.forEach((term) =>
|
||||
results.push(...Search.performObjectSearch(term, objectTerms))
|
||||
normalResults.push(...Search.performObjectSearch(term, objectTerms))
|
||||
);
|
||||
|
||||
// lookup as search terms in fulltext
|
||||
results.push(...Search.performTermsSearch(searchTerms, excludedTerms));
|
||||
normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms));
|
||||
|
||||
// let the scorer override scores with a custom scoring function
|
||||
if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
if (Scorer.score) {
|
||||
normalResults.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
}
|
||||
|
||||
// now sort the results by score (in opposite order of appearance, since the
|
||||
// display function below uses pop() to retrieve items) and then
|
||||
// alphabetically
|
||||
results.sort((a, b) => {
|
||||
const leftScore = a[4];
|
||||
const rightScore = b[4];
|
||||
if (leftScore === rightScore) {
|
||||
// same score: sort alphabetically
|
||||
const leftTitle = a[1].toLowerCase();
|
||||
const rightTitle = b[1].toLowerCase();
|
||||
if (leftTitle === rightTitle) return 0;
|
||||
return leftTitle > rightTitle ? -1 : 1; // inverted is intentional
|
||||
}
|
||||
return leftScore > rightScore ? 1 : -1;
|
||||
});
|
||||
// Sort each group of results by score and then alphabetically by name.
|
||||
normalResults.sort(_orderResultsByScoreThenName);
|
||||
nonMainIndexResults.sort(_orderResultsByScoreThenName);
|
||||
|
||||
// Combine the result groups in (reverse) order.
|
||||
// Non-main index entries are typically arbitrary cross-references,
|
||||
// so display them after other results.
|
||||
let results = [...nonMainIndexResults, ...normalResults];
|
||||
|
||||
// remove duplicate search results
|
||||
// note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept
|
||||
@@ -353,14 +409,19 @@ const Search = {
|
||||
return acc;
|
||||
}, []);
|
||||
|
||||
results = results.reverse();
|
||||
return results.reverse();
|
||||
},
|
||||
|
||||
query: (query) => {
|
||||
const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query);
|
||||
const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms);
|
||||
|
||||
// for debugging
|
||||
//Search.lastresults = results.slice(); // a copy
|
||||
// console.info("search results:", Search.lastresults);
|
||||
|
||||
// print the results
|
||||
_displayNextItem(results, results.length, searchTerms);
|
||||
_displayNextItem(results, results.length, searchTerms, highlightTerms);
|
||||
},
|
||||
|
||||
/**
|
||||
@@ -424,6 +485,7 @@ const Search = {
|
||||
descr,
|
||||
score,
|
||||
filenames[match[0]],
|
||||
SearchResultKind.object,
|
||||
]);
|
||||
};
|
||||
Object.keys(objects).forEach((prefix) =>
|
||||
@@ -458,14 +520,18 @@ const Search = {
|
||||
// add support for partial matches
|
||||
if (word.length > 2) {
|
||||
const escapedWord = _escapeRegExp(word);
|
||||
Object.keys(terms).forEach((term) => {
|
||||
if (term.match(escapedWord) && !terms[word])
|
||||
arr.push({ files: terms[term], score: Scorer.partialTerm });
|
||||
});
|
||||
Object.keys(titleTerms).forEach((term) => {
|
||||
if (term.match(escapedWord) && !titleTerms[word])
|
||||
arr.push({ files: titleTerms[word], score: Scorer.partialTitle });
|
||||
});
|
||||
if (!terms.hasOwnProperty(word)) {
|
||||
Object.keys(terms).forEach((term) => {
|
||||
if (term.match(escapedWord))
|
||||
arr.push({ files: terms[term], score: Scorer.partialTerm });
|
||||
});
|
||||
}
|
||||
if (!titleTerms.hasOwnProperty(word)) {
|
||||
Object.keys(titleTerms).forEach((term) => {
|
||||
if (term.match(escapedWord))
|
||||
arr.push({ files: titleTerms[term], score: Scorer.partialTitle });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// no match but word was a required one
|
||||
@@ -488,9 +554,8 @@ const Search = {
|
||||
|
||||
// create the mapping
|
||||
files.forEach((file) => {
|
||||
if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1)
|
||||
fileMap.get(file).push(word);
|
||||
else fileMap.set(file, [word]);
|
||||
if (!fileMap.has(file)) fileMap.set(file, [word]);
|
||||
else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -531,6 +596,7 @@ const Search = {
|
||||
null,
|
||||
score,
|
||||
filenames[file],
|
||||
SearchResultKind.text,
|
||||
]);
|
||||
}
|
||||
return results;
|
||||
@@ -541,8 +607,8 @@ const Search = {
|
||||
* search summary for a given text. keywords is a list
|
||||
* of stemmed words.
|
||||
*/
|
||||
makeSearchSummary: (htmlText, keywords) => {
|
||||
const text = Search.htmlToText(htmlText);
|
||||
makeSearchSummary: (htmlText, keywords, anchor) => {
|
||||
const text = Search.htmlToText(htmlText, anchor);
|
||||
if (text === "") return null;
|
||||
|
||||
const textLower = text.toLowerCase();
|
||||
|
||||
@@ -29,14 +29,19 @@ const _highlight = (node, addItems, text, className) => {
|
||||
}
|
||||
|
||||
span.appendChild(document.createTextNode(val.substr(pos, text.length)));
|
||||
const rest = document.createTextNode(val.substr(pos + text.length));
|
||||
parent.insertBefore(
|
||||
span,
|
||||
parent.insertBefore(
|
||||
document.createTextNode(val.substr(pos + text.length)),
|
||||
rest,
|
||||
node.nextSibling
|
||||
)
|
||||
);
|
||||
node.nodeValue = val.substr(0, pos);
|
||||
/* There may be more occurrences of search term in this node. So call this
|
||||
* function recursively on the remaining fragment.
|
||||
*/
|
||||
_highlight(rest, addItems, text, className);
|
||||
|
||||
if (isInSVG) {
|
||||
const rect = document.createElementNS(
|
||||
@@ -140,5 +145,10 @@ const SphinxHighlight = {
|
||||
},
|
||||
};
|
||||
|
||||
_ready(SphinxHighlight.highlightSearchWords);
|
||||
_ready(SphinxHighlight.initEscapeListener);
|
||||
_ready(() => {
|
||||
/* Do not call highlightSearchWords() when we are on the search page.
|
||||
* It will highlight words from the *previous* search query.
|
||||
*/
|
||||
if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords();
|
||||
SphinxHighlight.initEscapeListener();
|
||||
});
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -91,9 +85,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="contributing-to-parsedmarc">
|
||||
<h1>Contributing to parsedmarc<a class="headerlink" href="#contributing-to-parsedmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Contributing to parsedmarc<a class="headerlink" href="#contributing-to-parsedmarc" title="Link to this heading"></a></h1>
|
||||
<section id="bug-reports">
|
||||
<h2>Bug reports<a class="headerlink" href="#bug-reports" title="Permalink to this heading"></a></h2>
|
||||
<h2>Bug reports<a class="headerlink" href="#bug-reports" title="Link to this heading"></a></h2>
|
||||
<p>Please report bugs on the GitHub issue tracker</p>
|
||||
<p><a class="reference external" href="https://github.com/domainaware/parsedmarc/issues">https://github.com/domainaware/parsedmarc/issues</a></p>
|
||||
</section>
|
||||
|
||||
36
davmail.html
36
davmail.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,7 +86,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="accessing-an-inbox-using-owa-ews">
|
||||
<h1>Accessing an inbox using OWA/EWS<a class="headerlink" href="#accessing-an-inbox-using-owa-ews" title="Permalink to this heading"></a></h1>
|
||||
<h1>Accessing an inbox using OWA/EWS<a class="headerlink" href="#accessing-an-inbox-using-owa-ews" title="Link to this heading"></a></h1>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Starting in 8.0.0, parsedmarc supports accessing Microsoft/Office 365
|
||||
@@ -177,7 +171,7 @@ as a local EWS/OWA IMAP gateway. It can even work where
|
||||
</pre></div>
|
||||
</div>
|
||||
<section id="running-davmail-as-a-systemd-service">
|
||||
<h2>Running DavMail as a systemd service<a class="headerlink" href="#running-davmail-as-a-systemd-service" title="Permalink to this heading"></a></h2>
|
||||
<h2>Running DavMail as a systemd service<a class="headerlink" href="#running-davmail-as-a-systemd-service" title="Link to this heading"></a></h2>
|
||||
<p>Use systemd to run <code class="docutils literal notranslate"><span class="pre">davmail</span></code> as a service.</p>
|
||||
<p>Create a system user</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>useradd<span class="w"> </span>davmail<span class="w"> </span>-r<span class="w"> </span>-s<span class="w"> </span>/bin/false
|
||||
@@ -244,7 +238,7 @@ well as the current process (newest to oldest), run:</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="configuring-parsedmarc-for-davmail">
|
||||
<h2>Configuring parsedmarc for DavMail<a class="headerlink" href="#configuring-parsedmarc-for-davmail" title="Permalink to this heading"></a></h2>
|
||||
<h2>Configuring parsedmarc for DavMail<a class="headerlink" href="#configuring-parsedmarc-for-davmail" title="Link to this heading"></a></h2>
|
||||
<p>Because you are interacting with DavMail server over the loopback
|
||||
(i.e. <code class="docutils literal notranslate"><span class="pre">127.0.0.1</span></code>), add the following options to <code class="docutils literal notranslate"><span class="pre">parsedmarc.ini</span></code>
|
||||
config file:</p>
|
||||
|
||||
97
dmarc.html
97
dmarc.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Understanding DMARC — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Understanding DMARC — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -110,39 +104,34 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="understanding-dmarc">
|
||||
<h1>Understanding DMARC<a class="headerlink" href="#understanding-dmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Understanding DMARC<a class="headerlink" href="#understanding-dmarc" title="Link to this heading"></a></h1>
|
||||
<section id="resources">
|
||||
<h2>Resources<a class="headerlink" href="#resources" title="Permalink to this heading"></a></h2>
|
||||
<h2>Resources<a class="headerlink" href="#resources" title="Link to this heading"></a></h2>
|
||||
<section id="dmarc-guides">
|
||||
<h3>DMARC guides<a class="headerlink" href="#dmarc-guides" title="Permalink to this heading"></a></h3>
|
||||
<h3>DMARC guides<a class="headerlink" href="#dmarc-guides" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference external" href="https://seanthegeek.net/459/demystifying-dmarc/">Demystifying DMARC</a> - A complete guide to SPF, DKIM, and DMARC</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="spf-and-dmarc-record-validation">
|
||||
<h3>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Permalink to this heading"></a></h3>
|
||||
<h3>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Link to this heading"></a></h3>
|
||||
<p>If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
<a class="reference external" href="https://domainaware.github.io/checkdmarc/">checkdmarc</a>.</p>
|
||||
</section>
|
||||
<section id="lookalike-domains">
|
||||
<h3>Lookalike domains<a class="headerlink" href="#lookalike-domains" title="Permalink to this heading"></a></h3>
|
||||
<h3>Lookalike domains<a class="headerlink" href="#lookalike-domains" title="Link to this heading"></a></h3>
|
||||
<p>DMARC protects against domain spoofing, not lookalike domains. for open source
|
||||
lookalike domain monitoring, check out <a class="reference external" href="https://github.com/seanthegeek/domainaware">DomainAware</a>.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="dmarc-alignment-guide">
|
||||
<h2>DMARC Alignment Guide<a class="headerlink" href="#dmarc-alignment-guide" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Alignment Guide<a class="headerlink" href="#dmarc-alignment-guide" title="Link to this heading"></a></h2>
|
||||
<p>DMARC ensures that SPF and DKM authentication mechanisms actually authenticate
|
||||
against the same domain that the end user sees.</p>
|
||||
<p>A message passes a DMARC check by passing DKIM or SPF, <strong>as long as the related
|
||||
indicators are also in alignment</strong>.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 33%" />
|
||||
<col style="width: 33%" />
|
||||
<col style="width: 33%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td></td>
|
||||
<td><p><strong>DKIM</strong></p></td>
|
||||
@@ -180,7 +169,7 @@ header</p></td>
|
||||
</table>
|
||||
</section>
|
||||
<section id="what-if-a-sender-wont-support-dkim-dmarc">
|
||||
<h2>What if a sender won’t support DKIM/DMARC?<a class="headerlink" href="#what-if-a-sender-wont-support-dkim-dmarc" title="Permalink to this heading"></a></h2>
|
||||
<h2>What if a sender won’t support DKIM/DMARC?<a class="headerlink" href="#what-if-a-sender-wont-support-dkim-dmarc" title="Link to this heading"></a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Some vendors don’t know about DMARC yet; ask about SPF and DKIM/email
|
||||
authentication.</p></li>
|
||||
@@ -200,21 +189,21 @@ spoofing of your TLD and/or any subdomain.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="what-about-mailing-lists">
|
||||
<h2>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Permalink to this heading"></a></h2>
|
||||
<h2>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Link to this heading"></a></h2>
|
||||
<p>When you deploy DMARC on your domain, you might find that messages
|
||||
relayed by mailing lists are failing DMARC, most likely because the mailing
|
||||
list is spoofing your from address, and modifying the subject,
|
||||
footer, or other part of the message, thereby breaking the
|
||||
DKIM signature.</p>
|
||||
<section id="mailing-list-best-practices">
|
||||
<h3>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Link to this heading"></a></h3>
|
||||
<p>Ideally, a mailing list should forward messages without altering the
|
||||
headers or body content at all. <a class="reference external" href="https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html">Joe Nelson</a> does a fantastic job of
|
||||
explaining exactly what mailing lists should and shouldn’t do to be
|
||||
fully DMARC compliant. Rather than repeat his fine work, here’s a
|
||||
summary:</p>
|
||||
<section id="do">
|
||||
<h4>Do<a class="headerlink" href="#do" title="Permalink to this heading"></a></h4>
|
||||
<h4>Do<a class="headerlink" href="#do" title="Link to this heading"></a></h4>
|
||||
<ul class="simple">
|
||||
<li><p>Retain headers from the original message</p></li>
|
||||
<li><p>Add <a class="reference external" href="https://tools.ietf.org/html/rfc2369">RFC 2369</a> List-Unsubscribe headers to outgoing messages, instead of
|
||||
@@ -234,7 +223,7 @@ adding unsubscribe links to the body</p></li>
|
||||
these headers.</p>
|
||||
</section>
|
||||
<section id="do-not">
|
||||
<h4>Do not<a class="headerlink" href="#do-not" title="Permalink to this heading"></a></h4>
|
||||
<h4>Do not<a class="headerlink" href="#do-not" title="Link to this heading"></a></h4>
|
||||
<ul class="simple">
|
||||
<li><p>Remove or modify any existing headers from the original message, including
|
||||
From, Date, Subject, etc.</p></li>
|
||||
@@ -251,13 +240,9 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
</section>
|
||||
<section id="mailman-2">
|
||||
<h4>Mailman 2<a class="headerlink" href="#mailman-2" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 2<a class="headerlink" href="#mailman-2" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to General Settings, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 74%" />
|
||||
<col style="width: 26%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -287,10 +272,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Non-digest options, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 66%" />
|
||||
<col style="width: 34%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -308,10 +289,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -329,16 +306,12 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
</section>
|
||||
<section id="mailman-3">
|
||||
<h4>Mailman 3<a class="headerlink" href="#mailman-3" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 3<a class="headerlink" href="#mailman-3" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to Settings> List Identity</p>
|
||||
<p>Make Subject prefix blank.</p>
|
||||
<p>Navigate to Settings> Alter Messages</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -366,10 +339,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Navigate to Settings> DMARC Mitigation</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 52%" />
|
||||
<col style="width: 48%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -393,13 +362,13 @@ command line instead, for example:</p>
|
||||
<p>Then restart mailman core.</p>
|
||||
</section>
|
||||
<section id="listserv">
|
||||
<h4>LISTSERV<a class="headerlink" href="#listserv" title="Permalink to this heading"></a></h4>
|
||||
<h4>LISTSERV<a class="headerlink" href="#listserv" title="Link to this heading"></a></h4>
|
||||
<p><a class="reference external" href="https://www.lsoft.com/news/dmarc-issue1-2018.asp">LISTSERV 16.0-2017a</a> and higher will rewrite the From header for domains
|
||||
that enforce with a DMARC quarantine or reject policy.</p>
|
||||
<p>Some additional steps are needed for Linux hosts.</p>
|
||||
</section>
|
||||
<section id="workarounds">
|
||||
<h4>Workarounds<a class="headerlink" href="#workarounds" title="Permalink to this heading"></a></h4>
|
||||
<h4>Workarounds<a class="headerlink" href="#workarounds" title="Link to this heading"></a></h4>
|
||||
<p>If a mailing list must go <strong>against</strong> best practices and
|
||||
modify the message (e.g. to add a required legal footer), the mailing
|
||||
list administrator must configure the list to replace the From address of the
|
||||
@@ -407,13 +376,9 @@ message (also known as munging) with the address of the mailing list, so they
|
||||
no longer spoof email addresses with domains protected by DMARC.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
<section id="id1">
|
||||
<h5>Mailman 2<a class="headerlink" href="#id1" title="Permalink to this heading"></a></h5>
|
||||
<h5>Mailman 2<a class="headerlink" href="#id1" title="Link to this heading"></a></h5>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -442,13 +407,9 @@ the original sender.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h5>Mailman 3<a class="headerlink" href="#id2" title="Permalink to this heading"></a></h5>
|
||||
<h5>Mailman 3<a class="headerlink" href="#id2" title="Link to this heading"></a></h5>
|
||||
<p>In the DMARC Mitigations tab of the Settings page, configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 49%" />
|
||||
<col style="width: 51%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -93,14 +87,14 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="elasticsearch-and-kibana">
|
||||
<h1>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Permalink to this heading"></a></h1>
|
||||
<h1>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Link to this heading"></a></h1>
|
||||
<p>To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Elasticsearch and Kibana 6 or later are required</p>
|
||||
</div>
|
||||
<section id="installation">
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h2>
|
||||
<p>On Debian/Ubuntu based systems, run:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>apt-transport-https
|
||||
wget<span class="w"> </span>-qO<span class="w"> </span>-<span class="w"> </span>https://artifacts.elastic.co/GPG-KEY-elasticsearch<span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>gpg<span class="w"> </span>--dearmor<span class="w"> </span>-o<span class="w"> </span>/usr/share/keyrings/elasticsearch-keyring.gpg
|
||||
@@ -243,11 +237,13 @@ page of Kibana. (Hamburger menu -> “Management” -> “Stack Management
|
||||
visualizations, which could be used to restore them if you or someone else
|
||||
breaks them, as there are no permissions/access controls in Kibana without
|
||||
the commercial <a class="reference external" href="https://www.elastic.co/products/x-pack">X-Pack</a>.</p>
|
||||
<a class="reference external image-reference" href="_static/screenshots/saved-objects.png"><img alt="A screenshot of setting the Saved Objects Stack management UI in Kibana" class="align-center" src="_images/saved-objects.png" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/confirm-overwrite.png"><img alt="A screenshot of the overwrite conformation prompt" class="align-center" src="_images/confirm-overwrite.png" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/saved-objects.png"><img alt="A screenshot of setting the Saved Objects Stack management UI in Kibana" class="align-center" src="_images/saved-objects.png" />
|
||||
</a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/confirm-overwrite.png"><img alt="A screenshot of the overwrite conformation prompt" class="align-center" src="_images/confirm-overwrite.png" />
|
||||
</a>
|
||||
</section>
|
||||
<section id="upgrading-kibana-index-patterns">
|
||||
<h2>Upgrading Kibana index patterns<a class="headerlink" href="#upgrading-kibana-index-patterns" title="Permalink to this heading"></a></h2>
|
||||
<h2>Upgrading Kibana index patterns<a class="headerlink" href="#upgrading-kibana-index-patterns" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 5.0.0 makes some changes to the way data is indexed in
|
||||
Elasticsearch. if you are upgrading from a previous release of
|
||||
<code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code>, you need to complete the following steps to replace the
|
||||
@@ -266,7 +262,7 @@ Saved Objects page</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="records-retention">
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this heading"></a></h2>
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||||
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
|
||||
index for each day to make it easy to comply with records
|
||||
retention regulations such as GDPR. For more information,
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Index — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Index — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="#" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
|
||||
37
index.html
37
index.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -36,9 +33,6 @@
|
||||
<a href="#" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -87,7 +81,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer">
|
||||
<h1>parsedmarc documentation - Open source DMARC report analyzer and visualizer<a class="headerlink" href="#parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer" title="Permalink to this heading"></a></h1>
|
||||
<h1>parsedmarc documentation - Open source DMARC report analyzer and visualizer<a class="headerlink" href="#parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer" title="Link to this heading"></a></h1>
|
||||
<p><a class="reference external" href="https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml"><img alt="BuildStatus" src="https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml/badge.svg" /></a>
|
||||
<a class="reference external" href="https://codecov.io/gh/domainaware/parsedmarc"><img alt="CodeCoverage" src="https://codecov.io/gh/domainaware/parsedmarc/branch/master/graph/badge.svg" /></a>
|
||||
<a class="reference external" href="https://pypi.org/project/parsedmarc/"><img alt="PyPIPackage" src="https://img.shields.io/pypi/v/parsedmarc.svg" /></a>
|
||||
@@ -100,14 +94,15 @@ Please consider reviewing the open <a class="reference external" href="https://g
|
||||
Assistance on the pinned issues would be particularly helpful.</p>
|
||||
<p>Thanks to all <a class="reference external" href="https://github.com/domainaware/parsedmarc/graphs/contributors">contributors</a>!</p>
|
||||
</div>
|
||||
<a class="reference external image-reference" href="_static/screenshots/dmarc-summary-charts.png"><img alt="A screenshot of DMARC summary charts in Kibana" class="align-center" src="_images/dmarc-summary-charts.png" style="width: 754.0px; height: 449.0px;" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/dmarc-summary-charts.png"><img alt="A screenshot of DMARC summary charts in Kibana" class="align-center" src="_images/dmarc-summary-charts.png" style="width: 754.0px; height: 449.0px;" />
|
||||
</a>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> is a Python module and CLI utility for parsing DMARC reports.
|
||||
When used with Elasticsearch and Kibana (or Splunk), or with OpenSearch and Grafana, it works as a self-hosted
|
||||
open source alternative to commercial DMARC report processing services such
|
||||
as Agari Brand Protection, Dmarcian, OnDMARC, ProofPoint Email Fraud Defense,
|
||||
and Valimail.</p>
|
||||
<section id="features">
|
||||
<h2>Features<a class="headerlink" href="#features" title="Permalink to this heading"></a></h2>
|
||||
<h2>Features<a class="headerlink" href="#features" title="Link to this heading"></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p>Parses draft and 1.0 standard aggregate/rua reports</p></li>
|
||||
<li><p>Parses forensic/failure/ruf reports</p></li>
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Installation — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Installation — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -99,18 +93,18 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="installation">
|
||||
<h1>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h1>
|
||||
<h1>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h1>
|
||||
<section id="prerequisites">
|
||||
<h2>Prerequisites<a class="headerlink" href="#prerequisites" title="Permalink to this heading"></a></h2>
|
||||
<h2>Prerequisites<a class="headerlink" href="#prerequisites" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> works with Python 3 only.</p>
|
||||
<section id="testing-multiple-report-analyzers">
|
||||
<h3>Testing multiple report analyzers<a class="headerlink" href="#testing-multiple-report-analyzers" title="Permalink to this heading"></a></h3>
|
||||
<h3>Testing multiple report analyzers<a class="headerlink" href="#testing-multiple-report-analyzers" title="Link to this heading"></a></h3>
|
||||
<p>If you would like to test parsedmarc and another report processing
|
||||
solution at the same time, you can have up to two <code class="docutils literal notranslate"><span class="pre">mailto</span></code> URIs in each of the rua and ruf
|
||||
tags in your DMARC record, separated by commas.</p>
|
||||
</section>
|
||||
<section id="using-a-web-proxy">
|
||||
<h3>Using a web proxy<a class="headerlink" href="#using-a-web-proxy" title="Permalink to this heading"></a></h3>
|
||||
<h3>Using a web proxy<a class="headerlink" href="#using-a-web-proxy" title="Link to this heading"></a></h3>
|
||||
<p>If your system is behind a web proxy, you need to configure your system
|
||||
to use that proxy. To do this, edit <code class="docutils literal notranslate"><span class="pre">/etc/environment</span></code> and add your
|
||||
proxy details there, for example:</p>
|
||||
@@ -128,7 +122,7 @@ proxy details there, for example:</p>
|
||||
<p>This will set the proxy up for use system-wide, including for <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code>.</p>
|
||||
</section>
|
||||
<section id="using-microsoft-exchange">
|
||||
<h3>Using Microsoft Exchange<a class="headerlink" href="#using-microsoft-exchange" title="Permalink to this heading"></a></h3>
|
||||
<h3>Using Microsoft Exchange<a class="headerlink" href="#using-microsoft-exchange" title="Link to this heading"></a></h3>
|
||||
<p>If your mail server is Microsoft Exchange, ensure that it is patched to at
|
||||
least:</p>
|
||||
<ul class="simple">
|
||||
@@ -138,7 +132,7 @@ least:</p>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="geoipupdate-setup">
|
||||
<h3>geoipupdate setup<a class="headerlink" href="#geoipupdate-setup" title="Permalink to this heading"></a></h3>
|
||||
<h3>geoipupdate setup<a class="headerlink" href="#geoipupdate-setup" title="Link to this heading"></a></h3>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Starting in <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 7.1.0, a static copy of the
|
||||
@@ -210,7 +204,7 @@ job or scheduled task.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="installing-parsedmarc">
|
||||
<h2>Installing parsedmarc<a class="headerlink" href="#installing-parsedmarc" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installing parsedmarc<a class="headerlink" href="#installing-parsedmarc" title="Link to this heading"></a></h2>
|
||||
<p>On Debian or Ubuntu systems, run:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>python3-pip<span class="w"> </span>python3-virtualenv<span class="w"> </span>python3-dev<span class="w"> </span>libxml2-dev<span class="w"> </span>libxslt-dev
|
||||
</pre></div>
|
||||
@@ -245,7 +239,7 @@ explicitly tell <code class="docutils literal notranslate"><span class="pre">vir
|
||||
</div>
|
||||
</section>
|
||||
<section id="optional-dependencies">
|
||||
<h2>Optional dependencies<a class="headerlink" href="#optional-dependencies" title="Permalink to this heading"></a></h2>
|
||||
<h2>Optional dependencies<a class="headerlink" href="#optional-dependencies" title="Link to this heading"></a></h2>
|
||||
<p>If you would like to be able to parse emails saved from Microsoft
|
||||
Outlook (i.e. OLE .msg files), install <code class="docutils literal notranslate"><span class="pre">msgconvert</span></code>:</p>
|
||||
<p>On Debian or Ubuntu systems, run:</p>
|
||||
|
||||
36
kibana.html
36
kibana.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,7 +86,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="using-the-kibana-dashboards">
|
||||
<h1>Using the Kibana dashboards<a class="headerlink" href="#using-the-kibana-dashboards" title="Permalink to this heading"></a></h1>
|
||||
<h1>Using the Kibana dashboards<a class="headerlink" href="#using-the-kibana-dashboards" title="Link to this heading"></a></h1>
|
||||
<p>The Kibana DMARC dashboards are a human-friendly way to understand the
|
||||
results from incoming DMARC reports.</p>
|
||||
<div class="admonition note">
|
||||
@@ -101,7 +95,7 @@ results from incoming DMARC reports.</p>
|
||||
click on the Dashboard link on the left side menu of Kibana.</p>
|
||||
</div>
|
||||
<section id="dmarc-summary">
|
||||
<h2>DMARC Summary<a class="headerlink" href="#dmarc-summary" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Summary<a class="headerlink" href="#dmarc-summary" title="Link to this heading"></a></h2>
|
||||
<p>As the name suggests, this dashboard is the best place to start
|
||||
reviewing your aggregate DMARC data.</p>
|
||||
<p>Across the top of the dashboard, three pie charts display the percentage of
|
||||
@@ -158,7 +152,7 @@ the DMARC Summary dashboard. To view failures only, use the pie chart.</p>
|
||||
filters by clicking on Add Filter at the upper right of the page.</p>
|
||||
</section>
|
||||
<section id="dmarc-forensic-samples">
|
||||
<h2>DMARC Forensic Samples<a class="headerlink" href="#dmarc-forensic-samples" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Forensic Samples<a class="headerlink" href="#dmarc-forensic-samples" title="Link to this heading"></a></h2>
|
||||
<p>The DMARC Forensic Samples dashboard contains information on DMARC forensic
|
||||
reports (also known as failure reports or ruf reports). These reports contain
|
||||
samples of emails that have failed to pass DMARC.</p>
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>What about mailing lists? — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>What about mailing lists? — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -35,9 +32,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -86,21 +80,21 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="what-about-mailing-lists">
|
||||
<h1>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Permalink to this heading"></a></h1>
|
||||
<h1>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Link to this heading"></a></h1>
|
||||
<p>When you deploy DMARC on your domain, you might find that messages
|
||||
relayed by mailing lists are failing DMARC, most likely because the mailing
|
||||
list is spoofing your from address, and modifying the subject,
|
||||
footer, or other part of the message, thereby breaking the
|
||||
DKIM signature.</p>
|
||||
<section id="mailing-list-best-practices">
|
||||
<h2>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Permalink to this heading"></a></h2>
|
||||
<h2>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Link to this heading"></a></h2>
|
||||
<p>Ideally, a mailing list should forward messages without altering the
|
||||
headers or body content at all. <a class="reference external" href="https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html">Joe Nelson</a> does a fantastic job of
|
||||
explaining exactly what mailing lists should and shouldn’t do to be
|
||||
fully DMARC compliant. Rather than repeat his fine work, here’s a
|
||||
summary:</p>
|
||||
<section id="do">
|
||||
<h3>Do<a class="headerlink" href="#do" title="Permalink to this heading"></a></h3>
|
||||
<h3>Do<a class="headerlink" href="#do" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p>Retain headers from the original message</p></li>
|
||||
<li><p>Add <a class="reference external" href="https://tools.ietf.org/html/rfc2369">RFC 2369</a> List-Unsubscribe headers to outgoing messages, instead of
|
||||
@@ -120,7 +114,7 @@ adding unsubscribe links to the body</p></li>
|
||||
these headers.</p>
|
||||
</section>
|
||||
<section id="do-not">
|
||||
<h3>Do not<a class="headerlink" href="#do-not" title="Permalink to this heading"></a></h3>
|
||||
<h3>Do not<a class="headerlink" href="#do-not" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p>Remove or modify any existing headers from the original message, including
|
||||
From, Date, Subject, etc.</p></li>
|
||||
@@ -137,13 +131,9 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
</section>
|
||||
<section id="mailman-2">
|
||||
<h3>Mailman 2<a class="headerlink" href="#mailman-2" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailman 2<a class="headerlink" href="#mailman-2" title="Link to this heading"></a></h3>
|
||||
<p>Navigate to General Settings, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 74%" />
|
||||
<col style="width: 26%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -173,10 +163,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Non-digest options, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 66%" />
|
||||
<col style="width: 34%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -194,10 +180,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -215,16 +197,12 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
</section>
|
||||
<section id="mailman-3">
|
||||
<h3>Mailman 3<a class="headerlink" href="#mailman-3" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailman 3<a class="headerlink" href="#mailman-3" title="Link to this heading"></a></h3>
|
||||
<p>Navigate to Settings> List Identity</p>
|
||||
<p>Make Subject prefix blank.</p>
|
||||
<p>Navigate to Settings> Alter Messages</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -252,10 +230,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Navigate to Settings> DMARC Mitigation</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 52%" />
|
||||
<col style="width: 48%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -279,13 +253,13 @@ command line instead, for example:</p>
|
||||
<p>Then restart mailman core.</p>
|
||||
</section>
|
||||
<section id="listserv">
|
||||
<h3>LISTSERV<a class="headerlink" href="#listserv" title="Permalink to this heading"></a></h3>
|
||||
<h3>LISTSERV<a class="headerlink" href="#listserv" title="Link to this heading"></a></h3>
|
||||
<p><a class="reference external" href="https://www.lsoft.com/news/dmarc-issue1-2018.asp">LISTSERV 16.0-2017a</a> and higher will rewrite the From header for domains
|
||||
that enforce with a DMARC quarantine or reject policy.</p>
|
||||
<p>Some additional steps are needed for Linux hosts.</p>
|
||||
</section>
|
||||
<section id="workarounds">
|
||||
<h3>Workarounds<a class="headerlink" href="#workarounds" title="Permalink to this heading"></a></h3>
|
||||
<h3>Workarounds<a class="headerlink" href="#workarounds" title="Link to this heading"></a></h3>
|
||||
<p>If a mailing list must go <strong>against</strong> best practices and
|
||||
modify the message (e.g. to add a required legal footer), the mailing
|
||||
list administrator must configure the list to replace the From address of the
|
||||
@@ -293,13 +267,9 @@ message (also known as munging) with the address of the mailing list, so they
|
||||
no longer spoof email addresses with domains protected by DMARC.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
<section id="id1">
|
||||
<h4>Mailman 2<a class="headerlink" href="#id1" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 2<a class="headerlink" href="#id1" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -328,13 +298,9 @@ the original sender.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h4>Mailman 3<a class="headerlink" href="#id2" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 3<a class="headerlink" href="#id2" title="Link to this heading"></a></h4>
|
||||
<p>In the DMARC Mitigations tab of the Settings page, configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 49%" />
|
||||
<col style="width: 51%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
|
||||
BIN
objects.inv
BIN
objects.inv
Binary file not shown.
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,15 +86,15 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="opensearch-and-grafana">
|
||||
<h1>OpenSearch and Grafana<a class="headerlink" href="#opensearch-and-grafana" title="Permalink to this heading"></a></h1>
|
||||
<h1>OpenSearch and Grafana<a class="headerlink" href="#opensearch-and-grafana" title="Link to this heading"></a></h1>
|
||||
<p>To set up visual dashboards of DMARC data, install OpenSearch and Grafana.</p>
|
||||
<section id="installation">
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h2>
|
||||
<p>OpenSearch: <a class="reference external" href="https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/">https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/</a>
|
||||
Grafana: <a class="reference external" href="https://grafana.com/docs/grafana/latest/setup-grafana/installation/">https://grafana.com/docs/grafana/latest/setup-grafana/installation/</a></p>
|
||||
</section>
|
||||
<section id="records-retention">
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this heading"></a></h2>
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||||
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
|
||||
index for each day to make it easy to comply with records
|
||||
retention regulations such as GDPR.</p>
|
||||
|
||||
46
output.html
46
output.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Sample outputs — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Sample outputs — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -101,9 +95,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="sample-outputs">
|
||||
<h1>Sample outputs<a class="headerlink" href="#sample-outputs" title="Permalink to this heading"></a></h1>
|
||||
<h1>Sample outputs<a class="headerlink" href="#sample-outputs" title="Link to this heading"></a></h1>
|
||||
<section id="sample-aggregate-report-output">
|
||||
<h2>Sample aggregate report output<a class="headerlink" href="#sample-aggregate-report-output" title="Permalink to this heading"></a></h2>
|
||||
<h2>Sample aggregate report output<a class="headerlink" href="#sample-aggregate-report-output" title="Link to this heading"></a></h2>
|
||||
<p>Here are the results from parsing the <a class="reference external" href="https://dmarc.org/wiki/FAQ#I_need_to_implement_aggregate_reports.2C_what_do_they_look_like.3F">example</a>
|
||||
report from the <a class="reference external" href="http://dmarc.org">dmarc.org</a> wiki. It’s actually an older draft of
|
||||
the 1.0 report schema standardized in
|
||||
@@ -112,7 +106,7 @@ This draft schema is still in wide use.</p>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> produces consistent, normalized output, regardless
|
||||
of the report schema.</p>
|
||||
<section id="json-aggregate-report">
|
||||
<h3>JSON aggregate report<a class="headerlink" href="#json-aggregate-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON aggregate report<a class="headerlink" href="#json-aggregate-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"xml_schema"</span><span class="p">:</span><span class="w"> </span><span class="s2">"draft"</span><span class="p">,</span>
|
||||
<span class="w"> </span><span class="nt">"report_metadata"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||||
@@ -181,7 +175,7 @@ of the report schema.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="csv-aggregate-report">
|
||||
<h3>CSV aggregate report<a class="headerlink" href="#csv-aggregate-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>CSV aggregate report<a class="headerlink" href="#csv-aggregate-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
|
||||
draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,,example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
|
||||
</pre></div>
|
||||
@@ -189,11 +183,11 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</section>
|
||||
</section>
|
||||
<section id="sample-forensic-report-output">
|
||||
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Permalink to this heading"></a></h2>
|
||||
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Link to this heading"></a></h2>
|
||||
<p>Thanks to GitHub user <a class="reference external" href="https://github.com/xennn">xennn</a> for the anonymized
|
||||
<a class="reference external" href="https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml">forensic report email sample</a>.</p>
|
||||
<section id="json-forensic-report">
|
||||
<h3>JSON forensic report<a class="headerlink" href="#json-forensic-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON forensic report<a class="headerlink" href="#json-forensic-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"feedback_type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"auth-failure"</span><span class="p">,</span>
|
||||
<span class="w"> </span><span class="nt">"user_agent"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Lua/1.0"</span><span class="p">,</span>
|
||||
@@ -282,14 +276,14 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</div>
|
||||
</section>
|
||||
<section id="csv-forensic-report">
|
||||
<h3>CSV forensic report<a class="headerlink" href="#csv-forensic-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>CSV forensic report<a class="headerlink" href="#csv-forensic-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
|
||||
auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
|
||||
</pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="json-smtp-tls-report">
|
||||
<h3>JSON SMTP TLS report<a class="headerlink" href="#json-smtp-tls-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON SMTP TLS report<a class="headerlink" href="#json-smtp-tls-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
|
||||
<span class="w"> </span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"organization_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Example Inc."</span><span class="p">,</span>
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Python Module Index — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Python Module Index — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
|
||||
28
search.html
28
search.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Search — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Search — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<script src="_static/searchtools.js"></script>
|
||||
<script src="_static/language_data.js"></script>
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="#" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
|
||||
File diff suppressed because one or more lines are too long
32
splunk.html
32
splunk.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Splunk — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Splunk — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,7 +82,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="splunk">
|
||||
<h1>Splunk<a class="headerlink" href="#splunk" title="Permalink to this heading"></a></h1>
|
||||
<h1>Splunk<a class="headerlink" href="#splunk" title="Link to this heading"></a></h1>
|
||||
<p>Starting in version 4.3.0 <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> supports sending aggregate and/or
|
||||
forensic DMARC data to a Splunk <a class="reference external" href="http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC">HTTP Event collector (HEC)</a>.</p>
|
||||
<p>The project repository contains <a class="reference external" href="https://github.com/domainaware/parsedmarc/tree/master/splunk">XML files</a> for premade Splunk
|
||||
|
||||
60
usage.html
60
usage.html
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using parsedmarc — parsedmarc 8.15.0 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Using parsedmarc — parsedmarc 8.15.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=80d5e7a1" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=6add7570"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.15.0
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -93,9 +87,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="using-parsedmarc">
|
||||
<h1>Using parsedmarc<a class="headerlink" href="#using-parsedmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Using parsedmarc<a class="headerlink" href="#using-parsedmarc" title="Link to this heading"></a></h1>
|
||||
<section id="cli-help">
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Permalink to this heading"></a></h2>
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Link to this heading"></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME]
|
||||
[--forensic-json-filename FORENSIC_JSON_FILENAME]
|
||||
@@ -147,7 +141,7 @@ configuration file, described below.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="configuration-file">
|
||||
<h2>Configuration file<a class="headerlink" href="#configuration-file" title="Permalink to this heading"></a></h2>
|
||||
<h2>Configuration file<a class="headerlink" href="#configuration-file" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> can be configured by supplying the path to an INI file</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>parsedmarc<span class="w"> </span>-c<span class="w"> </span>/etc/parsedmarc.ini
|
||||
</pre></div>
|
||||
@@ -193,6 +187,12 @@ configuration file, described below.</p>
|
||||
<span class="na">host</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">logger</span>
|
||||
<span class="na">port</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">12201</span>
|
||||
<span class="na">mode</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">tcp</span>
|
||||
|
||||
<span class="k">[webhook]</span>
|
||||
<span class="na">aggregate_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://aggregate_url.example.com</span>
|
||||
<span class="na">forensic_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://forensic_url.example.com</span>
|
||||
<span class="na">smtp_tls_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">https://smtp_tls_url.example.com</span>
|
||||
<span class="na">timeout</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">60</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>The full set of configuration options are:</p>
|
||||
@@ -221,6 +221,8 @@ or DNS</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">reverse_dns_map_url</span></code> - Overrides the default download URL for the reverse DNS map</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">nameservers</span></code> - str: A comma separated list of
|
||||
DNS resolvers (Default: <code class="docutils literal notranslate"><span class="pre">[Cloudflare's</span> <span class="pre">public</span> <span class="pre">resolvers]</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dns_test_address</span></code> - str: a dummy address used for DNS pre-flight checks
|
||||
(Default: 1.1.1.1)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">dns_timeout</span></code> - float: DNS timeout period</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">debug</span></code> - bool: Print debugging messages</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">silent</span></code> - bool: Only print errors (Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
|
||||
@@ -489,6 +491,20 @@ When <code class="docutils literal notranslate"><span class="pre">False</span></
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">mode</span></code> - str: The GELF transport type to use. Valid modes: <code class="docutils literal notranslate"><span class="pre">tcp</span></code>, <code class="docutils literal notranslate"><span class="pre">udp</span></code>, <code class="docutils literal notranslate"><span class="pre">tls</span></code></p></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir</span></code></p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">reports_folder</span></code> - str: Full path for mailbox maidir location (Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_create</span></code> - bool: Create maildir if not present (Default: False)</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">webhook</span></code> - Post the individual reports to a webhook url with the report as the JSON body</p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">aggregate_url</span></code> - str: URL of the webhook which should receive the aggregate reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">forensic_url</span></code> - str: URL of the webhook which should receive the forensic reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">smtp_tls_url</span></code> - str: URL of the webhook which should receive the smtp_tls reports</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">timeout</span></code> - int: Interval in which the webhook call should timeout</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
<div class="admonition warning">
|
||||
<p class="admonition-title">Warning</p>
|
||||
@@ -551,7 +567,7 @@ Check current usage (from Management -> Dev Tools -> Console):</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="running-parsedmarc-as-a-systemd-service">
|
||||
<h2>Running parsedmarc as a systemd service<a class="headerlink" href="#running-parsedmarc-as-a-systemd-service" title="Permalink to this heading"></a></h2>
|
||||
<h2>Running parsedmarc as a systemd service<a class="headerlink" href="#running-parsedmarc-as-a-systemd-service" title="Link to this heading"></a></h2>
|
||||
<p>Use systemd to run <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> as a service and process reports as
|
||||
they arrive.</p>
|
||||
<p>Protect the <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> configuration file from prying eyes</p>
|
||||
|
||||
Reference in New Issue
Block a user