mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-05-16 17:05:24 +00:00
Update docs
This commit is contained in:
Sean Whalen
+12
-18
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>Overview: module code — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -98,7 +92,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+185
-109
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>parsedmarc — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -89,44 +83,45 @@
|
||||
|
||||
<span class="sd">"""A Python package for parsing DMARC reports"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">binascii</span>
|
||||
<span class="kn">import</span> <span class="nn">email</span>
|
||||
<span class="kn">import</span> <span class="nn">email.utils</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">import</span> <span class="nn">mailbox</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">import</span> <span class="nn">shutil</span>
|
||||
<span class="kn">import</span> <span class="nn">tempfile</span>
|
||||
<span class="kn">import</span> <span class="nn">xml.parsers.expat</span> <span class="k">as</span> <span class="nn">expat</span>
|
||||
<span class="kn">import</span> <span class="nn">zipfile</span>
|
||||
<span class="kn">import</span> <span class="nn">zlib</span>
|
||||
<span class="kn">from</span> <span class="nn">base64</span> <span class="kn">import</span> <span class="n">b64decode</span>
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span> <span class="nn">csv</span> <span class="kn">import</span> <span class="n">DictWriter</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span> <span class="nn">io</span> <span class="kn">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Callable</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">binascii</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">email</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">email.utils</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xml.parsers.expat</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">expat</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">zipfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">zlib</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">base64</span><span class="w"> </span><span class="kn">import</span> <span class="n">b64decode</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">csv</span><span class="w"> </span><span class="kn">import</span> <span class="n">DictWriter</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">,</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">io</span><span class="w"> </span><span class="kn">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">mailparser</span>
|
||||
<span class="kn">import</span> <span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span> <span class="nn">expiringdict</span> <span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">etree</span>
|
||||
<span class="kn">from</span> <span class="nn">mailsuite.smtp</span> <span class="kn">import</span> <span class="n">send_email</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">etree</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">mailsuite.smtp</span><span class="w"> </span><span class="kn">import</span> <span class="n">send_email</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.mail</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.mail</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">IMAPConnection</span><span class="p">,</span>
|
||||
<span class="n">MSGraphConnection</span><span class="p">,</span>
|
||||
<span class="n">GmailConnection</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"8.18.1"</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">__version__</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_email</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
|
||||
@@ -145,27 +140,42 @@
|
||||
<span class="n">REVERSE_DNS_MAP</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ParserError"><a class="viewcode-back" href="../api.html#parsedmarc.ParserError">[docs]</a><span class="k">class</span> <span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ParserError">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.ParserError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised whenever the parser fails for some reason"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidDMARCReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidDMARCReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidDMARCReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidDMARCReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidSMTPTLSReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidSMTPTLSReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid SMTP TLS report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidAggregateReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC aggregate report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC forensic report is encountered"""</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_report_record</span><span class="p">(</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_report_record</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
@@ -328,7 +338,7 @@
|
||||
<span class="k">return</span> <span class="n">new_record</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">):</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">new_failure_details</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">"result-type"</span><span class="p">],</span>
|
||||
@@ -364,7 +374,7 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">):</span>
|
||||
<span class="n">policy_types</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"tlsa"</span><span class="p">,</span> <span class="s2">"sts"</span><span class="p">,</span> <span class="s2">"no-policy-found"</span><span class="p">]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-domain"</span><span class="p">]</span>
|
||||
@@ -401,7 +411,9 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_smtp_tls_report_json"><a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a><span class="k">def</span> <span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="parse_smtp_tls_report_json">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses and validates an SMTP TLS report"""</span>
|
||||
<span class="n">required_fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"organization-name"</span><span class="p">,</span>
|
||||
@@ -440,7 +452,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Converts one oor more parsed SMTP TLS reports into a list of single</span>
|
||||
<span class="sd"> layer OrderedDict objects suitable for use in a CSV"""</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="ow">is</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
||||
@@ -475,7 +490,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed SMTP TLS reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -520,7 +538,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_xml"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_xml">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
<span class="n">xml</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
@@ -706,7 +727,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_report"><a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a><span class="k">def</span> <span class="nf">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="extract_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Extracts text from a zip or gzip file, as a base64-encoded string,</span>
|
||||
<span class="sd"> file-like object, or bytes.</span>
|
||||
@@ -760,7 +784,10 @@
|
||||
<span class="k">return</span> <span class="n">report</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_report_from_file_path"><a class="viewcode-back" href="../api.html#parsedmarc.extract_report_from_file_path">[docs]</a><span class="k">def</span> <span class="nf">extract_report_from_file_path</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="extract_report_from_file_path">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report_from_file_path">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report_from_file_path</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Extracts report from a file at the given file_path"""</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">report_file</span><span class="p">:</span>
|
||||
@@ -769,7 +796,10 @@
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"File was not found"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_file">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_file</span><span class="p">(</span>
|
||||
<span class="n">_input</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -817,7 +847,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed aggregate reports to list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
@@ -830,7 +863,7 @@
|
||||
<span class="sd"> format</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="ow">is</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
||||
@@ -935,7 +968,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed aggregate reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -1000,7 +1036,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">,</span>
|
||||
@@ -1140,7 +1179,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
@@ -1176,7 +1218,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -1228,7 +1273,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_report_email"><a class="viewcode-back" href="../api.html#parsedmarc.parse_report_email">[docs]</a><span class="k">def</span> <span class="nf">parse_report_email</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_report_email">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_email">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_email</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -1270,7 +1318,7 @@
|
||||
<span class="n">input_</span> <span class="o">=</span> <span class="n">input_</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
||||
<span class="n">msg</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">msg_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">())</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="s2">"Date"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Date"</span><span class="p">])</span>
|
||||
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
@@ -1286,12 +1334,16 @@
|
||||
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="p">[</span><span class="n">payload</span><span class="p">]</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"multipart/"</span><span class="p">):</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/html"</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"Feedback-Type"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
@@ -1302,13 +1354,12 @@
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">n"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/rfc822-headers"</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/rfc822"</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"application/tlsrpt+json"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"{"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">))</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
@@ -1320,7 +1371,6 @@
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/plain"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"A message claiming to be from you has failed"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
@@ -1347,13 +1397,14 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">ns</span> <span class="o">=</span> <span class="n">nameservers</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
||||
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
<span class="n">payload</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
||||
@@ -1361,25 +1412,24 @@
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
||||
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"aggregate"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">aggregate_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">InvalidAggregateReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"aggregate DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid DMARC report'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
@@ -1420,7 +1470,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_report_file"><a class="viewcode-back" href="../api.html#parsedmarc.parse_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_report_file</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_report_file">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_file">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_file</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
@@ -1499,7 +1552,10 @@
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mbox"><a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_mbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mbox</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
@@ -1584,7 +1640,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox"><a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
||||
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">"Archive"</span><span class="p">,</span>
|
||||
@@ -1666,7 +1725,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">since</span><span class="p">:</span>
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="mi">1440</span> <span class="c1"># default one day</span>
|
||||
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\d+[mhd]$"</span><span class="p">,</span> <span class="n">since</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\d+[mhdw]$"</span><span class="p">,</span> <span class="n">since</span><span class="p">):</span>
|
||||
<span class="n">s</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\d+)"</span><span class="p">,</span> <span class="n">since</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"m"</span><span class="p">:</span>
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
@@ -1688,16 +1747,20 @@
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are </span><span class="se">\</span>
|
||||
<span class="s2"> considered for IMAP conections. Examples: 2d or 1w"</span>
|
||||
<span class="s2"> considered for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">MSGraphConnection</span><span class="p">):</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">)</span>
|
||||
<span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">GmailConnection</span><span class="p">):</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">%s</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
@@ -1897,7 +1960,10 @@
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="watch_inbox"><a class="viewcode-back" href="../api.html#parsedmarc.watch_inbox">[docs]</a><span class="k">def</span> <span class="nf">watch_inbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="watch_inbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.watch_inbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">watch_inbox</span><span class="p">(</span>
|
||||
<span class="n">mailbox_connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">callback</span><span class="p">:</span> <span class="n">Callable</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
||||
@@ -1941,7 +2007,7 @@
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">check_callback</span><span class="p">(</span><span class="n">connection</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">check_callback</span><span class="p">(</span><span class="n">connection</span><span class="p">):</span>
|
||||
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
||||
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
||||
@@ -1965,7 +2031,8 @@
|
||||
<span class="n">mailbox_connection</span><span class="o">.</span><span class="n">watch</span><span class="p">(</span><span class="n">check_callback</span><span class="o">=</span><span class="n">check_callback</span><span class="p">,</span> <span class="n">check_timeout</span><span class="o">=</span><span class="n">check_timeout</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">append_json</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_json</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -1988,7 +2055,7 @@
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">output_json</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">csv</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">csv</span><span class="p">):</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># strip the headers from the CSV</span>
|
||||
@@ -2000,7 +2067,9 @@
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_output"><a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a><span class="k">def</span> <span class="nf">save_output</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="save_output">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_output</span><span class="p">(</span>
|
||||
<span class="n">results</span><span class="p">,</span>
|
||||
<span class="n">output_directory</span><span class="o">=</span><span class="s2">"output"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_json_filename</span><span class="o">=</span><span class="s2">"aggregate.json"</span><span class="p">,</span>
|
||||
@@ -2086,7 +2155,10 @@
|
||||
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_report_zip"><a class="viewcode-back" href="../api.html#parsedmarc.get_report_zip">[docs]</a><span class="k">def</span> <span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_report_zip">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_report_zip">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Creates a zip file of parsed report output</span>
|
||||
|
||||
@@ -2097,7 +2169,7 @@
|
||||
<span class="sd"> bytes: zip file bytes</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
||||
<span class="n">subdir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_dirs</span><span class="p">,</span> <span class="n">subdir_files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">):</span>
|
||||
<span class="k">for</span> <span class="n">subdir_file</span> <span class="ow">in</span> <span class="n">subdir_files</span><span class="p">:</span>
|
||||
@@ -2131,7 +2203,10 @@
|
||||
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="email_results"><a class="viewcode-back" href="../api.html#parsedmarc.email_results">[docs]</a><span class="k">def</span> <span class="nf">email_results</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="email_results">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.email_results">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">email_results</span><span class="p">(</span>
|
||||
<span class="n">results</span><span class="p">,</span>
|
||||
<span class="n">host</span><span class="p">,</span>
|
||||
<span class="n">mail_from</span><span class="p">,</span>
|
||||
@@ -2199,6 +2274,7 @@
|
||||
<span class="n">attachments</span><span class="o">=</span><span class="n">attachments</span><span class="p">,</span>
|
||||
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -2208,7 +2284,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,10 +82,10 @@
|
||||
<h1>Source code for parsedmarc.elastic</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
@@ -105,23 +99,26 @@
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ElasticsearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a><span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ElasticsearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an Elasticsearch error occurs"""</span></div>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -131,20 +128,20 @@
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -175,36 +172,36 @@
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -219,21 +216,21 @@
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
@@ -241,8 +238,8 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -264,7 +261,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -274,7 +271,7 @@
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -283,7 +280,7 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -309,8 +306,8 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -321,7 +318,7 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
@@ -342,17 +339,22 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -364,7 +366,7 @@
|
||||
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
||||
<span class="sd"> username (str): The username to use for authentication</span>
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> apiKey (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -379,12 +381,15 @@
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create Elasticsearch indexes</span>
|
||||
|
||||
@@ -407,7 +412,10 @@
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -453,7 +461,10 @@
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -604,7 +615,10 @@
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -640,8 +654,8 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
@@ -650,20 +664,35 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.from"</span><span class="p">:</span> <span class="n">from_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the FROM header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">from_</span><span class="p">[</span><span class="s2">"sample.headers.from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">from_query</span>
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.to"</span><span class="p">:</span> <span class="n">to_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the TO header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">to_</span><span class="p">[</span><span class="s2">"sample.headers.to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">to_query</span>
|
||||
<span class="k">if</span> <span class="s2">"subject"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.subject"</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
|
||||
@@ -677,7 +706,9 @@
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
@@ -713,7 +744,7 @@
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date</span><span class="p">,</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
@@ -754,7 +785,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -903,6 +937,7 @@
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -912,7 +947,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,9 +82,9 @@
|
||||
<h1>Source code for parsedmarc.opensearch</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
@@ -105,23 +99,26 @@
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="OpenSearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a><span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="OpenSearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an OpenSearch error occurs"""</span></div>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -131,20 +128,20 @@
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -175,36 +172,36 @@
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -219,21 +216,21 @@
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
@@ -241,8 +238,8 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -264,7 +261,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -274,7 +271,7 @@
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -283,7 +280,7 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -309,8 +306,8 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -321,7 +318,7 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
@@ -342,17 +339,22 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -364,7 +366,7 @@
|
||||
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
||||
<span class="sd"> username (str): The username to use for authentication</span>
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> apiKey (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -379,12 +381,15 @@
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create OpenSearch indexes</span>
|
||||
|
||||
@@ -407,7 +412,10 @@
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -453,7 +461,10 @@
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -604,7 +615,10 @@
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -640,8 +654,8 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
@@ -650,20 +664,35 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.from"</span><span class="p">:</span> <span class="n">from_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the FROM header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">from_</span><span class="p">[</span><span class="s2">"sample.headers.from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">from_query</span>
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.to"</span><span class="p">:</span> <span class="n">to_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the TO header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">to_</span><span class="p">[</span><span class="s2">"sample.headers.to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">to_query</span>
|
||||
<span class="k">if</span> <span class="s2">"subject"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.subject"</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
|
||||
@@ -677,7 +706,9 @@
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
@@ -713,7 +744,7 @@
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date</span><span class="p">,</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
@@ -754,7 +785,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -903,6 +937,7 @@
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -912,7 +947,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -86,31 +80,36 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<h1>Source code for parsedmarc.splunk</h1><div class="highlight"><pre>
|
||||
<span></span><span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
|
||||
<span class="kn">import</span> <span class="nn">socket</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span></span><span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlparse</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">socket</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">urllib3</span>
|
||||
<span class="kn">import</span> <span class="nn">requests</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib3</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">__version__</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
|
||||
|
||||
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SplunkError"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a><span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="SplunkError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a Splunk API error occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a><span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""A client for a Splunk HTTP Events Collector (HEC)"""</span>
|
||||
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -139,11 +138,13 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="s2">"parsedmarc/</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">),</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">,</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -201,7 +202,10 @@
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
|
||||
@@ -235,7 +239,10 @@
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -268,7 +275,9 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div></div>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
</div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -278,7 +287,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+156
-88
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,60 +82,75 @@
|
||||
<h1>Source code for parsedmarc.utils</h1><div class="highlight"><pre>
|
||||
<span></span><span class="sd">"""Utility functions that might be useful for other projects"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">logging</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">import</span> <span class="nn">tempfile</span>
|
||||
<span class="kn">import</span> <span class="nn">subprocess</span>
|
||||
<span class="kn">import</span> <span class="nn">shutil</span>
|
||||
<span class="kn">import</span> <span class="nn">mailparser</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">import</span> <span class="nn">hashlib</span>
|
||||
<span class="kn">import</span> <span class="nn">base64</span>
|
||||
<span class="kn">import</span> <span class="nn">mailbox</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">import</span> <span class="nn">csv</span>
|
||||
<span class="kn">import</span> <span class="nn">io</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">subprocess</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">csv</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib.resources</span> <span class="k">as</span> <span class="nn">pkg_resources</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
||||
<span class="c1"># Try backported to PY<37 `importlib_resources`</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib_resources</span> <span class="k">as</span> <span class="nn">pkg_resources</span>
|
||||
<span class="c1"># Try backported to PY<3 `importlib_resources`</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">dateutil.parser</span> <span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">parse_date</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.reversename</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.resolver</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.exception</span>
|
||||
<span class="kn">import</span> <span class="nn">geoip2.database</span>
|
||||
<span class="kn">import</span> <span class="nn">geoip2.errors</span>
|
||||
<span class="kn">import</span> <span class="nn">publicsuffixlist</span>
|
||||
<span class="kn">import</span> <span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.dbip</span>
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.maps</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">parse_date</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.reversename</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.resolver</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.exception</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.database</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.errors</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">publicsuffixlist</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.dbip</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.maps</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
<span class="n">psl_overrides_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"psl_overrides.txt"</span><span class="p">))</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_overrides_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">psl_overrides</span> <span class="o">=</span> <span class="p">[</span><span class="n">line</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()]</span>
|
||||
<span class="k">while</span> <span class="s2">""</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
|
||||
<span class="n">psl_overrides</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s2">""</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="EmailParserError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a><span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="EmailParserError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error parsing the email occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="DownloadError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a><span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="DownloadError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error occurs when downloading a file"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="decode_base64"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="decode_base64">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Decodes a base64 string, with padding being optional</span>
|
||||
|
||||
@@ -159,13 +168,17 @@
|
||||
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Gets the base domain name for the given domain</span>
|
||||
|
||||
<span class="sd"> .. note::</span>
|
||||
<span class="sd"> Results are based on a list of public domain suffixes at</span>
|
||||
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat.</span>
|
||||
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat and overrides included in</span>
|
||||
<span class="sd"> parsedmarc.resources.maps.psl_overrides.txt</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> domain (str): A domain or subdomain</span>
|
||||
@@ -174,11 +187,18 @@
|
||||
<span class="sd"> str: The base domain of the given domain</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span></div>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">publicsuffix</span> <span class="o">=</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">domain</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">override</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">override</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"-"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">publicsuffix</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="query_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a><span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="query_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Queries DNS</span>
|
||||
|
||||
@@ -239,7 +259,10 @@
|
||||
<span class="k">return</span> <span class="n">records</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a><span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
|
||||
|
||||
@@ -267,7 +290,10 @@
|
||||
<span class="k">return</span> <span class="n">hostname</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``datetime`` object</span>
|
||||
|
||||
@@ -280,7 +306,10 @@
|
||||
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
|
||||
|
||||
@@ -293,7 +322,10 @@
|
||||
<span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a Python ``datetime`` object</span>
|
||||
|
||||
@@ -312,7 +344,10 @@
|
||||
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
|
||||
|
||||
@@ -326,7 +361,10 @@
|
||||
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the ISO code for the country associated</span>
|
||||
<span class="sd"> with the given IPv4 or IPv6 address</span>
|
||||
@@ -353,7 +391,7 @@
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">):</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
@@ -368,14 +406,13 @@
|
||||
<span class="k">break</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span> <span class="s2">"dbip-country-lite.mmdb"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">path</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"dbip-country-lite.mmdb"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
|
||||
|
||||
@@ -389,7 +426,10 @@
|
||||
<span class="k">return</span> <span class="n">country</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
<span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -413,7 +453,7 @@
|
||||
<span class="sd"> the supplied reverse_dns_base_domain and the type will be None</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="n">reader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span>
|
||||
<span class="n">key</span> <span class="o">=</span> <span class="n">row</span><span class="p">[</span><span class="s2">"base_reverse_dns"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
@@ -433,20 +473,29 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Failed to fetch reverse DNS map: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Not a valid CSV file"</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Response body:"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">csv_file</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading included reverse DNS map..."</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span> <span class="s2">"base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"base_reverse_dns_map.csv"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">service</span> <span class="o">=</span> <span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">base_domain</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||||
@@ -455,7 +504,10 @@
|
||||
<span class="k">return</span> <span class="n">service</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -531,7 +583,8 @@
|
||||
<span class="k">return</span> <span class="n">info</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@@ -554,7 +607,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_filename_safe_string"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a><span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="get_filename_safe_string">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a string to a string that is safe for a filename</span>
|
||||
|
||||
@@ -576,7 +631,10 @@
|
||||
<span class="k">return</span> <span class="n">string</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_mbox"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a><span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_mbox">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an MBOX mailbox file</span>
|
||||
|
||||
@@ -597,7 +655,10 @@
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an Outlook msg OLE/MSG file</span>
|
||||
|
||||
@@ -612,7 +673,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
|
||||
<span class="sd"> standard RFC 822 format</span>
|
||||
@@ -648,7 +712,10 @@
|
||||
<span class="k">return</span> <span class="n">rfc822</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_email"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a><span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_email">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> A simplified email parser</span>
|
||||
|
||||
@@ -754,6 +821,7 @@
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">parsed_email</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -763,7 +831,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
:members:
|
||||
```
|
||||
|
||||
|
||||
## parsedmarc.splunk
|
||||
|
||||
```{eval-rst}
|
||||
|
||||
+24
-4
@@ -33,17 +33,37 @@ and Valimail.
|
||||
|
||||
## Features
|
||||
|
||||
- Parses draft and 1.0 standard aggregate/rua reports
|
||||
- Parses forensic/failure/ruf reports
|
||||
- Parses draft and 1.0 standard aggregate/rua DMARC reports
|
||||
- Parses forensic/failure/ruf DMARC reports
|
||||
- Parses reports from SMTP TLS Reporting
|
||||
- Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API
|
||||
- Transparently handles gzip or zip compressed reports
|
||||
- Consistent data structures
|
||||
- Simple JSON and/or CSV output
|
||||
- Optionally email the results
|
||||
- Optionally send the results to Elasticsearch/OpenSearch and/or Splunk, for use with
|
||||
premade dashboards
|
||||
- Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use
|
||||
with premade dashboards
|
||||
- Optionally send reports to Apache Kafka
|
||||
|
||||
## Python Compatibility
|
||||
|
||||
This project supports the following Python versions, which are either actively maintained or are the default versions
|
||||
for RHEL or Debian.
|
||||
|
||||
| Version | Supported | Reason |
|
||||
|---------|-----------|------------------------------------------------------------|
|
||||
| < 3.6 | ❌ | End of Life (EOL) |
|
||||
| 3.6 | ❌ | Used in RHHEL 8, but not supported by project dependencies |
|
||||
| 3.7 | ❌ | End of Life (EOL) |
|
||||
| 3.8 | ❌ | End of Life (EOL) |
|
||||
| 3.9 | ✅ | Supported until August 2026 (Debian 11); May 2032 (RHEL 9) |
|
||||
| 3.10 | ✅ | Actively maintained |
|
||||
| 3.11 | ✅ | Actively maintained; supported until June 2028 (Debian 12) |
|
||||
| 3.12 | ✅ | Actively maintained; supported until May 2035 (RHEL 10) |
|
||||
| 3.13 | ✅ | Actively maintained; supported until June 2030 (Debian 13) |
|
||||
| 3.14 | ✅ | Actively maintained |
|
||||
|
||||
|
||||
```{toctree}
|
||||
:caption: 'Contents'
|
||||
:maxdepth: 2
|
||||
|
||||
+28
-4
@@ -120,8 +120,10 @@ The full set of configuration options are:
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `save_smtp_tls` - bool: Save SMTP-STS report data to
|
||||
Elasticsearch, Splunk and/or S3
|
||||
- `index_prefix_domain_map` - bool: A path mapping of Opensearch/Elasticsearch index prefixes to domain names
|
||||
- `strip_attachment_payloads` - bool: Remove attachment
|
||||
payloads from results
|
||||
- `silent` - bool: Set this to `False` to output results to STDOUT
|
||||
- `output` - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.
|
||||
- `aggregate_json_filename` - str: filename for the aggregate
|
||||
JSON output file
|
||||
@@ -167,7 +169,7 @@ The full set of configuration options are:
|
||||
IDLE response or the number of seconds until the next
|
||||
mail check (Default: `30`)
|
||||
- `since` - str: Search for messages since certain time. (Examples: `5m|3h|2d|1w`)
|
||||
Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}).
|
||||
Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}.
|
||||
Defaults to `1d` if incorrect value is provided.
|
||||
- `imap`
|
||||
- `host` - str: The IMAP server hostname or IP address
|
||||
@@ -252,7 +254,7 @@ The full set of configuration options are:
|
||||
:::
|
||||
- `user` - str: Basic auth username
|
||||
- `password` - str: Basic auth password
|
||||
- `apiKey` - str: API key
|
||||
- `api_key` - str: API key
|
||||
- `ssl` - bool: Use an encrypted SSL/TLS connection
|
||||
(Default: `True`)
|
||||
- `timeout` - float: Timeout in seconds (Default: 60)
|
||||
@@ -275,7 +277,7 @@ The full set of configuration options are:
|
||||
:::
|
||||
- `user` - str: Basic auth username
|
||||
- `password` - str: Basic auth password
|
||||
- `apiKey` - str: API key
|
||||
- `api_key` - str: API key
|
||||
- `ssl` - bool: Use an encrypted SSL/TLS connection
|
||||
(Default: `True`)
|
||||
- `timeout` - float: Timeout in seconds (Default: 60)
|
||||
@@ -369,7 +371,7 @@ The full set of configuration options are:
|
||||
- `mode` - str: The GELF transport type to use. Valid modes: `tcp`, `udp`, `tls`
|
||||
|
||||
- `maildir`
|
||||
- `reports_folder` - str: Full path for mailbox maidir location (Default: `INBOX`)
|
||||
- `maildir_path` - str: Full path for mailbox maidir location (Default: `INBOX`)
|
||||
- `maildir_create` - bool: Create maildir if not present (Default: False)
|
||||
|
||||
- `webhook` - Post the individual reports to a webhook url with the report as the JSON body
|
||||
@@ -445,6 +447,28 @@ PUT _cluster/settings
|
||||
Increasing this value increases resource usage.
|
||||
:::
|
||||
|
||||
## Multi-tenant support
|
||||
|
||||
Starting in `8.19.0`, ParseDMARC provides multi-tenant support by placing data into separate OpenSearch or Elasticsearch index prefixes. To set this up, create a YAML file that is formatted where each key is a tenant name, and the value is a list of domains related to that tenant, not including subdomains, like this:
|
||||
|
||||
```yaml
|
||||
example:
|
||||
- example.com
|
||||
- example.net
|
||||
- example.org
|
||||
|
||||
whalensolutions:
|
||||
- whalensolutions.com
|
||||
```
|
||||
|
||||
Save it to disk where the user running ParseDMARC can read it, then set `index_prefix_domain_map` to that filepath in the `[general]` section of the ParseDMARC configuration file and do not set an `index_prefix` option in the `[elasticsearch]` or `[opensearch]` sections.
|
||||
|
||||
When configured correctly, if ParseDMARC finds that a report is related to a domain in the mapping, the report will be saved in an index name that has the tenant name prefixed to it with a trailing underscore. Then, you can use the security features of Opensearch or the ELK stack to only grant users access to the indexes that they need.
|
||||
|
||||
:::{note}
|
||||
A domain cannot be used in multiple tenant lists. Only the first prefix list that contains the matching domain is used.
|
||||
:::
|
||||
|
||||
## Running parsedmarc as a systemd service
|
||||
|
||||
Use systemd to run `parsedmarc` as a service and process reports as
|
||||
|
||||
@@ -1,20 +1,9 @@
|
||||
/*
|
||||
* _sphinx_javascript_frameworks_compat.js
|
||||
* ~~~~~~~~~~
|
||||
*
|
||||
* Compatability shim for jQuery and underscores.js.
|
||||
*
|
||||
* WILL BE REMOVED IN Sphinx 6.0
|
||||
* xref RemovedInSphinx60Warning
|
||||
/* Compatability shim for jQuery and underscores.js.
|
||||
*
|
||||
* Copyright Sphinx contributors
|
||||
* Released under the two clause BSD licence
|
||||
*/
|
||||
|
||||
/**
|
||||
* select a different prefix for underscore
|
||||
*/
|
||||
$u = _.noConflict();
|
||||
|
||||
|
||||
/**
|
||||
* small helper function to urldecode strings
|
||||
*
|
||||
|
||||
+42
-35
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* basic.css
|
||||
* ~~~~~~~~~
|
||||
*
|
||||
* Sphinx stylesheet -- basic theme.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
|
||||
/* -- main layout ----------------------------------------------------------- */
|
||||
@@ -115,15 +108,11 @@ img {
|
||||
/* -- search page ----------------------------------------------------------- */
|
||||
|
||||
ul.search {
|
||||
margin: 10px 0 0 20px;
|
||||
padding: 0;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
ul.search li {
|
||||
padding: 5px 0 5px 20px;
|
||||
background-image: url(file.png);
|
||||
background-repeat: no-repeat;
|
||||
background-position: 0 7px;
|
||||
padding: 5px 0;
|
||||
}
|
||||
|
||||
ul.search li a {
|
||||
@@ -236,17 +225,11 @@ div.body p, div.body dd, div.body li, div.body blockquote {
|
||||
a.headerlink {
|
||||
visibility: hidden;
|
||||
}
|
||||
a.brackets:before,
|
||||
span.brackets > a:before{
|
||||
content: "[";
|
||||
}
|
||||
|
||||
a.brackets:after,
|
||||
span.brackets > a:after {
|
||||
content: "]";
|
||||
a:visited {
|
||||
color: #551A8B;
|
||||
}
|
||||
|
||||
|
||||
h1:hover > a.headerlink,
|
||||
h2:hover > a.headerlink,
|
||||
h3:hover > a.headerlink,
|
||||
@@ -334,11 +317,17 @@ aside.sidebar {
|
||||
p.sidebar-title {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
nav.contents,
|
||||
aside.topic,
|
||||
div.admonition, div.topic, blockquote {
|
||||
clear: left;
|
||||
}
|
||||
|
||||
/* -- topics ---------------------------------------------------------------- */
|
||||
|
||||
nav.contents,
|
||||
aside.topic,
|
||||
div.topic {
|
||||
border: 1px solid #ccc;
|
||||
padding: 7px;
|
||||
@@ -377,6 +366,8 @@ div.body p.centered {
|
||||
|
||||
div.sidebar > :last-child,
|
||||
aside.sidebar > :last-child,
|
||||
nav.contents > :last-child,
|
||||
aside.topic > :last-child,
|
||||
div.topic > :last-child,
|
||||
div.admonition > :last-child {
|
||||
margin-bottom: 0;
|
||||
@@ -384,6 +375,8 @@ div.admonition > :last-child {
|
||||
|
||||
div.sidebar::after,
|
||||
aside.sidebar::after,
|
||||
nav.contents::after,
|
||||
aside.topic::after,
|
||||
div.topic::after,
|
||||
div.admonition::after,
|
||||
blockquote::after {
|
||||
@@ -608,19 +601,27 @@ ol.simple p,
|
||||
ul.simple p {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
dl.footnote > dt,
|
||||
dl.citation > dt {
|
||||
float: left;
|
||||
margin-right: 0.5em;
|
||||
}
|
||||
|
||||
dl.footnote > dd,
|
||||
dl.citation > dd {
|
||||
aside.footnote > span,
|
||||
div.citation > span {
|
||||
float: left;
|
||||
}
|
||||
aside.footnote > span:last-of-type,
|
||||
div.citation > span:last-of-type {
|
||||
padding-right: 0.5em;
|
||||
}
|
||||
aside.footnote > p {
|
||||
margin-left: 2em;
|
||||
}
|
||||
div.citation > p {
|
||||
margin-left: 4em;
|
||||
}
|
||||
aside.footnote > p:last-of-type,
|
||||
div.citation > p:last-of-type {
|
||||
margin-bottom: 0em;
|
||||
}
|
||||
|
||||
dl.footnote > dd:after,
|
||||
dl.citation > dd:after {
|
||||
aside.footnote > p:last-of-type:after,
|
||||
div.citation > p:last-of-type:after {
|
||||
content: "";
|
||||
clear: both;
|
||||
}
|
||||
@@ -636,10 +637,6 @@ dl.field-list > dt {
|
||||
padding-left: 0.5em;
|
||||
padding-right: 5px;
|
||||
}
|
||||
dl.field-list > dt:after {
|
||||
content: ":";
|
||||
}
|
||||
|
||||
|
||||
dl.field-list > dd {
|
||||
padding-left: 0.5em;
|
||||
@@ -666,6 +663,16 @@ dd {
|
||||
margin-left: 30px;
|
||||
}
|
||||
|
||||
.sig dd {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
.sig dl {
|
||||
margin-top: 0px;
|
||||
margin-bottom: 0px;
|
||||
}
|
||||
|
||||
dl > dd:last-child,
|
||||
dl > dd:last-child > :last-child {
|
||||
margin-bottom: 0;
|
||||
|
||||
@@ -1 +1 @@
|
||||
.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
|
||||
.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}
|
||||
File diff suppressed because one or more lines are too long
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* doctools.js
|
||||
* ~~~~~~~~~~~
|
||||
*
|
||||
* Base JavaScript utilities for all Sphinx HTML documentation.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
var DOCUMENTATION_OPTIONS = {
|
||||
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
|
||||
VERSION: '8.18.1',
|
||||
const DOCUMENTATION_OPTIONS = {
|
||||
VERSION: '8.19.1',
|
||||
LANGUAGE: 'en',
|
||||
COLLAPSE_INDEX: false,
|
||||
BUILDER: 'html',
|
||||
|
||||
@@ -1,19 +1,12 @@
|
||||
/*
|
||||
* language_data.js
|
||||
* ~~~~~~~~~~~~~~~~
|
||||
*
|
||||
* This script contains the language-specific data used by searchtools.js,
|
||||
* namely the list of stopwords, stemmer, scorer and splitter.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
|
||||
var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"];
|
||||
|
||||
|
||||
/* Non-minified version is copied as a separate JS file, is available */
|
||||
/* Non-minified version is copied as a separate JS file, if available */
|
||||
|
||||
/**
|
||||
* Porter Stemmer
|
||||
|
||||
+18
-18
@@ -6,9 +6,9 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
|
||||
.highlight .hll { background-color: #ffffcc }
|
||||
.highlight { background: #f8f8f8; }
|
||||
.highlight .c { color: #3D7B7B; font-style: italic } /* Comment */
|
||||
.highlight .err { border: 1px solid #FF0000 } /* Error */
|
||||
.highlight .err { border: 1px solid #F00 } /* Error */
|
||||
.highlight .k { color: #008000; font-weight: bold } /* Keyword */
|
||||
.highlight .o { color: #666666 } /* Operator */
|
||||
.highlight .o { color: #666 } /* Operator */
|
||||
.highlight .ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */
|
||||
.highlight .cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */
|
||||
.highlight .cp { color: #9C6500 } /* Comment.Preproc */
|
||||
@@ -25,34 +25,34 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
|
||||
.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
|
||||
.highlight .gs { font-weight: bold } /* Generic.Strong */
|
||||
.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
|
||||
.highlight .gt { color: #0044DD } /* Generic.Traceback */
|
||||
.highlight .gt { color: #04D } /* Generic.Traceback */
|
||||
.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
|
||||
.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
|
||||
.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
|
||||
.highlight .kp { color: #008000 } /* Keyword.Pseudo */
|
||||
.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
|
||||
.highlight .kt { color: #B00040 } /* Keyword.Type */
|
||||
.highlight .m { color: #666666 } /* Literal.Number */
|
||||
.highlight .m { color: #666 } /* Literal.Number */
|
||||
.highlight .s { color: #BA2121 } /* Literal.String */
|
||||
.highlight .na { color: #687822 } /* Name.Attribute */
|
||||
.highlight .nb { color: #008000 } /* Name.Builtin */
|
||||
.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
|
||||
.highlight .no { color: #880000 } /* Name.Constant */
|
||||
.highlight .nd { color: #AA22FF } /* Name.Decorator */
|
||||
.highlight .nc { color: #00F; font-weight: bold } /* Name.Class */
|
||||
.highlight .no { color: #800 } /* Name.Constant */
|
||||
.highlight .nd { color: #A2F } /* Name.Decorator */
|
||||
.highlight .ni { color: #717171; font-weight: bold } /* Name.Entity */
|
||||
.highlight .ne { color: #CB3F38; font-weight: bold } /* Name.Exception */
|
||||
.highlight .nf { color: #0000FF } /* Name.Function */
|
||||
.highlight .nf { color: #00F } /* Name.Function */
|
||||
.highlight .nl { color: #767600 } /* Name.Label */
|
||||
.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
|
||||
.highlight .nn { color: #00F; font-weight: bold } /* Name.Namespace */
|
||||
.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
|
||||
.highlight .nv { color: #19177C } /* Name.Variable */
|
||||
.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
|
||||
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
|
||||
.highlight .mb { color: #666666 } /* Literal.Number.Bin */
|
||||
.highlight .mf { color: #666666 } /* Literal.Number.Float */
|
||||
.highlight .mh { color: #666666 } /* Literal.Number.Hex */
|
||||
.highlight .mi { color: #666666 } /* Literal.Number.Integer */
|
||||
.highlight .mo { color: #666666 } /* Literal.Number.Oct */
|
||||
.highlight .ow { color: #A2F; font-weight: bold } /* Operator.Word */
|
||||
.highlight .w { color: #BBB } /* Text.Whitespace */
|
||||
.highlight .mb { color: #666 } /* Literal.Number.Bin */
|
||||
.highlight .mf { color: #666 } /* Literal.Number.Float */
|
||||
.highlight .mh { color: #666 } /* Literal.Number.Hex */
|
||||
.highlight .mi { color: #666 } /* Literal.Number.Integer */
|
||||
.highlight .mo { color: #666 } /* Literal.Number.Oct */
|
||||
.highlight .sa { color: #BA2121 } /* Literal.String.Affix */
|
||||
.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
|
||||
.highlight .sc { color: #BA2121 } /* Literal.String.Char */
|
||||
@@ -67,9 +67,9 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left:
|
||||
.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
|
||||
.highlight .ss { color: #19177C } /* Literal.String.Symbol */
|
||||
.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
|
||||
.highlight .fm { color: #0000FF } /* Name.Function.Magic */
|
||||
.highlight .fm { color: #00F } /* Name.Function.Magic */
|
||||
.highlight .vc { color: #19177C } /* Name.Variable.Class */
|
||||
.highlight .vg { color: #19177C } /* Name.Variable.Global */
|
||||
.highlight .vi { color: #19177C } /* Name.Variable.Instance */
|
||||
.highlight .vm { color: #19177C } /* Name.Variable.Magic */
|
||||
.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
|
||||
.highlight .il { color: #666 } /* Literal.Number.Integer.Long */
|
||||
+153
-84
@@ -1,12 +1,5 @@
|
||||
/*
|
||||
* searchtools.js
|
||||
* ~~~~~~~~~~~~~~~~
|
||||
*
|
||||
* Sphinx JavaScript utilities for the full-text search.
|
||||
*
|
||||
* :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
|
||||
* :license: BSD, see LICENSE for details.
|
||||
*
|
||||
*/
|
||||
"use strict";
|
||||
|
||||
@@ -20,7 +13,7 @@ if (typeof Scorer === "undefined") {
|
||||
// and returns the new score.
|
||||
/*
|
||||
score: result => {
|
||||
const [docname, title, anchor, descr, score, filename] = result
|
||||
const [docname, title, anchor, descr, score, filename, kind] = result
|
||||
return score
|
||||
},
|
||||
*/
|
||||
@@ -47,6 +40,14 @@ if (typeof Scorer === "undefined") {
|
||||
};
|
||||
}
|
||||
|
||||
// Global search result kind enum, used by themes to style search results.
|
||||
class SearchResultKind {
|
||||
static get index() { return "index"; }
|
||||
static get object() { return "object"; }
|
||||
static get text() { return "text"; }
|
||||
static get title() { return "title"; }
|
||||
}
|
||||
|
||||
const _removeChildren = (element) => {
|
||||
while (element && element.lastChild) element.removeChild(element.lastChild);
|
||||
};
|
||||
@@ -57,16 +58,20 @@ const _removeChildren = (element) => {
|
||||
const _escapeRegExp = (string) =>
|
||||
string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
|
||||
|
||||
const _displayItem = (item, searchTerms) => {
|
||||
const _displayItem = (item, searchTerms, highlightTerms) => {
|
||||
const docBuilder = DOCUMENTATION_OPTIONS.BUILDER;
|
||||
const docUrlRoot = DOCUMENTATION_OPTIONS.URL_ROOT;
|
||||
const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX;
|
||||
const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX;
|
||||
const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY;
|
||||
const contentRoot = document.documentElement.dataset.content_root;
|
||||
|
||||
const [docName, title, anchor, descr, score, _filename] = item;
|
||||
const [docName, title, anchor, descr, score, _filename, kind] = item;
|
||||
|
||||
let listItem = document.createElement("li");
|
||||
// Add a class representing the item's type:
|
||||
// can be used by a theme's CSS selector for styling
|
||||
// See SearchResultKind for the class names.
|
||||
listItem.classList.add(`kind-${kind}`);
|
||||
let requestUrl;
|
||||
let linkUrl;
|
||||
if (docBuilder === "dirhtml") {
|
||||
@@ -75,28 +80,35 @@ const _displayItem = (item, searchTerms) => {
|
||||
if (dirname.match(/\/index\/$/))
|
||||
dirname = dirname.substring(0, dirname.length - 6);
|
||||
else if (dirname === "index/") dirname = "";
|
||||
requestUrl = docUrlRoot + dirname;
|
||||
requestUrl = contentRoot + dirname;
|
||||
linkUrl = requestUrl;
|
||||
} else {
|
||||
// normal html builders
|
||||
requestUrl = docUrlRoot + docName + docFileSuffix;
|
||||
requestUrl = contentRoot + docName + docFileSuffix;
|
||||
linkUrl = docName + docLinkSuffix;
|
||||
}
|
||||
let linkEl = listItem.appendChild(document.createElement("a"));
|
||||
linkEl.href = linkUrl + anchor;
|
||||
linkEl.dataset.score = score;
|
||||
linkEl.innerHTML = title;
|
||||
if (descr)
|
||||
if (descr) {
|
||||
listItem.appendChild(document.createElement("span")).innerHTML =
|
||||
" (" + descr + ")";
|
||||
// highlight search terms in the description
|
||||
if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
|
||||
highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
|
||||
}
|
||||
else if (showSearchSummary)
|
||||
fetch(requestUrl)
|
||||
.then((responseData) => responseData.text())
|
||||
.then((data) => {
|
||||
if (data)
|
||||
listItem.appendChild(
|
||||
Search.makeSearchSummary(data, searchTerms)
|
||||
Search.makeSearchSummary(data, searchTerms, anchor)
|
||||
);
|
||||
// highlight search terms in the summary
|
||||
if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js
|
||||
highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted"));
|
||||
});
|
||||
Search.output.appendChild(listItem);
|
||||
};
|
||||
@@ -108,27 +120,46 @@ const _finishSearch = (resultCount) => {
|
||||
"Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories."
|
||||
);
|
||||
else
|
||||
Search.status.innerText = _(
|
||||
`Search finished, found ${resultCount} page(s) matching the search query.`
|
||||
);
|
||||
Search.status.innerText = Documentation.ngettext(
|
||||
"Search finished, found one page matching the search query.",
|
||||
"Search finished, found ${resultCount} pages matching the search query.",
|
||||
resultCount,
|
||||
).replace('${resultCount}', resultCount);
|
||||
};
|
||||
const _displayNextItem = (
|
||||
results,
|
||||
resultCount,
|
||||
searchTerms
|
||||
searchTerms,
|
||||
highlightTerms,
|
||||
) => {
|
||||
// results left, load the summary and display it
|
||||
// this is intended to be dynamic (don't sub resultsCount)
|
||||
if (results.length) {
|
||||
_displayItem(results.pop(), searchTerms);
|
||||
_displayItem(results.pop(), searchTerms, highlightTerms);
|
||||
setTimeout(
|
||||
() => _displayNextItem(results, resultCount, searchTerms),
|
||||
() => _displayNextItem(results, resultCount, searchTerms, highlightTerms),
|
||||
5
|
||||
);
|
||||
}
|
||||
// search finished, update title and status message
|
||||
else _finishSearch(resultCount);
|
||||
};
|
||||
// Helper function used by query() to order search results.
|
||||
// Each input is an array of [docname, title, anchor, descr, score, filename, kind].
|
||||
// Order the results by score (in opposite order of appearance, since the
|
||||
// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically.
|
||||
const _orderResultsByScoreThenName = (a, b) => {
|
||||
const leftScore = a[4];
|
||||
const rightScore = b[4];
|
||||
if (leftScore === rightScore) {
|
||||
// same score: sort alphabetically
|
||||
const leftTitle = a[1].toLowerCase();
|
||||
const rightTitle = b[1].toLowerCase();
|
||||
if (leftTitle === rightTitle) return 0;
|
||||
return leftTitle > rightTitle ? -1 : 1; // inverted is intentional
|
||||
}
|
||||
return leftScore > rightScore ? 1 : -1;
|
||||
};
|
||||
|
||||
/**
|
||||
* Default splitQuery function. Can be overridden in ``sphinx.search`` with a
|
||||
@@ -152,13 +183,26 @@ const Search = {
|
||||
_queued_query: null,
|
||||
_pulse_status: -1,
|
||||
|
||||
htmlToText: (htmlString) => {
|
||||
htmlToText: (htmlString, anchor) => {
|
||||
const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html');
|
||||
htmlElement.querySelectorAll(".headerlink").forEach((el) => { el.remove() });
|
||||
for (const removalQuery of [".headerlink", "script", "style"]) {
|
||||
htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() });
|
||||
}
|
||||
if (anchor) {
|
||||
const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`);
|
||||
if (anchorContent) return anchorContent.textContent;
|
||||
|
||||
console.warn(
|
||||
`Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.`
|
||||
);
|
||||
}
|
||||
|
||||
// if anchor not specified or not found, fall back to main content
|
||||
const docContent = htmlElement.querySelector('[role="main"]');
|
||||
if (docContent !== undefined) return docContent.textContent;
|
||||
if (docContent) return docContent.textContent;
|
||||
|
||||
console.warn(
|
||||
"Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template."
|
||||
"Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template."
|
||||
);
|
||||
return "";
|
||||
},
|
||||
@@ -211,6 +255,7 @@ const Search = {
|
||||
searchSummary.classList.add("search-summary");
|
||||
searchSummary.innerText = "";
|
||||
const searchList = document.createElement("ul");
|
||||
searchList.setAttribute("role", "list");
|
||||
searchList.classList.add("search");
|
||||
|
||||
const out = document.getElementById("search-results");
|
||||
@@ -231,16 +276,7 @@ const Search = {
|
||||
else Search.deferQuery(query);
|
||||
},
|
||||
|
||||
/**
|
||||
* execute search (requires search index to be loaded)
|
||||
*/
|
||||
query: (query) => {
|
||||
const filenames = Search._index.filenames;
|
||||
const docNames = Search._index.docnames;
|
||||
const titles = Search._index.titles;
|
||||
const allTitles = Search._index.alltitles;
|
||||
const indexEntries = Search._index.indexentries;
|
||||
|
||||
_parseQuery: (query) => {
|
||||
// stem the search terms and add them to the correct list
|
||||
const stemmer = new Stemmer();
|
||||
const searchTerms = new Set();
|
||||
@@ -276,22 +312,40 @@ const Search = {
|
||||
// console.info("required: ", [...searchTerms]);
|
||||
// console.info("excluded: ", [...excludedTerms]);
|
||||
|
||||
// array of [docname, title, anchor, descr, score, filename]
|
||||
let results = [];
|
||||
return [query, searchTerms, excludedTerms, highlightTerms, objectTerms];
|
||||
},
|
||||
|
||||
/**
|
||||
* execute search (requires search index to be loaded)
|
||||
*/
|
||||
_performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => {
|
||||
const filenames = Search._index.filenames;
|
||||
const docNames = Search._index.docnames;
|
||||
const titles = Search._index.titles;
|
||||
const allTitles = Search._index.alltitles;
|
||||
const indexEntries = Search._index.indexentries;
|
||||
|
||||
// Collect multiple result groups to be sorted separately and then ordered.
|
||||
// Each is an array of [docname, title, anchor, descr, score, filename, kind].
|
||||
const normalResults = [];
|
||||
const nonMainIndexResults = [];
|
||||
|
||||
_removeChildren(document.getElementById("search-progress"));
|
||||
|
||||
const queryLower = query.toLowerCase();
|
||||
const queryLower = query.toLowerCase().trim();
|
||||
for (const [title, foundTitles] of Object.entries(allTitles)) {
|
||||
if (title.toLowerCase().includes(queryLower) && (queryLower.length >= title.length/2)) {
|
||||
if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) {
|
||||
for (const [file, id] of foundTitles) {
|
||||
let score = Math.round(100 * queryLower.length / title.length)
|
||||
results.push([
|
||||
const score = Math.round(Scorer.title * queryLower.length / title.length);
|
||||
const boost = titles[file] === title ? 1 : 0; // add a boost for document titles
|
||||
normalResults.push([
|
||||
docNames[file],
|
||||
titles[file] !== title ? `${titles[file]} > ${title}` : title,
|
||||
id !== null ? "#" + id : "",
|
||||
null,
|
||||
score,
|
||||
score + boost,
|
||||
filenames[file],
|
||||
SearchResultKind.title,
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -300,46 +354,48 @@ const Search = {
|
||||
// search for explicit entries in index directives
|
||||
for (const [entry, foundEntries] of Object.entries(indexEntries)) {
|
||||
if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) {
|
||||
for (const [file, id] of foundEntries) {
|
||||
let score = Math.round(100 * queryLower.length / entry.length)
|
||||
results.push([
|
||||
for (const [file, id, isMain] of foundEntries) {
|
||||
const score = Math.round(100 * queryLower.length / entry.length);
|
||||
const result = [
|
||||
docNames[file],
|
||||
titles[file],
|
||||
id ? "#" + id : "",
|
||||
null,
|
||||
score,
|
||||
filenames[file],
|
||||
]);
|
||||
SearchResultKind.index,
|
||||
];
|
||||
if (isMain) {
|
||||
normalResults.push(result);
|
||||
} else {
|
||||
nonMainIndexResults.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lookup as object
|
||||
objectTerms.forEach((term) =>
|
||||
results.push(...Search.performObjectSearch(term, objectTerms))
|
||||
normalResults.push(...Search.performObjectSearch(term, objectTerms))
|
||||
);
|
||||
|
||||
// lookup as search terms in fulltext
|
||||
results.push(...Search.performTermsSearch(searchTerms, excludedTerms));
|
||||
normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms));
|
||||
|
||||
// let the scorer override scores with a custom scoring function
|
||||
if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
if (Scorer.score) {
|
||||
normalResults.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item)));
|
||||
}
|
||||
|
||||
// now sort the results by score (in opposite order of appearance, since the
|
||||
// display function below uses pop() to retrieve items) and then
|
||||
// alphabetically
|
||||
results.sort((a, b) => {
|
||||
const leftScore = a[4];
|
||||
const rightScore = b[4];
|
||||
if (leftScore === rightScore) {
|
||||
// same score: sort alphabetically
|
||||
const leftTitle = a[1].toLowerCase();
|
||||
const rightTitle = b[1].toLowerCase();
|
||||
if (leftTitle === rightTitle) return 0;
|
||||
return leftTitle > rightTitle ? -1 : 1; // inverted is intentional
|
||||
}
|
||||
return leftScore > rightScore ? 1 : -1;
|
||||
});
|
||||
// Sort each group of results by score and then alphabetically by name.
|
||||
normalResults.sort(_orderResultsByScoreThenName);
|
||||
nonMainIndexResults.sort(_orderResultsByScoreThenName);
|
||||
|
||||
// Combine the result groups in (reverse) order.
|
||||
// Non-main index entries are typically arbitrary cross-references,
|
||||
// so display them after other results.
|
||||
let results = [...nonMainIndexResults, ...normalResults];
|
||||
|
||||
// remove duplicate search results
|
||||
// note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept
|
||||
@@ -353,14 +409,19 @@ const Search = {
|
||||
return acc;
|
||||
}, []);
|
||||
|
||||
results = results.reverse();
|
||||
return results.reverse();
|
||||
},
|
||||
|
||||
query: (query) => {
|
||||
const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query);
|
||||
const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms);
|
||||
|
||||
// for debugging
|
||||
//Search.lastresults = results.slice(); // a copy
|
||||
// console.info("search results:", Search.lastresults);
|
||||
|
||||
// print the results
|
||||
_displayNextItem(results, results.length, searchTerms);
|
||||
_displayNextItem(results, results.length, searchTerms, highlightTerms);
|
||||
},
|
||||
|
||||
/**
|
||||
@@ -424,6 +485,7 @@ const Search = {
|
||||
descr,
|
||||
score,
|
||||
filenames[match[0]],
|
||||
SearchResultKind.object,
|
||||
]);
|
||||
};
|
||||
Object.keys(objects).forEach((prefix) =>
|
||||
@@ -451,21 +513,27 @@ const Search = {
|
||||
// perform the search on the required terms
|
||||
searchTerms.forEach((word) => {
|
||||
const files = [];
|
||||
// find documents, if any, containing the query word in their text/title term indices
|
||||
// use Object.hasOwnProperty to avoid mismatching against prototype properties
|
||||
const arr = [
|
||||
{ files: terms[word], score: Scorer.term },
|
||||
{ files: titleTerms[word], score: Scorer.title },
|
||||
{ files: terms.hasOwnProperty(word) ? terms[word] : undefined, score: Scorer.term },
|
||||
{ files: titleTerms.hasOwnProperty(word) ? titleTerms[word] : undefined, score: Scorer.title },
|
||||
];
|
||||
// add support for partial matches
|
||||
if (word.length > 2) {
|
||||
const escapedWord = _escapeRegExp(word);
|
||||
Object.keys(terms).forEach((term) => {
|
||||
if (term.match(escapedWord) && !terms[word])
|
||||
arr.push({ files: terms[term], score: Scorer.partialTerm });
|
||||
});
|
||||
Object.keys(titleTerms).forEach((term) => {
|
||||
if (term.match(escapedWord) && !titleTerms[word])
|
||||
arr.push({ files: titleTerms[word], score: Scorer.partialTitle });
|
||||
});
|
||||
if (!terms.hasOwnProperty(word)) {
|
||||
Object.keys(terms).forEach((term) => {
|
||||
if (term.match(escapedWord))
|
||||
arr.push({ files: terms[term], score: Scorer.partialTerm });
|
||||
});
|
||||
}
|
||||
if (!titleTerms.hasOwnProperty(word)) {
|
||||
Object.keys(titleTerms).forEach((term) => {
|
||||
if (term.match(escapedWord))
|
||||
arr.push({ files: titleTerms[term], score: Scorer.partialTitle });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// no match but word was a required one
|
||||
@@ -481,16 +549,16 @@ const Search = {
|
||||
|
||||
// set score for the word in each file
|
||||
recordFiles.forEach((file) => {
|
||||
if (!scoreMap.has(file)) scoreMap.set(file, {});
|
||||
scoreMap.get(file)[word] = record.score;
|
||||
if (!scoreMap.has(file)) scoreMap.set(file, new Map());
|
||||
const fileScores = scoreMap.get(file);
|
||||
fileScores.set(word, record.score);
|
||||
});
|
||||
});
|
||||
|
||||
// create the mapping
|
||||
files.forEach((file) => {
|
||||
if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1)
|
||||
fileMap.get(file).push(word);
|
||||
else fileMap.set(file, [word]);
|
||||
if (!fileMap.has(file)) fileMap.set(file, [word]);
|
||||
else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -522,7 +590,7 @@ const Search = {
|
||||
break;
|
||||
|
||||
// select one (max) score for the file.
|
||||
const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w]));
|
||||
const score = Math.max(...wordList.map((w) => scoreMap.get(file).get(w)));
|
||||
// add result to the result list
|
||||
results.push([
|
||||
docNames[file],
|
||||
@@ -531,6 +599,7 @@ const Search = {
|
||||
null,
|
||||
score,
|
||||
filenames[file],
|
||||
SearchResultKind.text,
|
||||
]);
|
||||
}
|
||||
return results;
|
||||
@@ -541,8 +610,8 @@ const Search = {
|
||||
* search summary for a given text. keywords is a list
|
||||
* of stemmed words.
|
||||
*/
|
||||
makeSearchSummary: (htmlText, keywords) => {
|
||||
const text = Search.htmlToText(htmlText);
|
||||
makeSearchSummary: (htmlText, keywords, anchor) => {
|
||||
const text = Search.htmlToText(htmlText, anchor);
|
||||
if (text === "") return null;
|
||||
|
||||
const textLower = text.toLowerCase();
|
||||
|
||||
@@ -29,14 +29,19 @@ const _highlight = (node, addItems, text, className) => {
|
||||
}
|
||||
|
||||
span.appendChild(document.createTextNode(val.substr(pos, text.length)));
|
||||
const rest = document.createTextNode(val.substr(pos + text.length));
|
||||
parent.insertBefore(
|
||||
span,
|
||||
parent.insertBefore(
|
||||
document.createTextNode(val.substr(pos + text.length)),
|
||||
rest,
|
||||
node.nextSibling
|
||||
)
|
||||
);
|
||||
node.nodeValue = val.substr(0, pos);
|
||||
/* There may be more occurrences of search term in this node. So call this
|
||||
* function recursively on the remaining fragment.
|
||||
*/
|
||||
_highlight(rest, addItems, text, className);
|
||||
|
||||
if (isInSVG) {
|
||||
const rect = document.createElementNS(
|
||||
@@ -140,5 +145,10 @@ const SphinxHighlight = {
|
||||
},
|
||||
};
|
||||
|
||||
_ready(SphinxHighlight.highlightSearchWords);
|
||||
_ready(SphinxHighlight.initEscapeListener);
|
||||
_ready(() => {
|
||||
/* Do not call highlightSearchWords() when we are on the search page.
|
||||
* It will highlight words from the *previous* search query.
|
||||
*/
|
||||
if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords();
|
||||
SphinxHighlight.initEscapeListener();
|
||||
});
|
||||
|
||||
+15
-21
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Contributing to parsedmarc — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -91,9 +85,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="contributing-to-parsedmarc">
|
||||
<h1>Contributing to parsedmarc<a class="headerlink" href="#contributing-to-parsedmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Contributing to parsedmarc<a class="headerlink" href="#contributing-to-parsedmarc" title="Link to this heading"></a></h1>
|
||||
<section id="bug-reports">
|
||||
<h2>Bug reports<a class="headerlink" href="#bug-reports" title="Permalink to this heading"></a></h2>
|
||||
<h2>Bug reports<a class="headerlink" href="#bug-reports" title="Link to this heading"></a></h2>
|
||||
<p>Please report bugs on the GitHub issue tracker</p>
|
||||
<p><a class="reference external" href="https://github.com/domainaware/parsedmarc/issues">https://github.com/domainaware/parsedmarc/issues</a></p>
|
||||
</section>
|
||||
@@ -110,7 +104,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+16
-22
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Accessing an inbox using OWA/EWS — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,7 +86,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="accessing-an-inbox-using-owa-ews">
|
||||
<h1>Accessing an inbox using OWA/EWS<a class="headerlink" href="#accessing-an-inbox-using-owa-ews" title="Permalink to this heading"></a></h1>
|
||||
<h1>Accessing an inbox using OWA/EWS<a class="headerlink" href="#accessing-an-inbox-using-owa-ews" title="Link to this heading"></a></h1>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Starting in 8.0.0, parsedmarc supports accessing Microsoft/Office 365
|
||||
@@ -177,7 +171,7 @@ as a local EWS/OWA IMAP gateway. It can even work where
|
||||
</pre></div>
|
||||
</div>
|
||||
<section id="running-davmail-as-a-systemd-service">
|
||||
<h2>Running DavMail as a systemd service<a class="headerlink" href="#running-davmail-as-a-systemd-service" title="Permalink to this heading"></a></h2>
|
||||
<h2>Running DavMail as a systemd service<a class="headerlink" href="#running-davmail-as-a-systemd-service" title="Link to this heading"></a></h2>
|
||||
<p>Use systemd to run <code class="docutils literal notranslate"><span class="pre">davmail</span></code> as a service.</p>
|
||||
<p>Create a system user</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>useradd<span class="w"> </span>davmail<span class="w"> </span>-r<span class="w"> </span>-s<span class="w"> </span>/bin/false
|
||||
@@ -244,7 +238,7 @@ well as the current process (newest to oldest), run:</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="configuring-parsedmarc-for-davmail">
|
||||
<h2>Configuring parsedmarc for DavMail<a class="headerlink" href="#configuring-parsedmarc-for-davmail" title="Permalink to this heading"></a></h2>
|
||||
<h2>Configuring parsedmarc for DavMail<a class="headerlink" href="#configuring-parsedmarc-for-davmail" title="Link to this heading"></a></h2>
|
||||
<p>Because you are interacting with DavMail server over the loopback
|
||||
(i.e. <code class="docutils literal notranslate"><span class="pre">127.0.0.1</span></code>), add the following options to <code class="docutils literal notranslate"><span class="pre">parsedmarc.ini</span></code>
|
||||
config file:</p>
|
||||
@@ -269,7 +263,7 @@ config file:</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+30
-69
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Understanding DMARC — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Understanding DMARC — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -110,39 +104,34 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="understanding-dmarc">
|
||||
<h1>Understanding DMARC<a class="headerlink" href="#understanding-dmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Understanding DMARC<a class="headerlink" href="#understanding-dmarc" title="Link to this heading"></a></h1>
|
||||
<section id="resources">
|
||||
<h2>Resources<a class="headerlink" href="#resources" title="Permalink to this heading"></a></h2>
|
||||
<h2>Resources<a class="headerlink" href="#resources" title="Link to this heading"></a></h2>
|
||||
<section id="dmarc-guides">
|
||||
<h3>DMARC guides<a class="headerlink" href="#dmarc-guides" title="Permalink to this heading"></a></h3>
|
||||
<h3>DMARC guides<a class="headerlink" href="#dmarc-guides" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p><a class="reference external" href="https://seanthegeek.net/459/demystifying-dmarc/">Demystifying DMARC</a> - A complete guide to SPF, DKIM, and DMARC</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="spf-and-dmarc-record-validation">
|
||||
<h3>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Permalink to this heading"></a></h3>
|
||||
<h3>SPF and DMARC record validation<a class="headerlink" href="#spf-and-dmarc-record-validation" title="Link to this heading"></a></h3>
|
||||
<p>If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
<a class="reference external" href="https://domainaware.github.io/checkdmarc/">checkdmarc</a>.</p>
|
||||
</section>
|
||||
<section id="lookalike-domains">
|
||||
<h3>Lookalike domains<a class="headerlink" href="#lookalike-domains" title="Permalink to this heading"></a></h3>
|
||||
<h3>Lookalike domains<a class="headerlink" href="#lookalike-domains" title="Link to this heading"></a></h3>
|
||||
<p>DMARC protects against domain spoofing, not lookalike domains. for open source
|
||||
lookalike domain monitoring, check out <a class="reference external" href="https://github.com/seanthegeek/domainaware">DomainAware</a>.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="dmarc-alignment-guide">
|
||||
<h2>DMARC Alignment Guide<a class="headerlink" href="#dmarc-alignment-guide" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Alignment Guide<a class="headerlink" href="#dmarc-alignment-guide" title="Link to this heading"></a></h2>
|
||||
<p>DMARC ensures that SPF and DKM authentication mechanisms actually authenticate
|
||||
against the same domain that the end user sees.</p>
|
||||
<p>A message passes a DMARC check by passing DKIM or SPF, <strong>as long as the related
|
||||
indicators are also in alignment</strong>.</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 33%" />
|
||||
<col style="width: 33%" />
|
||||
<col style="width: 33%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td></td>
|
||||
<td><p><strong>DKIM</strong></p></td>
|
||||
@@ -180,7 +169,7 @@ header</p></td>
|
||||
</table>
|
||||
</section>
|
||||
<section id="what-if-a-sender-wont-support-dkim-dmarc">
|
||||
<h2>What if a sender won’t support DKIM/DMARC?<a class="headerlink" href="#what-if-a-sender-wont-support-dkim-dmarc" title="Permalink to this heading"></a></h2>
|
||||
<h2>What if a sender won’t support DKIM/DMARC?<a class="headerlink" href="#what-if-a-sender-wont-support-dkim-dmarc" title="Link to this heading"></a></h2>
|
||||
<ol class="arabic simple">
|
||||
<li><p>Some vendors don’t know about DMARC yet; ask about SPF and DKIM/email
|
||||
authentication.</p></li>
|
||||
@@ -200,21 +189,21 @@ spoofing of your TLD and/or any subdomain.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="what-about-mailing-lists">
|
||||
<h2>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Permalink to this heading"></a></h2>
|
||||
<h2>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Link to this heading"></a></h2>
|
||||
<p>When you deploy DMARC on your domain, you might find that messages
|
||||
relayed by mailing lists are failing DMARC, most likely because the mailing
|
||||
list is spoofing your from address, and modifying the subject,
|
||||
footer, or other part of the message, thereby breaking the
|
||||
DKIM signature.</p>
|
||||
<section id="mailing-list-best-practices">
|
||||
<h3>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Link to this heading"></a></h3>
|
||||
<p>Ideally, a mailing list should forward messages without altering the
|
||||
headers or body content at all. <a class="reference external" href="https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html">Joe Nelson</a> does a fantastic job of
|
||||
explaining exactly what mailing lists should and shouldn’t do to be
|
||||
fully DMARC compliant. Rather than repeat his fine work, here’s a
|
||||
summary:</p>
|
||||
<section id="do">
|
||||
<h4>Do<a class="headerlink" href="#do" title="Permalink to this heading"></a></h4>
|
||||
<h4>Do<a class="headerlink" href="#do" title="Link to this heading"></a></h4>
|
||||
<ul class="simple">
|
||||
<li><p>Retain headers from the original message</p></li>
|
||||
<li><p>Add <a class="reference external" href="https://tools.ietf.org/html/rfc2369">RFC 2369</a> List-Unsubscribe headers to outgoing messages, instead of
|
||||
@@ -234,7 +223,7 @@ adding unsubscribe links to the body</p></li>
|
||||
these headers.</p>
|
||||
</section>
|
||||
<section id="do-not">
|
||||
<h4>Do not<a class="headerlink" href="#do-not" title="Permalink to this heading"></a></h4>
|
||||
<h4>Do not<a class="headerlink" href="#do-not" title="Link to this heading"></a></h4>
|
||||
<ul class="simple">
|
||||
<li><p>Remove or modify any existing headers from the original message, including
|
||||
From, Date, Subject, etc.</p></li>
|
||||
@@ -251,13 +240,9 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
</section>
|
||||
<section id="mailman-2">
|
||||
<h4>Mailman 2<a class="headerlink" href="#mailman-2" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 2<a class="headerlink" href="#mailman-2" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to General Settings, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 74%" />
|
||||
<col style="width: 26%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -287,10 +272,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Non-digest options, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 66%" />
|
||||
<col style="width: 34%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -308,10 +289,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -329,16 +306,12 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
</section>
|
||||
<section id="mailman-3">
|
||||
<h4>Mailman 3<a class="headerlink" href="#mailman-3" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 3<a class="headerlink" href="#mailman-3" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to Settings> List Identity</p>
|
||||
<p>Make Subject prefix blank.</p>
|
||||
<p>Navigate to Settings> Alter Messages</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -366,10 +339,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Navigate to Settings> DMARC Mitigation</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 52%" />
|
||||
<col style="width: 48%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -393,13 +362,13 @@ command line instead, for example:</p>
|
||||
<p>Then restart mailman core.</p>
|
||||
</section>
|
||||
<section id="listserv">
|
||||
<h4>LISTSERV<a class="headerlink" href="#listserv" title="Permalink to this heading"></a></h4>
|
||||
<h4>LISTSERV<a class="headerlink" href="#listserv" title="Link to this heading"></a></h4>
|
||||
<p><a class="reference external" href="https://www.lsoft.com/news/dmarc-issue1-2018.asp">LISTSERV 16.0-2017a</a> and higher will rewrite the From header for domains
|
||||
that enforce with a DMARC quarantine or reject policy.</p>
|
||||
<p>Some additional steps are needed for Linux hosts.</p>
|
||||
</section>
|
||||
<section id="workarounds">
|
||||
<h4>Workarounds<a class="headerlink" href="#workarounds" title="Permalink to this heading"></a></h4>
|
||||
<h4>Workarounds<a class="headerlink" href="#workarounds" title="Link to this heading"></a></h4>
|
||||
<p>If a mailing list must go <strong>against</strong> best practices and
|
||||
modify the message (e.g. to add a required legal footer), the mailing
|
||||
list administrator must configure the list to replace the From address of the
|
||||
@@ -407,13 +376,9 @@ message (also known as munging) with the address of the mailing list, so they
|
||||
no longer spoof email addresses with domains protected by DMARC.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
<section id="id1">
|
||||
<h5>Mailman 2<a class="headerlink" href="#id1" title="Permalink to this heading"></a></h5>
|
||||
<h5>Mailman 2<a class="headerlink" href="#id1" title="Link to this heading"></a></h5>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -442,13 +407,9 @@ the original sender.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h5>Mailman 3<a class="headerlink" href="#id2" title="Permalink to this heading"></a></h5>
|
||||
<h5>Mailman 3<a class="headerlink" href="#id2" title="Link to this heading"></a></h5>
|
||||
<p>In the DMARC Mitigations tab of the Settings page, configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 49%" />
|
||||
<col style="width: 51%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -488,7 +449,7 @@ the original sender.</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+21
-25
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Elasticsearch and Kibana — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -93,14 +87,14 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="elasticsearch-and-kibana">
|
||||
<h1>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Permalink to this heading"></a></h1>
|
||||
<h1>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Link to this heading"></a></h1>
|
||||
<p>To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Elasticsearch and Kibana 6 or later are required</p>
|
||||
</div>
|
||||
<section id="installation">
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h2>
|
||||
<p>On Debian/Ubuntu based systems, run:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>apt-transport-https
|
||||
wget<span class="w"> </span>-qO<span class="w"> </span>-<span class="w"> </span>https://artifacts.elastic.co/GPG-KEY-elasticsearch<span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>gpg<span class="w"> </span>--dearmor<span class="w"> </span>-o<span class="w"> </span>/usr/share/keyrings/elasticsearch-keyring.gpg
|
||||
@@ -243,11 +237,13 @@ page of Kibana. (Hamburger menu -> “Management” -> “Stack Management
|
||||
visualizations, which could be used to restore them if you or someone else
|
||||
breaks them, as there are no permissions/access controls in Kibana without
|
||||
the commercial <a class="reference external" href="https://www.elastic.co/products/x-pack">X-Pack</a>.</p>
|
||||
<a class="reference external image-reference" href="_static/screenshots/saved-objects.png"><img alt="A screenshot of setting the Saved Objects Stack management UI in Kibana" class="align-center" src="_images/saved-objects.png" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/confirm-overwrite.png"><img alt="A screenshot of the overwrite conformation prompt" class="align-center" src="_images/confirm-overwrite.png" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/saved-objects.png"><img alt="A screenshot of setting the Saved Objects Stack management UI in Kibana" class="align-center" src="_images/saved-objects.png" />
|
||||
</a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/confirm-overwrite.png"><img alt="A screenshot of the overwrite conformation prompt" class="align-center" src="_images/confirm-overwrite.png" />
|
||||
</a>
|
||||
</section>
|
||||
<section id="upgrading-kibana-index-patterns">
|
||||
<h2>Upgrading Kibana index patterns<a class="headerlink" href="#upgrading-kibana-index-patterns" title="Permalink to this heading"></a></h2>
|
||||
<h2>Upgrading Kibana index patterns<a class="headerlink" href="#upgrading-kibana-index-patterns" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 5.0.0 makes some changes to the way data is indexed in
|
||||
Elasticsearch. if you are upgrading from a previous release of
|
||||
<code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code>, you need to complete the following steps to replace the
|
||||
@@ -266,7 +262,7 @@ Saved Objects page</p></li>
|
||||
</ol>
|
||||
</section>
|
||||
<section id="records-retention">
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this heading"></a></h2>
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||||
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
|
||||
index for each day to make it easy to comply with records
|
||||
retention regulations such as GDPR. For more information,
|
||||
@@ -285,7 +281,7 @@ check out the Elastic guide to <a class="reference external" href="https://www.e
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+12
-18
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Index — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Index — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="#" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -396,7 +390,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+78
-26
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -36,9 +33,6 @@
|
||||
<a href="#" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -87,7 +81,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer">
|
||||
<h1>parsedmarc documentation - Open source DMARC report analyzer and visualizer<a class="headerlink" href="#parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer" title="Permalink to this heading"></a></h1>
|
||||
<h1>parsedmarc documentation - Open source DMARC report analyzer and visualizer<a class="headerlink" href="#parsedmarc-documentation-open-source-dmarc-report-analyzer-and-visualizer" title="Link to this heading"></a></h1>
|
||||
<p><a class="reference external" href="https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml"><img alt="BuildStatus" src="https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml/badge.svg" /></a>
|
||||
<a class="reference external" href="https://codecov.io/gh/domainaware/parsedmarc"><img alt="CodeCoverage" src="https://codecov.io/gh/domainaware/parsedmarc/branch/master/graph/badge.svg" /></a>
|
||||
<a class="reference external" href="https://pypi.org/project/parsedmarc/"><img alt="PyPIPackage" src="https://img.shields.io/pypi/v/parsedmarc.svg" /></a>
|
||||
@@ -100,26 +94,83 @@ Please consider reviewing the open <a class="reference external" href="https://g
|
||||
Assistance on the pinned issues would be particularly helpful.</p>
|
||||
<p>Thanks to all <a class="reference external" href="https://github.com/domainaware/parsedmarc/graphs/contributors">contributors</a>!</p>
|
||||
</div>
|
||||
<a class="reference external image-reference" href="_static/screenshots/dmarc-summary-charts.png"><img alt="A screenshot of DMARC summary charts in Kibana" class="align-center" src="_images/dmarc-summary-charts.png" style="width: 754.0px; height: 449.0px;" /></a>
|
||||
<a class="reference external image-reference" href="_static/screenshots/dmarc-summary-charts.png"><img alt="A screenshot of DMARC summary charts in Kibana" class="align-center" src="_images/dmarc-summary-charts.png" style="width: 754.0px; height: 449.0px;" />
|
||||
</a>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> is a Python module and CLI utility for parsing DMARC reports.
|
||||
When used with Elasticsearch and Kibana (or Splunk), or with OpenSearch and Grafana, it works as a self-hosted
|
||||
open source alternative to commercial DMARC report processing services such
|
||||
as Agari Brand Protection, Dmarcian, OnDMARC, ProofPoint Email Fraud Defense,
|
||||
and Valimail.</p>
|
||||
<section id="features">
|
||||
<h2>Features<a class="headerlink" href="#features" title="Permalink to this heading"></a></h2>
|
||||
<h2>Features<a class="headerlink" href="#features" title="Link to this heading"></a></h2>
|
||||
<ul class="simple">
|
||||
<li><p>Parses draft and 1.0 standard aggregate/rua reports</p></li>
|
||||
<li><p>Parses forensic/failure/ruf reports</p></li>
|
||||
<li><p>Parses draft and 1.0 standard aggregate/rua DMARC reports</p></li>
|
||||
<li><p>Parses forensic/failure/ruf DMARC reports</p></li>
|
||||
<li><p>Parses reports from SMTP TLS Reporting</p></li>
|
||||
<li><p>Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API</p></li>
|
||||
<li><p>Transparently handles gzip or zip compressed reports</p></li>
|
||||
<li><p>Consistent data structures</p></li>
|
||||
<li><p>Simple JSON and/or CSV output</p></li>
|
||||
<li><p>Optionally email the results</p></li>
|
||||
<li><p>Optionally send the results to Elasticsearch/OpenSearch and/or Splunk, for use with
|
||||
premade dashboards</p></li>
|
||||
<li><p>Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use
|
||||
with premade dashboards</p></li>
|
||||
<li><p>Optionally send reports to Apache Kafka</p></li>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="python-compatibility">
|
||||
<h2>Python Compatibility<a class="headerlink" href="#python-compatibility" title="Link to this heading"></a></h2>
|
||||
<p>This project supports the following Python versions, which are either actively maintained or are the default versions
|
||||
for RHEL or Debian.</p>
|
||||
<table class="docutils align-default">
|
||||
<thead>
|
||||
<tr class="row-odd"><th class="head"><p>Version</p></th>
|
||||
<th class="head"><p>Supported</p></th>
|
||||
<th class="head"><p>Reason</p></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="row-even"><td><p>< 3.6</p></td>
|
||||
<td><p>❌</p></td>
|
||||
<td><p>End of Life (EOL)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>3.6</p></td>
|
||||
<td><p>❌</p></td>
|
||||
<td><p>Used in RHHEL 8, but not supported by project dependencies</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>3.7</p></td>
|
||||
<td><p>❌</p></td>
|
||||
<td><p>End of Life (EOL)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>3.8</p></td>
|
||||
<td><p>❌</p></td>
|
||||
<td><p>End of Life (EOL)</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>3.9</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Supported until August 2026 (Debian 11); May 2032 (RHEL 9)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>3.10</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Actively maintained</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>3.11</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Actively maintained; supported until June 2028 (Debian 12)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>3.12</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Actively maintained; supported until May 2035 (RHEL 10)</p></td>
|
||||
</tr>
|
||||
<tr class="row-even"><td><p>3.13</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Actively maintained; supported until June 2030 (Debian 13)</p></td>
|
||||
</tr>
|
||||
<tr class="row-odd"><td><p>3.14</p></td>
|
||||
<td><p>✅</p></td>
|
||||
<td><p>Actively maintained</p></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="toctree-wrapper compound">
|
||||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||||
<ul>
|
||||
@@ -132,6 +183,7 @@ premade dashboards</p></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="usage.html">Using parsedmarc</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="usage.html#cli-help">CLI help</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="usage.html#configuration-file">Configuration file</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="usage.html#multi-tenant-support">Multi-tenant support</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="usage.html#running-parsedmarc-as-a-systemd-service">Running parsedmarc as a systemd service</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -197,7 +249,7 @@ premade dashboards</p></li>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+21
-27
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Installation — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Installation — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -99,18 +93,18 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="installation">
|
||||
<h1>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h1>
|
||||
<h1>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h1>
|
||||
<section id="prerequisites">
|
||||
<h2>Prerequisites<a class="headerlink" href="#prerequisites" title="Permalink to this heading"></a></h2>
|
||||
<h2>Prerequisites<a class="headerlink" href="#prerequisites" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> works with Python 3 only.</p>
|
||||
<section id="testing-multiple-report-analyzers">
|
||||
<h3>Testing multiple report analyzers<a class="headerlink" href="#testing-multiple-report-analyzers" title="Permalink to this heading"></a></h3>
|
||||
<h3>Testing multiple report analyzers<a class="headerlink" href="#testing-multiple-report-analyzers" title="Link to this heading"></a></h3>
|
||||
<p>If you would like to test parsedmarc and another report processing
|
||||
solution at the same time, you can have up to two <code class="docutils literal notranslate"><span class="pre">mailto</span></code> URIs in each of the rua and ruf
|
||||
tags in your DMARC record, separated by commas.</p>
|
||||
</section>
|
||||
<section id="using-a-web-proxy">
|
||||
<h3>Using a web proxy<a class="headerlink" href="#using-a-web-proxy" title="Permalink to this heading"></a></h3>
|
||||
<h3>Using a web proxy<a class="headerlink" href="#using-a-web-proxy" title="Link to this heading"></a></h3>
|
||||
<p>If your system is behind a web proxy, you need to configure your system
|
||||
to use that proxy. To do this, edit <code class="docutils literal notranslate"><span class="pre">/etc/environment</span></code> and add your
|
||||
proxy details there, for example:</p>
|
||||
@@ -128,7 +122,7 @@ proxy details there, for example:</p>
|
||||
<p>This will set the proxy up for use system-wide, including for <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code>.</p>
|
||||
</section>
|
||||
<section id="using-microsoft-exchange">
|
||||
<h3>Using Microsoft Exchange<a class="headerlink" href="#using-microsoft-exchange" title="Permalink to this heading"></a></h3>
|
||||
<h3>Using Microsoft Exchange<a class="headerlink" href="#using-microsoft-exchange" title="Link to this heading"></a></h3>
|
||||
<p>If your mail server is Microsoft Exchange, ensure that it is patched to at
|
||||
least:</p>
|
||||
<ul class="simple">
|
||||
@@ -138,7 +132,7 @@ least:</p>
|
||||
</ul>
|
||||
</section>
|
||||
<section id="geoipupdate-setup">
|
||||
<h3>geoipupdate setup<a class="headerlink" href="#geoipupdate-setup" title="Permalink to this heading"></a></h3>
|
||||
<h3>geoipupdate setup<a class="headerlink" href="#geoipupdate-setup" title="Link to this heading"></a></h3>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>Starting in <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 7.1.0, a static copy of the
|
||||
@@ -210,7 +204,7 @@ job or scheduled task.</p>
|
||||
</section>
|
||||
</section>
|
||||
<section id="installing-parsedmarc">
|
||||
<h2>Installing parsedmarc<a class="headerlink" href="#installing-parsedmarc" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installing parsedmarc<a class="headerlink" href="#installing-parsedmarc" title="Link to this heading"></a></h2>
|
||||
<p>On Debian or Ubuntu systems, run:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>python3-pip<span class="w"> </span>python3-virtualenv<span class="w"> </span>python3-dev<span class="w"> </span>libxml2-dev<span class="w"> </span>libxslt-dev
|
||||
</pre></div>
|
||||
@@ -245,7 +239,7 @@ explicitly tell <code class="docutils literal notranslate"><span class="pre">vir
|
||||
</div>
|
||||
</section>
|
||||
<section id="optional-dependencies">
|
||||
<h2>Optional dependencies<a class="headerlink" href="#optional-dependencies" title="Permalink to this heading"></a></h2>
|
||||
<h2>Optional dependencies<a class="headerlink" href="#optional-dependencies" title="Link to this heading"></a></h2>
|
||||
<p>If you would like to be able to parse emails saved from Microsoft
|
||||
Outlook (i.e. OLE .msg files), install <code class="docutils literal notranslate"><span class="pre">msgconvert</span></code>:</p>
|
||||
<p>On Debian or Ubuntu systems, run:</p>
|
||||
@@ -266,7 +260,7 @@ Outlook (i.e. OLE .msg files), install <code class="docutils literal notranslate
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+16
-22
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Using the Kibana dashboards — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,7 +86,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="using-the-kibana-dashboards">
|
||||
<h1>Using the Kibana dashboards<a class="headerlink" href="#using-the-kibana-dashboards" title="Permalink to this heading"></a></h1>
|
||||
<h1>Using the Kibana dashboards<a class="headerlink" href="#using-the-kibana-dashboards" title="Link to this heading"></a></h1>
|
||||
<p>The Kibana DMARC dashboards are a human-friendly way to understand the
|
||||
results from incoming DMARC reports.</p>
|
||||
<div class="admonition note">
|
||||
@@ -101,7 +95,7 @@ results from incoming DMARC reports.</p>
|
||||
click on the Dashboard link on the left side menu of Kibana.</p>
|
||||
</div>
|
||||
<section id="dmarc-summary">
|
||||
<h2>DMARC Summary<a class="headerlink" href="#dmarc-summary" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Summary<a class="headerlink" href="#dmarc-summary" title="Link to this heading"></a></h2>
|
||||
<p>As the name suggests, this dashboard is the best place to start
|
||||
reviewing your aggregate DMARC data.</p>
|
||||
<p>Across the top of the dashboard, three pie charts display the percentage of
|
||||
@@ -158,7 +152,7 @@ the DMARC Summary dashboard. To view failures only, use the pie chart.</p>
|
||||
filters by clicking on Add Filter at the upper right of the page.</p>
|
||||
</section>
|
||||
<section id="dmarc-forensic-samples">
|
||||
<h2>DMARC Forensic Samples<a class="headerlink" href="#dmarc-forensic-samples" title="Permalink to this heading"></a></h2>
|
||||
<h2>DMARC Forensic Samples<a class="headerlink" href="#dmarc-forensic-samples" title="Link to this heading"></a></h2>
|
||||
<p>The DMARC Forensic Samples dashboard contains information on DMARC forensic
|
||||
reports (also known as failure reports or ruf reports). These reports contain
|
||||
samples of emails that have failed to pass DMARC.</p>
|
||||
@@ -182,7 +176,7 @@ supply the headers of sample emails. Very few provide the entire email.</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+23
-57
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>What about mailing lists? — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>What about mailing lists? — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -35,9 +32,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -86,21 +80,21 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section id="what-about-mailing-lists">
|
||||
<h1>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Permalink to this heading"></a></h1>
|
||||
<h1>What about mailing lists?<a class="headerlink" href="#what-about-mailing-lists" title="Link to this heading"></a></h1>
|
||||
<p>When you deploy DMARC on your domain, you might find that messages
|
||||
relayed by mailing lists are failing DMARC, most likely because the mailing
|
||||
list is spoofing your from address, and modifying the subject,
|
||||
footer, or other part of the message, thereby breaking the
|
||||
DKIM signature.</p>
|
||||
<section id="mailing-list-best-practices">
|
||||
<h2>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Permalink to this heading"></a></h2>
|
||||
<h2>Mailing list best practices<a class="headerlink" href="#mailing-list-best-practices" title="Link to this heading"></a></h2>
|
||||
<p>Ideally, a mailing list should forward messages without altering the
|
||||
headers or body content at all. <a class="reference external" href="https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html">Joe Nelson</a> does a fantastic job of
|
||||
explaining exactly what mailing lists should and shouldn’t do to be
|
||||
fully DMARC compliant. Rather than repeat his fine work, here’s a
|
||||
summary:</p>
|
||||
<section id="do">
|
||||
<h3>Do<a class="headerlink" href="#do" title="Permalink to this heading"></a></h3>
|
||||
<h3>Do<a class="headerlink" href="#do" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p>Retain headers from the original message</p></li>
|
||||
<li><p>Add <a class="reference external" href="https://tools.ietf.org/html/rfc2369">RFC 2369</a> List-Unsubscribe headers to outgoing messages, instead of
|
||||
@@ -120,7 +114,7 @@ adding unsubscribe links to the body</p></li>
|
||||
these headers.</p>
|
||||
</section>
|
||||
<section id="do-not">
|
||||
<h3>Do not<a class="headerlink" href="#do-not" title="Permalink to this heading"></a></h3>
|
||||
<h3>Do not<a class="headerlink" href="#do-not" title="Link to this heading"></a></h3>
|
||||
<ul class="simple">
|
||||
<li><p>Remove or modify any existing headers from the original message, including
|
||||
From, Date, Subject, etc.</p></li>
|
||||
@@ -137,13 +131,9 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
</section>
|
||||
<section id="mailman-2">
|
||||
<h3>Mailman 2<a class="headerlink" href="#mailman-2" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailman 2<a class="headerlink" href="#mailman-2" title="Link to this heading"></a></h3>
|
||||
<p>Navigate to General Settings, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 74%" />
|
||||
<col style="width: 26%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -173,10 +163,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Non-digest options, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 66%" />
|
||||
<col style="width: 34%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -194,10 +180,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -215,16 +197,12 @@ to the mailing list post address, and not their email address.</p>
|
||||
</table>
|
||||
</section>
|
||||
<section id="mailman-3">
|
||||
<h3>Mailman 3<a class="headerlink" href="#mailman-3" title="Permalink to this heading"></a></h3>
|
||||
<h3>Mailman 3<a class="headerlink" href="#mailman-3" title="Link to this heading"></a></h3>
|
||||
<p>Navigate to Settings> List Identity</p>
|
||||
<p>Make Subject prefix blank.</p>
|
||||
<p>Navigate to Settings> Alter Messages</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -252,10 +230,6 @@ to the mailing list post address, and not their email address.</p>
|
||||
<p>Navigate to Settings> DMARC Mitigation</p>
|
||||
<p>Configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 52%" />
|
||||
<col style="width: 48%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -279,13 +253,13 @@ command line instead, for example:</p>
|
||||
<p>Then restart mailman core.</p>
|
||||
</section>
|
||||
<section id="listserv">
|
||||
<h3>LISTSERV<a class="headerlink" href="#listserv" title="Permalink to this heading"></a></h3>
|
||||
<h3>LISTSERV<a class="headerlink" href="#listserv" title="Link to this heading"></a></h3>
|
||||
<p><a class="reference external" href="https://www.lsoft.com/news/dmarc-issue1-2018.asp">LISTSERV 16.0-2017a</a> and higher will rewrite the From header for domains
|
||||
that enforce with a DMARC quarantine or reject policy.</p>
|
||||
<p>Some additional steps are needed for Linux hosts.</p>
|
||||
</section>
|
||||
<section id="workarounds">
|
||||
<h3>Workarounds<a class="headerlink" href="#workarounds" title="Permalink to this heading"></a></h3>
|
||||
<h3>Workarounds<a class="headerlink" href="#workarounds" title="Link to this heading"></a></h3>
|
||||
<p>If a mailing list must go <strong>against</strong> best practices and
|
||||
modify the message (e.g. to add a required legal footer), the mailing
|
||||
list administrator must configure the list to replace the From address of the
|
||||
@@ -293,13 +267,9 @@ message (also known as munging) with the address of the mailing list, so they
|
||||
no longer spoof email addresses with domains protected by DMARC.</p>
|
||||
<p>Configuration steps for common mailing list platforms are listed below.</p>
|
||||
<section id="id1">
|
||||
<h4>Mailman 2<a class="headerlink" href="#id1" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 2<a class="headerlink" href="#id1" title="Link to this heading"></a></h4>
|
||||
<p>Navigate to Privacy Options> Sending Filters, and configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 79%" />
|
||||
<col style="width: 21%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -328,13 +298,9 @@ the original sender.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="id2">
|
||||
<h4>Mailman 3<a class="headerlink" href="#id2" title="Permalink to this heading"></a></h4>
|
||||
<h4>Mailman 3<a class="headerlink" href="#id2" title="Link to this heading"></a></h4>
|
||||
<p>In the DMARC Mitigations tab of the Settings page, configure the settings below</p>
|
||||
<table class="docutils align-default">
|
||||
<colgroup>
|
||||
<col style="width: 49%" />
|
||||
<col style="width: 51%" />
|
||||
</colgroup>
|
||||
<tbody>
|
||||
<tr class="row-odd"><td><p><strong>Setting</strong></p></td>
|
||||
<td><p><strong>Value</strong></p></td>
|
||||
@@ -370,7 +336,7 @@ the original sender.</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
BIN
Binary file not shown.
+16
-22
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>OpenSearch and Grafana — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -92,15 +86,15 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="opensearch-and-grafana">
|
||||
<h1>OpenSearch and Grafana<a class="headerlink" href="#opensearch-and-grafana" title="Permalink to this heading"></a></h1>
|
||||
<h1>OpenSearch and Grafana<a class="headerlink" href="#opensearch-and-grafana" title="Link to this heading"></a></h1>
|
||||
<p>To set up visual dashboards of DMARC data, install OpenSearch and Grafana.</p>
|
||||
<section id="installation">
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Permalink to this heading"></a></h2>
|
||||
<h2>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h2>
|
||||
<p>OpenSearch: <a class="reference external" href="https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/">https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/</a>
|
||||
Grafana: <a class="reference external" href="https://grafana.com/docs/grafana/latest/setup-grafana/installation/">https://grafana.com/docs/grafana/latest/setup-grafana/installation/</a></p>
|
||||
</section>
|
||||
<section id="records-retention">
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this heading"></a></h2>
|
||||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||||
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
|
||||
index for each day to make it easy to comply with records
|
||||
retention regulations such as GDPR.</p>
|
||||
@@ -118,7 +112,7 @@ retention regulations such as GDPR.</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+21
-27
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Sample outputs — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Sample outputs — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -101,9 +95,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="sample-outputs">
|
||||
<h1>Sample outputs<a class="headerlink" href="#sample-outputs" title="Permalink to this heading"></a></h1>
|
||||
<h1>Sample outputs<a class="headerlink" href="#sample-outputs" title="Link to this heading"></a></h1>
|
||||
<section id="sample-aggregate-report-output">
|
||||
<h2>Sample aggregate report output<a class="headerlink" href="#sample-aggregate-report-output" title="Permalink to this heading"></a></h2>
|
||||
<h2>Sample aggregate report output<a class="headerlink" href="#sample-aggregate-report-output" title="Link to this heading"></a></h2>
|
||||
<p>Here are the results from parsing the <a class="reference external" href="https://dmarc.org/wiki/FAQ#I_need_to_implement_aggregate_reports.2C_what_do_they_look_like.3F">example</a>
|
||||
report from the <a class="reference external" href="http://dmarc.org">dmarc.org</a> wiki. It’s actually an older draft of
|
||||
the 1.0 report schema standardized in
|
||||
@@ -112,7 +106,7 @@ This draft schema is still in wide use.</p>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> produces consistent, normalized output, regardless
|
||||
of the report schema.</p>
|
||||
<section id="json-aggregate-report">
|
||||
<h3>JSON aggregate report<a class="headerlink" href="#json-aggregate-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON aggregate report<a class="headerlink" href="#json-aggregate-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"xml_schema"</span><span class="p">:</span><span class="w"> </span><span class="s2">"draft"</span><span class="p">,</span>
|
||||
<span class="w"> </span><span class="nt">"report_metadata"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||||
@@ -181,7 +175,7 @@ of the report schema.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="csv-aggregate-report">
|
||||
<h3>CSV aggregate report<a class="headerlink" href="#csv-aggregate-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>CSV aggregate report<a class="headerlink" href="#csv-aggregate-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
|
||||
draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,,example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
|
||||
</pre></div>
|
||||
@@ -189,11 +183,11 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</section>
|
||||
</section>
|
||||
<section id="sample-forensic-report-output">
|
||||
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Permalink to this heading"></a></h2>
|
||||
<h2>Sample forensic report output<a class="headerlink" href="#sample-forensic-report-output" title="Link to this heading"></a></h2>
|
||||
<p>Thanks to GitHub user <a class="reference external" href="https://github.com/xennn">xennn</a> for the anonymized
|
||||
<a class="reference external" href="https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml">forensic report email sample</a>.</p>
|
||||
<section id="json-forensic-report">
|
||||
<h3>JSON forensic report<a class="headerlink" href="#json-forensic-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON forensic report<a class="headerlink" href="#json-forensic-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"feedback_type"</span><span class="p">:</span><span class="w"> </span><span class="s2">"auth-failure"</span><span class="p">,</span>
|
||||
<span class="w"> </span><span class="nt">"user_agent"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Lua/1.0"</span><span class="p">,</span>
|
||||
@@ -282,14 +276,14 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391
|
||||
</div>
|
||||
</section>
|
||||
<section id="csv-forensic-report">
|
||||
<h3>CSV forensic report<a class="headerlink" href="#csv-forensic-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>CSV forensic report<a class="headerlink" href="#csv-forensic-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
|
||||
auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
|
||||
</pre></div>
|
||||
</div>
|
||||
</section>
|
||||
<section id="json-smtp-tls-report">
|
||||
<h3>JSON SMTP TLS report<a class="headerlink" href="#json-smtp-tls-report" title="Permalink to this heading"></a></h3>
|
||||
<h3>JSON SMTP TLS report<a class="headerlink" href="#json-smtp-tls-report" title="Link to this heading"></a></h3>
|
||||
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
|
||||
<span class="w"> </span><span class="p">{</span>
|
||||
<span class="w"> </span><span class="nt">"organization_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Example Inc."</span><span class="p">,</span>
|
||||
@@ -345,7 +339,7 @@ auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+12
-18
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Python Module Index — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Python Module Index — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -133,7 +127,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+12
-18
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Search — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Search — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<script src="_static/searchtools.js"></script>
|
||||
<script src="_static/language_data.js"></script>
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="#" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -106,7 +100,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+14
-20
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Splunk — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Splunk — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,7 +82,7 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="splunk">
|
||||
<h1>Splunk<a class="headerlink" href="#splunk" title="Permalink to this heading"></a></h1>
|
||||
<h1>Splunk<a class="headerlink" href="#splunk" title="Link to this heading"></a></h1>
|
||||
<p>Starting in version 4.3.0 <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> supports sending aggregate and/or
|
||||
forensic DMARC data to a Splunk <a class="reference external" href="http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC">HTTP Event collector (HEC)</a>.</p>
|
||||
<p>The project repository contains <a class="reference external" href="https://github.com/domainaware/parsedmarc/tree/master/splunk">XML files</a> for premade Splunk
|
||||
@@ -116,7 +110,7 @@ easier and more flexible filtering options.</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+43
-27
@@ -1,24 +1,21 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="./">
|
||||
<head>
|
||||
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
|
||||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Using parsedmarc — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
|
||||
<title>Using parsedmarc — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/sphinx_highlight.js"></script>
|
||||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
@@ -37,9 +34,6 @@
|
||||
<a href="index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -54,6 +48,7 @@
|
||||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Using parsedmarc</a><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#cli-help">CLI help</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#configuration-file">Configuration file</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#multi-tenant-support">Multi-tenant support</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="#running-parsedmarc-as-a-systemd-service">Running parsedmarc as a systemd service</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -93,9 +88,9 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<section class="tex2jax_ignore mathjax_ignore" id="using-parsedmarc">
|
||||
<h1>Using parsedmarc<a class="headerlink" href="#using-parsedmarc" title="Permalink to this heading"></a></h1>
|
||||
<h1>Using parsedmarc<a class="headerlink" href="#using-parsedmarc" title="Link to this heading"></a></h1>
|
||||
<section id="cli-help">
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Permalink to this heading"></a></h2>
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Link to this heading"></a></h2>
|
||||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[--aggregate-json-filename AGGREGATE_JSON_FILENAME]
|
||||
[--forensic-json-filename FORENSIC_JSON_FILENAME]
|
||||
@@ -147,7 +142,7 @@ configuration file, described below.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="configuration-file">
|
||||
<h2>Configuration file<a class="headerlink" href="#configuration-file" title="Permalink to this heading"></a></h2>
|
||||
<h2>Configuration file<a class="headerlink" href="#configuration-file" title="Link to this heading"></a></h2>
|
||||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> can be configured by supplying the path to an INI file</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>parsedmarc<span class="w"> </span>-c<span class="w"> </span>/etc/parsedmarc.ini
|
||||
</pre></div>
|
||||
@@ -211,8 +206,10 @@ Elasticsearch, Splunk and/or S3</p></li>
|
||||
Elasticsearch, Splunk and/or S3</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">save_smtp_tls</span></code> - bool: Save SMTP-STS report data to
|
||||
Elasticsearch, Splunk and/or S3</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">index_prefix_domain_map</span></code> - bool: A path mapping of Opensearch/Elasticsearch index prefixes to domain names</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">strip_attachment_payloads</span></code> - bool: Remove attachment
|
||||
payloads from results</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">silent</span></code> - bool: Set this to <code class="docutils literal notranslate"><span class="pre">False</span></code> to output results to STDOUT</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">output</span></code> - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options.</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">aggregate_json_filename</span></code> - str: filename for the aggregate
|
||||
JSON output file</p></li>
|
||||
@@ -261,7 +258,7 @@ before saving. Default <code class="docutils literal notranslate"><span class="p
|
||||
IDLE response or the number of seconds until the next
|
||||
mail check (Default: <code class="docutils literal notranslate"><span class="pre">30</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">since</span></code> - str: Search for messages since certain time. (Examples: <code class="docutils literal notranslate"><span class="pre">5m|3h|2d|1w</span></code>)
|
||||
Acceptable units - {“m”:“minutes”, “h”:“hours”, “d”:“days”, “w”:“weeks”}).
|
||||
Acceptable units - {“m”:“minutes”, “h”:“hours”, “d”:“days”, “w”:“weeks”}.
|
||||
Defaults to <code class="docutils literal notranslate"><span class="pre">1d</span></code> if incorrect value is provided.</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -355,7 +352,7 @@ or URLs (e.g. <code class="docutils literal notranslate"><span class="pre">127.0
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">user</span></code> - str: Basic auth username</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">password</span></code> - str: Basic auth password</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">apiKey</span></code> - str: API key</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">api_key</span></code> - str: API key</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">ssl</span></code> - bool: Use an encrypted SSL/TLS connection
|
||||
(Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">timeout</span></code> - float: Timeout in seconds (Default: 60)</p></li>
|
||||
@@ -382,7 +379,7 @@ or URLs (e.g. <code class="docutils literal notranslate"><span class="pre">127.0
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">user</span></code> - str: Basic auth username</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">password</span></code> - str: Basic auth password</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">apiKey</span></code> - str: API key</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">api_key</span></code> - str: API key</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">ssl</span></code> - bool: Use an encrypted SSL/TLS connection
|
||||
(Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">timeout</span></code> - float: Timeout in seconds (Default: 60)</p></li>
|
||||
@@ -504,7 +501,7 @@ When <code class="docutils literal notranslate"><span class="pre">False</span></
|
||||
</li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir</span></code></p>
|
||||
<ul class="simple">
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">reports_folder</span></code> - str: Full path for mailbox maidir location (Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_path</span></code> - str: Full path for mailbox maidir location (Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
|
||||
<li><p><code class="docutils literal notranslate"><span class="pre">maildir_create</span></code> - bool: Create maildir if not present (Default: False)</p></li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -577,8 +574,27 @@ Check current usage (from Management -> Dev Tools -> Console):</p>
|
||||
<p>Increasing this value increases resource usage.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="multi-tenant-support">
|
||||
<h2>Multi-tenant support<a class="headerlink" href="#multi-tenant-support" title="Link to this heading"></a></h2>
|
||||
<p>Starting in <code class="docutils literal notranslate"><span class="pre">8.19.0</span></code>, ParseDMARC provides multi-tenant support by placing data into separate OpenSearch or Elasticsearch index prefixes. To set this up, create a YAML file that is formatted where each key is a tenant name, and the value is a list of domains related to that tenant, not including subdomains, like this:</p>
|
||||
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">example</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">example.com</span>
|
||||
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">example.net</span>
|
||||
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">example.org</span>
|
||||
|
||||
<span class="nt">whalensolutions</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">whalensolutions.com</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>Save it to disk where the user running ParseDMARC can read it, then set <code class="docutils literal notranslate"><span class="pre">index_prefix_domain_map</span></code> to that filepath in the <code class="docutils literal notranslate"><span class="pre">[general]</span></code> section of the ParseDMARC configuration file and do not set an <code class="docutils literal notranslate"><span class="pre">index_prefix</span></code> option in the <code class="docutils literal notranslate"><span class="pre">[elasticsearch]</span></code> or <code class="docutils literal notranslate"><span class="pre">[opensearch]</span></code> sections.</p>
|
||||
<p>When configured correctly, if ParseDMARC finds that a report is related to a domain in the mapping, the report will be saved in an index name that has the tenant name prefixed to it with a trailing underscore. Then, you can use the security features of Opensearch or the ELK stack to only grant users access to the indexes that they need.</p>
|
||||
<div class="admonition note">
|
||||
<p class="admonition-title">Note</p>
|
||||
<p>A domain cannot be used in multiple tenant lists. Only the first prefix list that contains the matching domain is used.</p>
|
||||
</div>
|
||||
</section>
|
||||
<section id="running-parsedmarc-as-a-systemd-service">
|
||||
<h2>Running parsedmarc as a systemd service<a class="headerlink" href="#running-parsedmarc-as-a-systemd-service" title="Permalink to this heading"></a></h2>
|
||||
<h2>Running parsedmarc as a systemd service<a class="headerlink" href="#running-parsedmarc-as-a-systemd-service" title="Link to this heading"></a></h2>
|
||||
<p>Use systemd to run <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> as a service and process reports as
|
||||
they arrive.</p>
|
||||
<p>Protect the <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> configuration file from prying eyes</p>
|
||||
@@ -654,7 +670,7 @@ as well as the current process (newest to oldest), run:</p>
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
Reference in New Issue
Block a user