mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-09-03 12:47:15 +00:00
Update docs
This commit is contained in:
Sean Whalen
+12
-18
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>Overview: module code — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -98,7 +92,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+185
-109
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
|
||||
<title>parsedmarc — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
|
||||
<script src="../_static/jquery.js"></script>
|
||||
<script src="../_static/underscore.js"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../_static/doctools.js"></script>
|
||||
<script src="../_static/sphinx_highlight.js"></script>
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -89,44 +83,45 @@
|
||||
|
||||
<span class="sd">"""A Python package for parsing DMARC reports"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">binascii</span>
|
||||
<span class="kn">import</span> <span class="nn">email</span>
|
||||
<span class="kn">import</span> <span class="nn">email.utils</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">import</span> <span class="nn">mailbox</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">import</span> <span class="nn">shutil</span>
|
||||
<span class="kn">import</span> <span class="nn">tempfile</span>
|
||||
<span class="kn">import</span> <span class="nn">xml.parsers.expat</span> <span class="k">as</span> <span class="nn">expat</span>
|
||||
<span class="kn">import</span> <span class="nn">zipfile</span>
|
||||
<span class="kn">import</span> <span class="nn">zlib</span>
|
||||
<span class="kn">from</span> <span class="nn">base64</span> <span class="kn">import</span> <span class="n">b64decode</span>
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span> <span class="nn">csv</span> <span class="kn">import</span> <span class="n">DictWriter</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span> <span class="nn">io</span> <span class="kn">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
||||
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Callable</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">binascii</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">email</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">email.utils</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xml.parsers.expat</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">expat</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">zipfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">zlib</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">base64</span><span class="w"> </span><span class="kn">import</span> <span class="n">b64decode</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">csv</span><span class="w"> </span><span class="kn">import</span> <span class="n">DictWriter</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">,</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">io</span><span class="w"> </span><span class="kn">import</span> <span class="n">BytesIO</span><span class="p">,</span> <span class="n">StringIO</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Callable</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">mailparser</span>
|
||||
<span class="kn">import</span> <span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span> <span class="nn">expiringdict</span> <span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span> <span class="nn">lxml</span> <span class="kn">import</span> <span class="n">etree</span>
|
||||
<span class="kn">from</span> <span class="nn">mailsuite.smtp</span> <span class="kn">import</span> <span class="n">send_email</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">xmltodict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">lxml</span><span class="w"> </span><span class="kn">import</span> <span class="n">etree</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">mailsuite.smtp</span><span class="w"> </span><span class="kn">import</span> <span class="n">send_email</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.mail</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.mail</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">IMAPConnection</span><span class="p">,</span>
|
||||
<span class="n">MSGraphConnection</span><span class="p">,</span>
|
||||
<span class="n">GmailConnection</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"8.18.1"</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">__version__</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_base_domain</span><span class="p">,</span> <span class="n">get_ip_address_info</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">is_outlook_msg</span><span class="p">,</span> <span class="n">convert_outlook_msg</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse_email</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
|
||||
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
|
||||
@@ -145,27 +140,42 @@
|
||||
<span class="n">REVERSE_DNS_MAP</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ParserError"><a class="viewcode-back" href="../api.html#parsedmarc.ParserError">[docs]</a><span class="k">class</span> <span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ParserError">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.ParserError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised whenever the parser fails for some reason"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidDMARCReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidDMARCReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidDMARCReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidDMARCReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidDMARCReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidSMTPTLSReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidSMTPTLSReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid SMTP TLS report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidAggregateReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC aggregate report is encountered"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="InvalidForensicReport">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.InvalidForensicReport">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">InvalidForensicReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an invalid DMARC forensic report is encountered"""</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_report_record</span><span class="p">(</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_report_record</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
@@ -328,7 +338,7 @@
|
||||
<span class="k">return</span> <span class="n">new_record</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">):</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">new_failure_details</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="n">result_type</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">"result-type"</span><span class="p">],</span>
|
||||
@@ -364,7 +374,7 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">):</span>
|
||||
<span class="n">policy_types</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"tlsa"</span><span class="p">,</span> <span class="s2">"sts"</span><span class="p">,</span> <span class="s2">"no-policy-found"</span><span class="p">]</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">"policy"</span><span class="p">][</span><span class="s2">"policy-domain"</span><span class="p">]</span>
|
||||
@@ -401,7 +411,9 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_smtp_tls_report_json"><a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a><span class="k">def</span> <span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="parse_smtp_tls_report_json">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Parses and validates an SMTP TLS report"""</span>
|
||||
<span class="n">required_fields</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"organization-name"</span><span class="p">,</span>
|
||||
@@ -440,7 +452,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Converts one oor more parsed SMTP TLS reports into a list of single</span>
|
||||
<span class="sd"> layer OrderedDict objects suitable for use in a CSV"""</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="ow">is</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
||||
@@ -475,7 +490,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed SMTP TLS reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -520,7 +538,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_xml"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_xml">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
<span class="n">xml</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
@@ -706,7 +727,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_report"><a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a><span class="k">def</span> <span class="nf">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="extract_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Extracts text from a zip or gzip file, as a base64-encoded string,</span>
|
||||
<span class="sd"> file-like object, or bytes.</span>
|
||||
@@ -760,7 +784,10 @@
|
||||
<span class="k">return</span> <span class="n">report</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="extract_report_from_file_path"><a class="viewcode-back" href="../api.html#parsedmarc.extract_report_from_file_path">[docs]</a><span class="k">def</span> <span class="nf">extract_report_from_file_path</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="extract_report_from_file_path">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.extract_report_from_file_path">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">extract_report_from_file_path</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Extracts report from a file at the given file_path"""</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file_path</span><span class="p">,</span> <span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">report_file</span><span class="p">:</span>
|
||||
@@ -769,7 +796,10 @@
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"File was not found"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_aggregate_report_file">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_file</span><span class="p">(</span>
|
||||
<span class="n">_input</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -817,7 +847,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed aggregate reports to list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
@@ -830,7 +863,7 @@
|
||||
<span class="sd"> format</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">to_str</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="ow">is</span> <span class="n">OrderedDict</span><span class="p">:</span>
|
||||
@@ -935,7 +968,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_aggregate_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_aggregate_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_aggregate_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed aggregate reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -1000,7 +1036,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report"><a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a><span class="k">def</span> <span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_forensic_report">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_forensic_report">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_forensic_report</span><span class="p">(</span>
|
||||
<span class="n">feedback_report</span><span class="p">,</span>
|
||||
<span class="n">sample</span><span class="p">,</span>
|
||||
<span class="n">msg_date</span><span class="p">,</span>
|
||||
@@ -1140,7 +1179,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span><span class="s2">"Unexpected error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv_rows">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to a list of dicts in flat CSV</span>
|
||||
<span class="sd"> format</span>
|
||||
@@ -1176,7 +1218,10 @@
|
||||
<span class="k">return</span> <span class="n">rows</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parsed_forensic_reports_to_csv">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parsed_forensic_reports_to_csv">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts one or more parsed forensic reports to flat CSV format, including</span>
|
||||
<span class="sd"> headers</span>
|
||||
@@ -1228,7 +1273,10 @@
|
||||
<span class="k">return</span> <span class="n">csv_file</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_report_email"><a class="viewcode-back" href="../api.html#parsedmarc.parse_report_email">[docs]</a><span class="k">def</span> <span class="nf">parse_report_email</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_report_email">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_email">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_email</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -1270,7 +1318,7 @@
|
||||
<span class="n">input_</span> <span class="o">=</span> <span class="n">input_</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoding</span><span class="o">=</span><span class="s2">"utf8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
||||
<span class="n">msg</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
<span class="n">msg_headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">msg</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">())</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">format_datetime</span><span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="s2">"Date"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Date"</span><span class="p">])</span>
|
||||
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
|
||||
@@ -1286,12 +1334,16 @@
|
||||
<span class="k">if</span> <span class="s2">"Subject"</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">"Subject"</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_content_type</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">part</span><span class="o">.</span><span class="n">get_payload</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="p">[</span><span class="n">payload</span><span class="p">]</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"multipart/"</span><span class="p">):</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/html"</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/feedback-report"</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"Feedback-Type"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
@@ -1302,13 +1354,12 @@
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">feedback_report</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"</span><span class="se">\\</span><span class="s2">n"</span><span class="p">,</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
||||
<span class="n">feedback_report</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/rfc822-headers"</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"message/rfc822"</span><span class="p">:</span>
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"application/tlsrpt+json"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"{"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">))</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
@@ -1320,7 +1371,6 @@
|
||||
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">"text/plain"</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"A message claiming to be from you has failed"</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
@@ -1347,13 +1397,14 @@
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">ns</span> <span class="o">=</span> <span class="n">nameservers</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
|
||||
<span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">"utf-8"</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">"replace"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"{"</span><span class="p">):</span>
|
||||
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"smtp_tls"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">payload</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">"<"</span><span class="p">):</span>
|
||||
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
<span class="n">payload</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
|
||||
@@ -1361,25 +1412,24 @@
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
|
||||
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
||||
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
|
||||
<span class="p">[(</span><span class="s2">"report_type"</span><span class="p">,</span> <span class="s2">"aggregate"</span><span class="p">),</span> <span class="p">(</span><span class="s2">"report"</span><span class="p">,</span> <span class="n">aggregate_report</span><span class="p">)]</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">result</span>
|
||||
|
||||
<span class="k">except</span> <span class="p">(</span><span class="ne">TypeError</span><span class="p">,</span> <span class="ne">ValueError</span><span class="p">,</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">):</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
<span class="k">except</span> <span class="n">InvalidAggregateReport</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" '</span>
|
||||
<span class="s2">"is not a valid "</span>
|
||||
<span class="s2">"aggregate DMARC report: </span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Message with subject "</span><span class="si">{0}</span><span class="s1">" is not a valid DMARC report'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">subject</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="s1">'Unable to parse message with subject "</span><span class="si">{0}</span><span class="s1">": </span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
@@ -1420,7 +1470,10 @@
|
||||
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_report_file"><a class="viewcode-back" href="../api.html#parsedmarc.parse_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_report_file</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_report_file">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_report_file">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_report_file</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
@@ -1499,7 +1552,10 @@
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mbox"><a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_mbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mbox</span><span class="p">(</span>
|
||||
<span class="n">input_</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
|
||||
@@ -1584,7 +1640,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox"><a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
||||
<span class="n">archive_folder</span><span class="o">=</span><span class="s2">"Archive"</span><span class="p">,</span>
|
||||
@@ -1666,7 +1725,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="n">since</span><span class="p">:</span>
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="mi">1440</span> <span class="c1"># default one day</span>
|
||||
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\d+[mhd]$"</span><span class="p">,</span> <span class="n">since</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\d+[mhdw]$"</span><span class="p">,</span> <span class="n">since</span><span class="p">):</span>
|
||||
<span class="n">s</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">r</span><span class="s2">"(\d+)"</span><span class="p">,</span> <span class="n">since</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">s</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">==</span> <span class="s2">"m"</span><span class="p">:</span>
|
||||
<span class="n">_since</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">s</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
@@ -1688,16 +1747,20 @@
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">IMAPConnection</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Only days and weeks values in 'since' option are </span><span class="se">\</span>
|
||||
<span class="s2"> considered for IMAP conections. Examples: 2d or 1w"</span>
|
||||
<span class="s2"> considered for IMAP connections. Examples: 2d or 1w"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">MSGraphConnection</span><span class="p">):</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span>
|
||||
<span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">)</span>
|
||||
<span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span> <span class="o">+</span> <span class="s2">"Z"</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">connection</span><span class="p">,</span> <span class="n">GmailConnection</span><span class="p">):</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">since</span> <span class="o">=</span> <span class="p">(</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="o">-</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">minutes</span><span class="o">=</span><span class="n">_since</span><span class="p">))</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">%s</span><span class="s2">"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">current_time</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
@@ -1897,7 +1960,10 @@
|
||||
<span class="k">return</span> <span class="n">results</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="watch_inbox"><a class="viewcode-back" href="../api.html#parsedmarc.watch_inbox">[docs]</a><span class="k">def</span> <span class="nf">watch_inbox</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="watch_inbox">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.watch_inbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">watch_inbox</span><span class="p">(</span>
|
||||
<span class="n">mailbox_connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
|
||||
<span class="n">callback</span><span class="p">:</span> <span class="n">Callable</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="s2">"INBOX"</span><span class="p">,</span>
|
||||
@@ -1941,7 +2007,7 @@
|
||||
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">check_callback</span><span class="p">(</span><span class="n">connection</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">check_callback</span><span class="p">(</span><span class="n">connection</span><span class="p">):</span>
|
||||
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
|
||||
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_mailbox</span><span class="p">(</span>
|
||||
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
||||
@@ -1965,7 +2031,8 @@
|
||||
<span class="n">mailbox_connection</span><span class="o">.</span><span class="n">watch</span><span class="p">(</span><span class="n">check_callback</span><span class="o">=</span><span class="n">check_callback</span><span class="p">,</span> <span class="n">check_timeout</span><span class="o">=</span><span class="n">check_timeout</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">append_json</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_json</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="n">output_json</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="n">ensure_ascii</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">indent</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
@@ -1988,7 +2055,7 @@
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">output_json</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">csv</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">append_csv</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="n">csv</span><span class="p">):</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">filename</span><span class="p">,</span> <span class="s2">"a+"</span><span class="p">,</span> <span class="n">newline</span><span class="o">=</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">output</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_END</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># strip the headers from the CSV</span>
|
||||
@@ -2000,7 +2067,9 @@
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">csv</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_output"><a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a><span class="k">def</span> <span class="nf">save_output</span><span class="p">(</span>
|
||||
<div class="viewcode-block" id="save_output">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_output</span><span class="p">(</span>
|
||||
<span class="n">results</span><span class="p">,</span>
|
||||
<span class="n">output_directory</span><span class="o">=</span><span class="s2">"output"</span><span class="p">,</span>
|
||||
<span class="n">aggregate_json_filename</span><span class="o">=</span><span class="s2">"aggregate.json"</span><span class="p">,</span>
|
||||
@@ -2086,7 +2155,10 @@
|
||||
<span class="n">sample_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_report_zip"><a class="viewcode-back" href="../api.html#parsedmarc.get_report_zip">[docs]</a><span class="k">def</span> <span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_report_zip">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.get_report_zip">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_report_zip</span><span class="p">(</span><span class="n">results</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Creates a zip file of parsed report output</span>
|
||||
|
||||
@@ -2097,7 +2169,7 @@
|
||||
<span class="sd"> bytes: zip file bytes</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_subdir</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">):</span>
|
||||
<span class="n">subdir_path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_path</span><span class="p">,</span> <span class="n">subdir</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">subdir_root</span><span class="p">,</span> <span class="n">subdir_dirs</span><span class="p">,</span> <span class="n">subdir_files</span> <span class="ow">in</span> <span class="n">os</span><span class="o">.</span><span class="n">walk</span><span class="p">(</span><span class="n">subdir_path</span><span class="p">):</span>
|
||||
<span class="k">for</span> <span class="n">subdir_file</span> <span class="ow">in</span> <span class="n">subdir_files</span><span class="p">:</span>
|
||||
@@ -2131,7 +2203,10 @@
|
||||
<span class="k">return</span> <span class="n">storage</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="email_results"><a class="viewcode-back" href="../api.html#parsedmarc.email_results">[docs]</a><span class="k">def</span> <span class="nf">email_results</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="email_results">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.email_results">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">email_results</span><span class="p">(</span>
|
||||
<span class="n">results</span><span class="p">,</span>
|
||||
<span class="n">host</span><span class="p">,</span>
|
||||
<span class="n">mail_from</span><span class="p">,</span>
|
||||
@@ -2199,6 +2274,7 @@
|
||||
<span class="n">attachments</span><span class="o">=</span><span class="n">attachments</span><span class="p">,</span>
|
||||
<span class="n">plain_message</span><span class="o">=</span><span class="n">message</span><span class="p">,</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -2208,7 +2284,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.elastic — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,10 +82,10 @@
|
||||
<h1>Source code for parsedmarc.elastic</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl.search</span> <span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch_dsl</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
<span class="n">Document</span><span class="p">,</span>
|
||||
@@ -105,23 +99,26 @@
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">elasticsearch.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="ElasticsearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a><span class="k">class</span> <span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="ElasticsearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an Elasticsearch error occurs"""</span></div>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -131,20 +128,20 @@
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -175,36 +172,36 @@
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -219,21 +216,21 @@
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
@@ -241,8 +238,8 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -264,7 +261,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -274,7 +271,7 @@
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -283,7 +280,7 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -309,8 +306,8 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -321,7 +318,7 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
@@ -342,17 +339,22 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -364,7 +366,7 @@
|
||||
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
||||
<span class="sd"> username (str): The username to use for authentication</span>
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> apiKey (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -379,12 +381,15 @@
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create Elasticsearch indexes</span>
|
||||
|
||||
@@ -407,7 +412,10 @@
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -453,7 +461,10 @@
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -604,7 +615,10 @@
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -640,8 +654,8 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
@@ -650,20 +664,35 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.from"</span><span class="p">:</span> <span class="n">from_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the FROM header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">from_</span><span class="p">[</span><span class="s2">"sample.headers.from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">from_query</span>
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.to"</span><span class="p">:</span> <span class="n">to_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the TO header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">to_</span><span class="p">[</span><span class="s2">"sample.headers.to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">to_query</span>
|
||||
<span class="k">if</span> <span class="s2">"subject"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.subject"</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
|
||||
@@ -677,7 +706,9 @@
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="s2">"Elasticsearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
@@ -713,7 +744,7 @@
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date</span><span class="p">,</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
@@ -754,7 +785,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -903,6 +937,7 @@
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">"Elasticsearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -912,7 +947,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,9 +82,9 @@
|
||||
<h1>Source code for parsedmarc.opensearch</h1><div class="highlight"><pre>
|
||||
<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy</span> <span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
|
||||
<span class="n">Q</span><span class="p">,</span>
|
||||
<span class="n">connections</span><span class="p">,</span>
|
||||
<span class="n">Object</span><span class="p">,</span>
|
||||
@@ -105,23 +99,26 @@
|
||||
<span class="n">Date</span><span class="p">,</span>
|
||||
<span class="n">Search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">opensearchpy.helpers</span> <span class="kn">import</span> <span class="n">reindex</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="OpenSearchError"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a><span class="k">class</span> <span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="OpenSearchError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an OpenSearch error occurs"""</span></div>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -131,20 +128,20 @@
|
||||
<span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_aggregate"</span>
|
||||
|
||||
<span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -175,36 +172,36 @@
|
||||
<span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
|
||||
<span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
|
||||
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
|
||||
<span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
|
||||
@@ -219,21 +216,21 @@
|
||||
<span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
|
||||
<span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
|
||||
@@ -241,8 +238,8 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
|
||||
<span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -264,7 +261,7 @@
|
||||
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
|
||||
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -274,7 +271,7 @@
|
||||
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
|
||||
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -283,7 +280,7 @@
|
||||
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
|
||||
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">result_type</span><span class="p">,</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
@@ -309,8 +306,8 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span> <span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="s2">"smtp_tls"</span>
|
||||
|
||||
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
@@ -321,7 +318,7 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">,</span>
|
||||
@@ -342,17 +339,22 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="set_hosts">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
|
||||
<span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">api_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -364,7 +366,7 @@
|
||||
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
||||
<span class="sd"> username (str): The username to use for authentication</span>
|
||||
<span class="sd"> password (str): The password to use for authentication</span>
|
||||
<span class="sd"> apiKey (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> api_key (str): The Base64 encoded API key to use for authentication</span>
|
||||
<span class="sd"> timeout (float): Timeout in seconds</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
|
||||
@@ -379,12 +381,15 @@
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"verify_certs"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"http_auth"</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="n">password</span>
|
||||
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
|
||||
<span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s2">"api_key"</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="create_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Create OpenSearch indexes</span>
|
||||
|
||||
@@ -407,7 +412,10 @@
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a><span class="k">def</span> <span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="migrate_indexes">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span><span class="n">aggregate_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">forensic_indexes</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Updates index mappings</span>
|
||||
|
||||
@@ -453,7 +461,10 @@
|
||||
<span class="k">pass</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -604,7 +615,10 @@
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -640,8 +654,8 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date_human</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
|
||||
@@ -650,20 +664,35 @@
|
||||
<span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">search_index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
|
||||
<span class="n">arrival_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"arrival_date"</span><span class="p">:</span> <span class="n">arrival_date</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="n">arrival_query</span><span class="p">)</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.from"</span><span class="p">:</span> <span class="n">from_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">from_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the FROM header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">from_</span><span class="p">[</span><span class="s2">"sample.headers.from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span>
|
||||
<span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">from_query</span>
|
||||
<span class="k">if</span> <span class="s2">"to"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.to"</span><span class="p">:</span> <span class="n">to_</span><span class="p">}}</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">Q</span><span class="p">(</span><span class="n">to_query</span><span class="p">)</span>
|
||||
<span class="c1"># We convert the TO header from a string list to a flat string.</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">" <"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">])</span> <span class="o">+</span> <span class="s2">">"</span>
|
||||
|
||||
<span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
|
||||
<span class="n">to_</span><span class="p">[</span><span class="s2">"sample.headers.to"</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"to"</span><span class="p">]</span>
|
||||
<span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
|
||||
<span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&</span> <span class="n">to_query</span>
|
||||
<span class="k">if</span> <span class="s2">"subject"</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
|
||||
<span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">"subject"</span><span class="p">]</span>
|
||||
<span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"match_phrase"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"sample.headers.subject"</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
|
||||
@@ -677,7 +706,9 @@
|
||||
<span class="s2">"A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> "</span>
|
||||
<span class="s2">"with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> "</span>
|
||||
<span class="s2">"already exists in "</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">arrival_date_human</span><span class="p">)</span>
|
||||
<span class="s2">"OpenSearch"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">"parsed_sample"</span><span class="p">]</span>
|
||||
@@ -713,7 +744,7 @@
|
||||
<span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"user_agent"</span><span class="p">],</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"version"</span><span class="p">],</span>
|
||||
<span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_mail_from"</span><span class="p">],</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date</span><span class="p">,</span>
|
||||
<span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
|
||||
<span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"reported_domain"</span><span class="p">],</span>
|
||||
<span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"original_envelope_id"</span><span class="p">],</span>
|
||||
<span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">"authentication_results"</span><span class="p">],</span>
|
||||
@@ -754,7 +785,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch"><a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">index_prefix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -903,6 +937,7 @@
|
||||
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">"OpenSearch error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -912,7 +947,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.splunk — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -86,31 +80,36 @@
|
||||
<div itemprop="articleBody">
|
||||
|
||||
<h1>Source code for parsedmarc.splunk</h1><div class="highlight"><pre>
|
||||
<span></span><span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
|
||||
<span class="kn">import</span> <span class="nn">socket</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span></span><span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlparse</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">socket</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">urllib3</span>
|
||||
<span class="kn">import</span> <span class="nn">requests</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">urllib3</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">__version__</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
|
||||
|
||||
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SplunkError"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a><span class="k">class</span> <span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="SplunkError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when a Splunk API error occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="HECClient"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a><span class="k">class</span> <span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""A client for a Splunk HTTP Events Collector (HEC)"""</span>
|
||||
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
|
||||
<span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
|
||||
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span> <span class="n">url</span><span class="p">,</span> <span class="n">access_token</span><span class="p">,</span> <span class="n">index</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="s2">"parsedmarc"</span><span class="p">,</span> <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span>
|
||||
<span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
@@ -139,11 +138,13 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span><span class="p">)</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="s2">"parsedmarc/</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">),</span>
|
||||
<span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">,</span>
|
||||
<span class="s2">"Authorization"</span><span class="p">:</span> <span class="s2">"Splunk </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -201,7 +202,10 @@
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves forensic DMARC reports to Splunk</span>
|
||||
|
||||
@@ -235,7 +239,10 @@
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
|
||||
|
||||
@@ -268,7 +275,9 @@
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">"code"</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div></div>
|
||||
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">"text"</span><span class="p">])</span></div>
|
||||
</div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -278,7 +287,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
+156
-88
@@ -1,23 +1,20 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
<html class="writer-html5" lang="en">
|
||||
<html class="writer-html5" lang="en" data-content_root="../../">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.18.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
|
||||
<title>parsedmarc.utils — parsedmarc 8.19.1 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
|
||||
|
||||
<!--[if lt IE 9]>
|
||||
<script src="../../_static/js/html5shiv.min.js"></script>
|
||||
<![endif]-->
|
||||
|
||||
<script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
|
||||
<script src="../../_static/jquery.js"></script>
|
||||
<script src="../../_static/underscore.js"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
|
||||
<script src="../../_static/doctools.js"></script>
|
||||
<script src="../../_static/sphinx_highlight.js"></script>
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=4eb1041f"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -34,9 +31,6 @@
|
||||
<a href="../../index.html" class="icon icon-home">
|
||||
parsedmarc
|
||||
</a>
|
||||
<div class="version">
|
||||
8.18.1
|
||||
</div>
|
||||
<div role="search">
|
||||
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||||
@@ -88,60 +82,75 @@
|
||||
<h1>Source code for parsedmarc.utils</h1><div class="highlight"><pre>
|
||||
<span></span><span class="sd">"""Utility functions that might be useful for other projects"""</span>
|
||||
|
||||
<span class="kn">import</span> <span class="nn">logging</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">import</span> <span class="nn">tempfile</span>
|
||||
<span class="kn">import</span> <span class="nn">subprocess</span>
|
||||
<span class="kn">import</span> <span class="nn">shutil</span>
|
||||
<span class="kn">import</span> <span class="nn">mailparser</span>
|
||||
<span class="kn">import</span> <span class="nn">json</span>
|
||||
<span class="kn">import</span> <span class="nn">hashlib</span>
|
||||
<span class="kn">import</span> <span class="nn">base64</span>
|
||||
<span class="kn">import</span> <span class="nn">mailbox</span>
|
||||
<span class="kn">import</span> <span class="nn">re</span>
|
||||
<span class="kn">import</span> <span class="nn">csv</span>
|
||||
<span class="kn">import</span> <span class="nn">io</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">timezone</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">timedelta</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">subprocess</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">csv</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib.resources</span> <span class="k">as</span> <span class="nn">pkg_resources</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
||||
<span class="c1"># Try backported to PY<37 `importlib_resources`</span>
|
||||
<span class="kn">import</span> <span class="nn">importlib_resources</span> <span class="k">as</span> <span class="nn">pkg_resources</span>
|
||||
<span class="c1"># Try backported to PY<3 `importlib_resources`</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">dateutil.parser</span> <span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">parse_date</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.reversename</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.resolver</span>
|
||||
<span class="kn">import</span> <span class="nn">dns.exception</span>
|
||||
<span class="kn">import</span> <span class="nn">geoip2.database</span>
|
||||
<span class="kn">import</span> <span class="nn">geoip2.errors</span>
|
||||
<span class="kn">import</span> <span class="nn">publicsuffixlist</span>
|
||||
<span class="kn">import</span> <span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.dbip</span>
|
||||
<span class="kn">import</span> <span class="nn">parsedmarc.resources.maps</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">parse_date</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.reversename</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.resolver</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.exception</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.database</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.errors</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">publicsuffixlist</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.dbip</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.maps</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
|
||||
|
||||
<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">"\s*\(.*\)\s*"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">"w"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"mailparser"</span><span class="p">)</span>
|
||||
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
<span class="n">psl_overrides_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"psl_overrides.txt"</span><span class="p">))</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_overrides_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
|
||||
<span class="n">psl_overrides</span> <span class="o">=</span> <span class="p">[</span><span class="n">line</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()]</span>
|
||||
<span class="k">while</span> <span class="s2">""</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
|
||||
<span class="n">psl_overrides</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s2">""</span><span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="EmailParserError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a><span class="k">class</span> <span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="EmailParserError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error parsing the email occurs"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="DownloadError"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a><span class="k">class</span> <span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="DownloadError">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Raised when an error occurs when downloading a file"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="decode_base64"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a><span class="k">def</span> <span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="decode_base64">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Decodes a base64 string, with padding being optional</span>
|
||||
|
||||
@@ -159,13 +168,17 @@
|
||||
<span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Gets the base domain name for the given domain</span>
|
||||
|
||||
<span class="sd"> .. note::</span>
|
||||
<span class="sd"> Results are based on a list of public domain suffixes at</span>
|
||||
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat.</span>
|
||||
<span class="sd"> https://publicsuffix.org/list/public_suffix_list.dat and overrides included in</span>
|
||||
<span class="sd"> parsedmarc.resources.maps.psl_overrides.txt</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> domain (str): A domain or subdomain</span>
|
||||
@@ -174,11 +187,18 @@
|
||||
<span class="sd"> str: The base domain of the given domain</span>
|
||||
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span></div>
|
||||
<span class="n">domain</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
|
||||
<span class="n">publicsuffix</span> <span class="o">=</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">domain</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">override</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">override</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">"-"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">publicsuffix</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="query_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a><span class="k">def</span> <span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="query_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">query_dns</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Queries DNS</span>
|
||||
|
||||
@@ -239,7 +259,10 @@
|
||||
<span class="k">return</span> <span class="n">records</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a><span class="k">def</span> <span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_reverse_dns">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_reverse_dns</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Resolves an IP address to a hostname using a reverse DNS query</span>
|
||||
|
||||
@@ -267,7 +290,10 @@
|
||||
<span class="k">return</span> <span class="n">hostname</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a Python ``datetime`` object</span>
|
||||
|
||||
@@ -280,7 +306,10 @@
|
||||
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a><span class="k">def</span> <span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="timestamp_to_human">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a UNIX/DMARC timestamp to a human-readable string</span>
|
||||
|
||||
@@ -293,7 +322,10 @@
|
||||
<span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a Python ``datetime`` object</span>
|
||||
|
||||
@@ -312,7 +344,10 @@
|
||||
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
|
||||
|
||||
@@ -326,7 +361,10 @@
|
||||
<span class="k">return</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_country">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the ISO code for the country associated</span>
|
||||
<span class="sd"> with the given IPv4 or IPv6 address</span>
|
||||
@@ -353,7 +391,7 @@
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">):</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an "</span>
|
||||
@@ -368,14 +406,13 @@
|
||||
<span class="k">break</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">,</span> <span class="s2">"dbip-country-lite.mmdb"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">path</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"dbip-country-lite.mmdb"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IP database is more than a month old"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
|
||||
|
||||
@@ -389,7 +426,10 @@
|
||||
<span class="k">return</span> <span class="n">country</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a><span class="k">def</span> <span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
|
||||
<span class="n">base_domain</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_file</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">local_file_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -413,7 +453,7 @@
|
||||
<span class="sd"> the supplied reverse_dns_base_domain and the type will be None</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
|
||||
<span class="n">reader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span>
|
||||
<span class="n">key</span> <span class="o">=</span> <span class="n">row</span><span class="p">[</span><span class="s2">"base_reverse_dns"</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
|
||||
@@ -433,20 +473,29 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Trying to fetch reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">..."</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">)</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"User-Agent"</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
|
||||
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Failed to fetch reverse DNS map: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Not a valid CSV file"</span><span class="p">)</span>
|
||||
<span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Response body:"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">csv_file</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Loading included reverse DNS map..."</span><span class="p">)</span>
|
||||
<span class="k">with</span> <span class="n">pkg_resources</span><span class="o">.</span><span class="n">path</span><span class="p">(</span>
|
||||
<span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">,</span> <span class="s2">"base_reverse_dns_map.csv"</span>
|
||||
<span class="p">)</span> <span class="k">as</span> <span class="n">path</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"base_reverse_dns_map.csv"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
|
||||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
|
||||
<span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">service</span> <span class="o">=</span> <span class="n">reverse_dns_map</span><span class="p">[</span><span class="n">base_domain</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
|
||||
@@ -455,7 +504,10 @@
|
||||
<span class="k">return</span> <span class="n">service</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a><span class="k">def</span> <span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
|
||||
<div class="viewcode-block" id="get_ip_address_info">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_info</span><span class="p">(</span>
|
||||
<span class="n">ip_address</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">reverse_dns_map_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
@@ -531,7 +583,8 @@
|
||||
<span class="k">return</span> <span class="n">info</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">""</span><span class="p">:</span>
|
||||
<span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@@ -554,7 +607,9 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="get_filename_safe_string"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a><span class="k">def</span> <span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="get_filename_safe_string">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a string to a string that is safe for a filename</span>
|
||||
|
||||
@@ -576,7 +631,10 @@
|
||||
<span class="k">return</span> <span class="n">string</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_mbox"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a><span class="k">def</span> <span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_mbox">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an MBOX mailbox file</span>
|
||||
|
||||
@@ -597,7 +655,10 @@
|
||||
<span class="k">return</span> <span class="n">_is_mbox</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="is_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Checks if the given content is an Outlook msg OLE/MSG file</span>
|
||||
|
||||
@@ -612,7 +673,10 @@
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a><span class="k">def</span> <span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="convert_outlook_msg">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
|
||||
<span class="sd"> standard RFC 822 format</span>
|
||||
@@ -648,7 +712,10 @@
|
||||
<span class="k">return</span> <span class="n">rfc822</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="parse_email"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a><span class="k">def</span> <span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
|
||||
<div class="viewcode-block" id="parse_email">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_email</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> A simplified email parser</span>
|
||||
|
||||
@@ -754,6 +821,7 @@
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"body"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">parsed_email</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</div>
|
||||
@@ -763,7 +831,7 @@
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<p>© Copyright 2018 - 2023, Sean Whalen and contributors.</p>
|
||||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||||
</div>
|
||||
|
||||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||||
|
||||
Reference in New Issue
Block a user