amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-07-03 15:34:19 +00:00
Code Issues Projects Releases Wiki Activity

Update docs

Browse Source
This commit is contained in:
Sean Whalen
2024-02-19 19:23:01 -05:00
parent ac737c395d
commit 51fd81a918
26 changed files with 912 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_modules/index.html
+7 -5
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Overview: module code &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<title>Overview: module code &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="../_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
_modules/parsedmarc.html
+327 -60
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<title>parsedmarc &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="../_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -118,7 +120,7 @@
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.6.4&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.7.0&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;parsedmarc v</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
@@ -130,6 +132,7 @@
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">&quot;</span>
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x1F\x8B</span><span class="s2">&quot;</span>
<span class="n">MAGIC_XML</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x3c\x3f\x78\x6d\x6c\x20</span><span class="s2">&quot;</span>
<span class="n">MAGIC_JSON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\7</span><span class="s2">b&quot;</span>
<span class="n">IP_ADDRESS_CACHE</span> <span class="o">=</span> <span class="n">ExpiringDict</span><span class="p">(</span><span class="n">max_len</span><span class="o">=</span><span class="mi">10000</span><span class="p">,</span> <span class="n">max_age_seconds</span><span class="o">=</span><span class="mi">1800</span><span class="p">)</span>
@@ -142,6 +145,10 @@
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC report is encountered&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="InvalidSMTPTLSReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidSMTPTLSReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidSMTPTLSReport</span><span class="p">(</span><span class="n">ParserError</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raised when an invalid SMTP TLS report is encountered&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="InvalidAggregateReport"><a class="viewcode-back" href="../api.html#parsedmarc.InvalidAggregateReport">[docs]</a><span class="k">class</span> <span class="nc">InvalidAggregateReport</span><span class="p">(</span><span class="n">InvalidDMARCReport</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raised when an invalid DMARC aggregate report is encountered&quot;&quot;&quot;</span></div>
@@ -288,6 +295,179 @@
<span class="k">return</span> <span class="n">new_record</span>
<span class="k">def</span> <span class="nf">_parse_smtp_tls_failure_details</span><span class="p">(</span><span class="n">failure_details</span><span class="p">):</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">new_failure_details</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
<span class="n">result_type</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;result-type&quot;</span><span class="p">],</span>
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;failed-session-count&quot;</span><span class="p">],</span>
<span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;sending-mta-ip&quot;</span><span class="p">],</span>
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;receiving-ip&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;receiving-mx-hostname&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;receiving-mx-hostname&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;receiving-mx-helo&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;receiving-mx-helo&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;additional-info-uri&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;additional_info_uri&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;additional-info-uri&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;failure-reason-code&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">new_failure_details</span><span class="p">[</span><span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;failure-reason-code&quot;</span><span class="p">]</span>
<span class="k">return</span> <span class="n">new_failure_details</span>
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Missing required failure details field:&quot;</span>
<span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">):</span>
<span class="n">policy_types</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;tlsa&quot;</span><span class="p">,</span> <span class="s2">&quot;sts&quot;</span><span class="p">,</span> <span class="s2">&quot;no-policy-found&quot;</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;policy-domain&quot;</span><span class="p">]</span>
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;policy-type&quot;</span><span class="p">]</span>
<span class="n">failure_details</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">if</span> <span class="n">policy_type</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">policy_types</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Invalid policy type &quot;</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">policy_type</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">new_policy</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span><span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;policy-string&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;policy-string&quot;</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;policy-string&quot;</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">new_policy</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span>
<span class="s2">&quot;policy-string&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;mx-host-pattern&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;mx-host-pattern&quot;</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span><span class="s2">&quot;mx-host-pattern&quot;</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">new_policy</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy&quot;</span><span class="p">][</span>
<span class="s2">&quot;mx-host-pattern&quot;</span><span class="p">]</span>
<span class="n">new_policy</span><span class="p">[</span><span class="s2">&quot;successful_session_count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;summary&quot;</span><span class="p">][</span>
<span class="s2">&quot;total-successful-session-count&quot;</span><span class="p">]</span>
<span class="n">new_policy</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;summary&quot;</span><span class="p">][</span>
<span class="s2">&quot;total-failure-session-count&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;failure-details&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="k">for</span> <span class="n">details</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failure-details&quot;</span><span class="p">]:</span>
<span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_smtp_tls_failure_details</span><span class="p">(</span>
<span class="n">details</span><span class="p">))</span>
<span class="n">new_policy</span><span class="p">[</span><span class="s2">&quot;failure_details&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span>
<span class="k">return</span> <span class="n">new_policy</span>
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Missing required policy field: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<div class="viewcode-block" id="parse_smtp_tls_report_json"><a class="viewcode-back" href="../api.html#parsedmarc.parse_smtp_tls_report_json">[docs]</a><span class="k">def</span> <span class="nf">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">report</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Parses and validates an SMTP TLS report&quot;&quot;&quot;</span>
<span class="n">required_fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;organization-name&quot;</span><span class="p">,</span> <span class="s2">&quot;date-range&quot;</span><span class="p">,</span>
<span class="s2">&quot;contact-info&quot;</span><span class="p">,</span> <span class="s2">&quot;report-id&quot;</span><span class="p">,</span>
<span class="s2">&quot;policies&quot;</span><span class="p">]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">policies</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">report</span><span class="p">)</span>
<span class="k">for</span> <span class="n">required_field</span> <span class="ow">in</span> <span class="n">required_fields</span><span class="p">:</span>
<span class="k">if</span> <span class="n">required_field</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">report</span><span class="p">:</span>
<span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Missing required field: </span><span class="si">{</span><span class="n">required_field</span><span class="si">}</span><span class="s2">]&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">],</span> <span class="nb">list</span><span class="p">):</span>
<span class="n">policies_type</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">])</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;policies must be a list, &quot;</span>
<span class="sa">f</span><span class="s2">&quot;not </span><span class="si">{</span><span class="n">policies_type</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">]:</span>
<span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_parse_smtp_tls_report_policy</span><span class="p">(</span><span class="n">policy</span><span class="p">))</span>
<span class="n">new_report</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
<span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization-name&quot;</span><span class="p">],</span>
<span class="n">begin_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date-range&quot;</span><span class="p">][</span><span class="s2">&quot;start-datetime&quot;</span><span class="p">],</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date-range&quot;</span><span class="p">][</span><span class="s2">&quot;end-datetime&quot;</span><span class="p">],</span>
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report-id&quot;</span><span class="p">],</span>
<span class="n">policies</span><span class="o">=</span><span class="n">policies</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">new_report</span>
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Missing required field: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span></div>
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv_rows"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Converts one oor more parsed SMTP TLS reports into a list of single</span>
<span class="sd"> layer OrderedDict objects suitable for use in a CSV&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="ow">is</span> <span class="n">OrderedDict</span><span class="p">:</span>
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
<span class="n">rows</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
<span class="n">common_fields</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">(</span>
<span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">],</span>
<span class="n">begin_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span>
<span class="n">end_date</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span>
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="n">record</span> <span class="o">=</span> <span class="n">common_fields</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">]:</span>
<span class="k">if</span> <span class="s2">&quot;policy_strings&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;|&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">])</span>
<span class="k">if</span> <span class="s2">&quot;mx_host_patterns&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="n">record</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;|&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
<span class="n">policy</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">])</span>
<span class="n">successful_record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">successful_record</span><span class="p">[</span><span class="s2">&quot;successful_session_count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span>
<span class="s2">&quot;successful_session_count&quot;</span><span class="p">]</span>
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">successful_record</span><span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;failure_details&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="k">for</span> <span class="n">failure_details</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failure_details&quot;</span><span class="p">]:</span>
<span class="n">failure_record</span> <span class="o">=</span> <span class="n">record</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
<span class="n">failure_record</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
<span class="n">rows</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">failure_record</span><span class="p">)</span>
<span class="k">return</span> <span class="n">rows</span></div>
<div class="viewcode-block" id="parsed_smtp_tls_reports_to_csv"><a class="viewcode-back" href="../api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">[docs]</a><span class="k">def</span> <span class="nf">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">reports</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts one or more parsed SMTP TLS reports to flat CSV format, including</span>
<span class="sd"> headers</span>
<span class="sd"> Args:</span>
<span class="sd"> reports: A parsed aggregate report or list of parsed aggregate reports</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: Parsed aggregate report data in flat CSV format, including headers</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">fields</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">,</span> <span class="s2">&quot;begin_date&quot;</span><span class="p">,</span> <span class="s2">&quot;end_date&quot;</span><span class="p">,</span> <span class="s2">&quot;report_id&quot;</span><span class="p">,</span>
<span class="s2">&quot;successful_session_count&quot;</span><span class="p">,</span> <span class="s2">&quot;failed_session_count&quot;</span><span class="p">,</span>
<span class="s2">&quot;policy_domain&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_type&quot;</span><span class="p">,</span> <span class="s2">&quot;policy_strings&quot;</span><span class="p">,</span>
<span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">,</span> <span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_ip&quot;</span><span class="p">,</span>
<span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">,</span> <span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">,</span>
<span class="s2">&quot;additional_info_uri&quot;</span><span class="p">,</span> <span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span>
<span class="n">csv_file_object</span> <span class="o">=</span> <span class="n">StringIO</span><span class="p">(</span><span class="n">newline</span><span class="o">=</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">writer</span> <span class="o">=</span> <span class="n">DictWriter</span><span class="p">(</span><span class="n">csv_file_object</span><span class="p">,</span> <span class="n">fields</span><span class="p">)</span>
<span class="n">writer</span><span class="o">.</span><span class="n">writeheader</span><span class="p">()</span>
<span class="n">rows</span> <span class="o">=</span> <span class="n">parsed_smtp_tls_reports_to_csv_rows</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span>
<span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">rows</span><span class="p">:</span>
<span class="n">writer</span><span class="o">.</span><span class="n">writerow</span><span class="p">(</span><span class="n">row</span><span class="p">)</span>
<span class="n">csv_file_object</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="k">return</span> <span class="n">csv_file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">()</span></div>
<div class="viewcode-block" id="parse_aggregate_report_xml"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">2.0</span><span class="p">,</span>
<span class="n">parallel</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">keep_alive</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -308,6 +488,8 @@
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">errors</span> <span class="o">=</span> <span class="p">[]</span>
<span class="c1"># Parse XML and recover from errors</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">&quot;feedback&quot;</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
@@ -452,21 +634,27 @@
<span class="s2">&quot;Unexpected error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<div class="viewcode-block" id="extract_xml"><a class="viewcode-back" href="../api.html#parsedmarc.extract_xml">[docs]</a><span class="k">def</span> <span class="nf">extract_xml</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
<div class="viewcode-block" id="extract_report"><a class="viewcode-back" href="../api.html#parsedmarc.extract_report">[docs]</a><span class="k">def</span> <span class="nf">extract_report</span><span class="p">(</span><span class="n">input_</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Extracts xml from a zip or gzip file at the given path, file-like object,</span>
<span class="sd"> Extracts text from a zip or gzip file at the given path, file-like object,</span>
<span class="sd"> or bytes.</span>
<span class="sd"> Args:</span>
<span class="sd"> input_: A path to a file, a file like object, or bytes</span>
<span class="sd"> Returns:</span>
<span class="sd"> str: The extracted XML</span>
<span class="sd"> str: The extracted text</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">()</span>
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">str</span><span class="p">:</span>
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
<span class="k">except</span> <span class="n">binascii</span><span class="o">.</span><span class="n">Error</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">if</span> <span class="n">file_object</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">file_object</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">input_</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span>
<span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">bytes</span><span class="p">:</span>
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
@@ -476,30 +664,31 @@
<span class="n">file_object</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="k">if</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">):</span>
<span class="n">_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span>
<span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">(),</span>
<span class="n">zlib</span><span class="o">.</span><span class="n">MAX_WBITS</span> <span class="o">|</span> <span class="mi">16</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">):</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span>
<span class="n">file_object</span><span class="o">.</span><span class="n">getvalue</span><span class="p">(),</span>
<span class="n">zlib</span><span class="o">.</span><span class="n">MAX_WBITS</span> <span class="o">|</span> <span class="mi">16</span><span class="p">)</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="k">elif</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">)</span> <span class="ow">or</span> <span class="n">header</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_JSON</span><span class="p">):</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">file_object</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s1">&#39;ignore&#39;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">&quot;Not a valid zip, gzip, or xml file&quot;</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">&quot;Not a valid zip, gzip, json, or xml file&quot;</span><span class="p">)</span>
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">&quot;File was not found&quot;</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">&quot;File was not found&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">UnicodeDecodeError</span><span class="p">:</span>
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="s2">&quot;File objects must be opened in binary &quot;</span>
<span class="s2">&quot;(rb) mode&quot;</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">&quot;File objects must be opened in binary (rb) mode&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">file_object</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span>
<span class="s2">&quot;Invalid archive file: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
<span class="k">return</span> <span class="n">xml</span></div>
<span class="k">return</span> <span class="n">report</span></div>
<div class="viewcode-block" id="parse_aggregate_report_file"><a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_file">[docs]</a><span class="k">def</span> <span class="nf">parse_aggregate_report_file</span><span class="p">(</span><span class="n">_input</span><span class="p">,</span> <span class="n">offline</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ip_db_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
@@ -523,7 +712,11 @@
<span class="sd"> Returns:</span>
<span class="sd"> OrderedDict: The parsed DMARC aggregate report</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">extract_xml</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">_input</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
<span class="k">return</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span><span class="n">xml</span><span class="p">,</span>
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
@@ -591,7 +784,7 @@
<span class="n">row</span><span class="p">[</span><span class="s2">&quot;dmarc_aligned&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;dmarc&quot;</span><span class="p">]</span>
<span class="n">row</span><span class="p">[</span><span class="s2">&quot;disposition&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;disposition&quot;</span><span class="p">]</span>
<span class="n">policy_override_reasons</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
<span class="k">lambda</span> <span class="n">r_</span><span class="p">:</span> <span class="n">r_</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">],</span>
<span class="k">lambda</span> <span class="n">r_</span><span class="p">:</span> <span class="n">r_</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span> <span class="ow">or</span> <span class="s2">&quot;none&quot;</span><span class="p">,</span>
<span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">]</span>
<span class="p">[</span><span class="s2">&quot;policy_override_reasons&quot;</span><span class="p">]))</span>
<span class="n">policy_override_comments</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span>
@@ -903,12 +1096,14 @@
<span class="n">msg</span> <span class="o">=</span> <span class="n">email</span><span class="o">.</span><span class="n">message_from_string</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">sample</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;From&quot;</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Parsing mail from </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">&quot;From&quot;</span><span class="p">]))</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Parsing mail from </span><span class="si">{0}</span><span class="s2"> on </span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">msg_headers</span><span class="p">[</span><span class="s2">&quot;From&quot;</span><span class="p">],</span>
<span class="n">date</span><span class="p">))</span>
<span class="k">if</span> <span class="s2">&quot;Subject&quot;</span> <span class="ow">in</span> <span class="n">msg_headers</span><span class="p">:</span>
<span class="n">subject</span> <span class="o">=</span> <span class="n">msg_headers</span><span class="p">[</span><span class="s2">&quot;Subject&quot;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">msg</span><span class="o">.</span><span class="n">walk</span><span class="p">():</span>
@@ -934,34 +1129,57 @@
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;application/tlsrpt+json&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;{&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">))</span>
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)])</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;application/tlsrpt+gzip&quot;</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)])</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/plain&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;A message claiming to be from you has failed&quot;</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;detected.&quot;</span><span class="p">)</span>
<span class="n">field_matches</span> <span class="o">=</span> <span class="n">text_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">fields</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">field_matches</span><span class="p">:</span>
<span class="n">field_name</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="n">fields</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="s2">&quot;Arrival-Date: </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span> \
<span class="s2">&quot;Source-IP: </span><span class="si">{}</span><span class="s2">&quot;</span> \
<span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s2">&quot;received-date&quot;</span><span class="p">],</span>
<span class="n">fields</span><span class="p">[</span><span class="s2">&quot;sender-ip-address&quot;</span><span class="p">])</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;detected.&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">field_matches</span> <span class="o">=</span> <span class="n">text_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">fields</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">field_matches</span><span class="p">:</span>
<span class="n">field_name</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="n">fields</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="s2">&quot;Arrival-Date: </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span> \
<span class="s2">&quot;Source-IP: </span><span class="si">{}</span><span class="s2">&quot;</span> \
<span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s2">&quot;received-date&quot;</span><span class="p">],</span>
<span class="n">fields</span><span class="p">[</span><span class="s2">&quot;sender-ip-address&quot;</span><span class="p">])</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Unable to parse message with &#39;</span> \
<span class="s1">&#39;subject &quot;</span><span class="si">{0}</span><span class="s1">&quot;: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">sample</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;=</span><span class="se">\r\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)</span> <span class="ow">or</span> \
<span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">)</span> <span class="ow">or</span> \
<span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_XML</span><span class="p">):</span>
<span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">MAGIC_GZIP</span><span class="p">):</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">extract_report</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="n">ns</span> <span class="o">=</span> <span class="n">nameservers</span>
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_file</span><span class="p">(</span>
<span class="k">if</span> <span class="n">payload</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;{&quot;</span><span class="p">):</span>
<span class="n">smtp_tls_report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">smtp_tls_report</span><span class="p">)])</span>
<span class="k">return</span> <span class="n">result</span>
<span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">parse_aggregate_report_xml</span><span class="p">(</span>
<span class="n">payload</span><span class="p">,</span>
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">,</span>
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">)</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;aggregate&quot;</span><span class="p">),</span>
@@ -975,12 +1193,12 @@
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Message with subject &quot;</span><span class="si">{0}</span><span class="s1">&quot; &#39;</span> \
<span class="s1">&#39;is not a valid &#39;</span> \
<span class="s1">&#39;aggregate DMARC report: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Unable to parse message with &#39;</span> \
<span class="s1">&#39;subject &quot;</span><span class="si">{0}</span><span class="s1">&quot;: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">if</span> <span class="n">feedback_report</span> <span class="ow">and</span> <span class="n">sample</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
@@ -1007,7 +1225,7 @@
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Message with subject &quot;</span><span class="si">{0}</span><span class="s1">&quot; is &#39;</span> \
<span class="s1">&#39;not a valid DMARC report&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
<span class="s1">&#39;not a valid report&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span></div>
@@ -1054,18 +1272,22 @@
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">report</span><span class="p">)])</span>
<span class="k">except</span> <span class="n">InvalidAggregateReport</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span><span class="n">content</span><span class="p">,</span>
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">sa</span><span class="p">,</span>
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">,</span>
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">)</span>
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">&quot;Not a valid aggregate or forensic &quot;</span>
<span class="s2">&quot;report&quot;</span><span class="p">)</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">parse_smtp_tls_report_json</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;report_type&quot;</span><span class="p">,</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;report&quot;</span><span class="p">,</span> <span class="n">report</span><span class="p">)])</span>
<span class="k">except</span> <span class="n">InvalidSMTPTLSReport</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">sa</span> <span class="o">=</span> <span class="n">strip_attachment_payloads</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">parse_report_email</span><span class="p">(</span><span class="n">content</span><span class="p">,</span>
<span class="n">ip_db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">,</span>
<span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">sa</span><span class="p">,</span>
<span class="n">parallel</span><span class="o">=</span><span class="n">parallel</span><span class="p">,</span>
<span class="n">keep_alive</span><span class="o">=</span><span class="n">keep_alive</span><span class="p">)</span>
<span class="k">except</span> <span class="n">InvalidDMARCReport</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">&quot;Not a valid report&quot;</span><span class="p">)</span>
<span class="k">return</span> <span class="n">results</span></div>
@@ -1094,6 +1316,7 @@
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">input_</span><span class="p">)</span>
<span class="n">message_keys</span> <span class="o">=</span> <span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
@@ -1119,12 +1342,15 @@
<span class="n">aggregate_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report&quot;</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report_type&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;forensic&quot;</span><span class="p">:</span>
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report&quot;</span><span class="p">])</span>
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report_type&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">:</span>
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report&quot;</span><span class="p">])</span>
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">except</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">NoSuchMailboxError</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="s2">&quot;Mailbox </span><span class="si">{0}</span><span class="s2"> does not exist&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">input_</span><span class="p">))</span>
<span class="k">return</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;aggregate_reports&quot;</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">)])</span></div>
<span class="p">(</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;smtp_tls_reports&quot;</span><span class="p">,</span> <span class="n">smtp_tls_reports</span><span class="p">)])</span></div>
<div class="viewcode-block" id="get_dmarc_reports_from_mailbox"><a class="viewcode-back" href="../api.html#parsedmarc.get_dmarc_reports_from_mailbox">[docs]</a><span class="k">def</span> <span class="nf">get_dmarc_reports_from_mailbox</span><span class="p">(</span><span class="n">connection</span><span class="p">:</span> <span class="n">MailboxConnection</span><span class="p">,</span>
@@ -1172,20 +1398,25 @@
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">aggregate_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">forensic_report_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">smtp_tls_msg_uids</span> <span class="o">=</span> <span class="p">[]</span>
<span class="n">aggregate_reports_folder</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/Aggregate&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="n">forensic_reports_folder</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/Forensic&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="n">smtp_tls_reports_folder</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/SMTP-TLS&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="n">invalid_reports_folder</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">/Invalid&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="k">if</span> <span class="n">results</span><span class="p">:</span>
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;aggregate_reports&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;smtp_tls_reports&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span> <span class="ow">and</span> <span class="n">create_folders</span><span class="p">:</span>
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">archive_folder</span><span class="p">)</span>
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">aggregate_reports_folder</span><span class="p">)</span>
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">forensic_reports_folder</span><span class="p">)</span>
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
<span class="n">connection</span><span class="o">.</span><span class="n">create_folder</span><span class="p">(</span><span class="n">invalid_reports_folder</span><span class="p">)</span>
<span class="n">messages</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">fetch_messages</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">)</span>
@@ -1221,7 +1452,10 @@
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report_type&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;forensic&quot;</span><span class="p">:</span>
<span class="n">forensic_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report&quot;</span><span class="p">])</span>
<span class="n">forensic_report_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
<span class="k">except</span> <span class="n">InvalidDMARCReport</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="k">elif</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report_type&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">:</span>
<span class="n">smtp_tls_reports</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;report&quot;</span><span class="p">])</span>
<span class="n">smtp_tls_msg_uids</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">)</span>
<span class="k">except</span> <span class="n">ParserError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
@@ -1237,7 +1471,8 @@
<span class="k">if</span> <span class="ow">not</span> <span class="n">test</span><span class="p">:</span>
<span class="k">if</span> <span class="n">delete</span><span class="p">:</span>
<span class="n">processed_messages</span> <span class="o">=</span> <span class="n">aggregate_report_msg_uids</span> <span class="o">+</span> \
<span class="n">forensic_report_msg_uids</span>
<span class="n">forensic_report_msg_uids</span> <span class="o">+</span> \
<span class="n">smtp_tls_msg_uids</span>
<span class="n">number_of_processed_msgs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">processed_messages</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_processed_msgs</span><span class="p">):</span>
@@ -1292,8 +1527,29 @@
<span class="n">e</span> <span class="o">=</span> <span class="s2">&quot;Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Mailbox error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">message</span> <span class="o">=</span> <span class="s2">&quot;Moving SMTP TLS report messages from&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
<span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> to </span><span class="si">{2}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">message</span><span class="p">,</span>
<span class="n">reports_folder</span><span class="p">,</span>
<span class="n">smtp_tls_reports_folder</span><span class="p">))</span>
<span class="n">number_of_smtp_tls_uids</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">smtp_tls_msg_uids</span><span class="p">)</span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">number_of_smtp_tls_uids</span><span class="p">):</span>
<span class="n">msg_uid</span> <span class="o">=</span> <span class="n">smtp_tls_msg_uids</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
<span class="n">message</span> <span class="o">=</span> <span class="s2">&quot;Moving message&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">{0}</span><span class="s2"> </span><span class="si">{1}</span><span class="s2"> of </span><span class="si">{2}</span><span class="s2">: UID </span><span class="si">{3}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">message</span><span class="p">,</span>
<span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">smtp_tls_msg_uids</span><span class="p">,</span> <span class="n">msg_uid</span><span class="p">))</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">connection</span><span class="o">.</span><span class="n">move_message</span><span class="p">(</span><span class="n">msg_uid</span><span class="p">,</span>
<span class="n">smtp_tls_reports_folder</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">e</span> <span class="o">=</span> <span class="s2">&quot;Error moving message UID </span><span class="si">{0}</span><span class="s2">: </span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="n">msg_uid</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Mailbox error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">OrderedDict</span><span class="p">([(</span><span class="s2">&quot;aggregate_reports&quot;</span><span class="p">,</span> <span class="n">aggregate_reports</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">)])</span>
<span class="p">(</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">,</span> <span class="n">forensic_reports</span><span class="p">),</span>
<span class="p">(</span><span class="s2">&quot;smtp_tls_reports&quot;</span><span class="p">,</span> <span class="n">smtp_tls_reports</span><span class="p">)])</span>
<span class="n">total_messages</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">connection</span><span class="o">.</span><span class="n">fetch_messages</span><span class="p">(</span><span class="n">reports_folder</span><span class="p">))</span>
@@ -1405,8 +1661,10 @@
<div class="viewcode-block" id="save_output"><a class="viewcode-back" href="../api.html#parsedmarc.save_output">[docs]</a><span class="k">def</span> <span class="nf">save_output</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">output_directory</span><span class="o">=</span><span class="s2">&quot;output&quot;</span><span class="p">,</span>
<span class="n">aggregate_json_filename</span><span class="o">=</span><span class="s2">&quot;aggregate.json&quot;</span><span class="p">,</span>
<span class="n">forensic_json_filename</span><span class="o">=</span><span class="s2">&quot;forensic.json&quot;</span><span class="p">,</span>
<span class="n">smtp_tls_json_filename</span><span class="o">=</span><span class="s2">&quot;smtp_tls.json&quot;</span><span class="p">,</span>
<span class="n">aggregate_csv_filename</span><span class="o">=</span><span class="s2">&quot;aggregate.csv&quot;</span><span class="p">,</span>
<span class="n">forensic_csv_filename</span><span class="o">=</span><span class="s2">&quot;forensic.csv&quot;</span><span class="p">):</span>
<span class="n">forensic_csv_filename</span><span class="o">=</span><span class="s2">&quot;forensic.csv&quot;</span><span class="p">,</span>
<span class="n">smtp_tls_csv_filename</span><span class="o">=</span><span class="s2">&quot;smtp_tls.csv&quot;</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Save report data in the given directory</span>
@@ -1415,12 +1673,15 @@
<span class="sd"> output_directory (str): The path to the directory to save in</span>
<span class="sd"> aggregate_json_filename (str): Filename for the aggregate JSON file</span>
<span class="sd"> forensic_json_filename (str): Filename for the forensic JSON file</span>
<span class="sd"> smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file</span>
<span class="sd"> aggregate_csv_filename (str): Filename for the aggregate CSV file</span>
<span class="sd"> forensic_csv_filename (str): Filename for the forensic CSV file</span>
<span class="sd"> smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">aggregate_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;aggregate_reports&quot;</span><span class="p">]</span>
<span class="n">forensic_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;forensic_reports&quot;</span><span class="p">]</span>
<span class="n">smtp_tls_reports</span> <span class="o">=</span> <span class="n">results</span><span class="p">[</span><span class="s2">&quot;smtp_tls_reports&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isdir</span><span class="p">(</span><span class="n">output_directory</span><span class="p">):</span>
@@ -1440,6 +1701,12 @@
<span class="n">append_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">forensic_csv_filename</span><span class="p">),</span>
<span class="n">parsed_forensic_reports_to_csv</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">))</span>
<span class="n">append_json</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">smtp_tls_json_filename</span><span class="p">),</span>
<span class="n">smtp_tls_reports</span><span class="p">)</span>
<span class="n">append_csv</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="n">smtp_tls_csv_filename</span><span class="p">),</span>
<span class="n">parsed_smtp_tls_reports_to_csv</span><span class="p">(</span><span class="n">smtp_tls_reports</span><span class="p">))</span>
<span class="n">samples_directory</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">output_directory</span><span class="p">,</span> <span class="s2">&quot;samples&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">):</span>
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">samples_directory</span><span class="p">)</span>
_modules/parsedmarc/elastic.html
+201 -6
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc.elastic &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<title>parsedmarc.elastic &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
@@ -249,12 +251,78 @@
<span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
<span class="k">class</span> <span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
<span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="k">class</span> <span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
<span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">successful_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
<span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">add_failure_details</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">result_type</span><span class="p">,</span> <span class="n">ip_address</span><span class="p">,</span>
<span class="n">receiving_ip</span><span class="p">,</span>
<span class="n">receiving_mx_helo</span><span class="p">,</span>
<span class="n">failed_session_count</span><span class="p">,</span>
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">additional_information_uri</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">failure_reason_code</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
<span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
<span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
<span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
<span class="p">)</span>
<span class="k">class</span> <span class="nc">_SMTPTLSFailureReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
<span class="k">class</span> <span class="nc">Index</span><span class="p">:</span>
<span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
<span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
<span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
<span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
<span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">add_policy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">policy_type</span><span class="p">,</span> <span class="n">policy_domain</span><span class="p">,</span>
<span class="n">successful_session_count</span><span class="p">,</span>
<span class="n">failed_session_count</span><span class="p">,</span>
<span class="n">policy_string</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">failure_details</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">)</span>
<div class="viewcode-block" id="AlreadySaved"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a><span class="k">class</span> <span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;Raised when a report to be saved matches an existing report&quot;&quot;&quot;</span></div>
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
<span class="n">username</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">password</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">apiKey</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mf">60.0</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Sets the Elasticsearch hosts to use</span>
@@ -264,6 +332,7 @@
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
<span class="sd"> username (str): The username to use for authentication</span>
<span class="sd"> password (str): The password to use for authentication</span>
<span class="sd"> apiKey (str): The Base64 encoded API key to use for authentication</span>
<span class="sd"> timeout (float): Timeout in seconds</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
@@ -281,6 +350,8 @@
<span class="n">conn_params</span><span class="p">[</span><span class="s1">&#39;verify_certs&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">if</span> <span class="n">username</span><span class="p">:</span>
<span class="n">conn_params</span><span class="p">[</span><span class="s1">&#39;http_auth&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">username</span><span class="o">+</span><span class="s2">&quot;:&quot;</span><span class="o">+</span><span class="n">password</span><span class="p">)</span>
<span class="k">if</span> <span class="n">apiKey</span><span class="p">:</span>
<span class="n">conn_params</span><span class="p">[</span><span class="s1">&#39;api_key&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">apiKey</span>
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
@@ -635,6 +706,130 @@
<span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
<span class="s2">&quot;Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch"><a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span><span class="n">report</span><span class="p">,</span>
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves a parsed SMTP TLS report to elasticSearch</span>
<span class="sd"> Args:</span>
<span class="sd"> report (OrderedDict): A parsed SMTP TLS report</span>
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
<span class="sd"> number_of_shards (int): The number of shards to use in the index</span>
<span class="sd"> number_of_replicas (int): The number of replicas to use in the index</span>
<span class="sd"> Raises:</span>
<span class="sd"> AlreadySaved</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving aggregate report to Elasticsearch&quot;</span><span class="p">)</span>
<span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
<span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span>
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span>
<span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
<span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
<span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
<span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
<span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
<span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">&quot;smtp_tls_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">))</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="s2">&quot;smtp_tls&quot;</span><span class="p">)</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">report_id_query</span>
<span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> <span class="n">end_date_query</span>
<span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">&quot;Elasticsearch&#39;s search for existing report </span><span class="se">\</span>
<span class="s2"> error: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from &quot;</span>
<span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of &quot;</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to &quot;</span>
<span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already &quot;</span>
<span class="s2">&quot;exists in Elasticsearch&quot;</span><span class="p">)</span>
<span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
<span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
<span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
<span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span>
<span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span><span class="p">)</span>
<span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureReportDoc</span><span class="p">(</span>
<span class="n">organization_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">],</span>
<span class="n">date_range</span><span class="o">=</span><span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date_begin&quot;</span><span class="p">],</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;date_end&quot;</span><span class="p">]],</span>
<span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date_begin&quot;</span><span class="p">],</span>
<span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;date_end&quot;</span><span class="p">],</span>
<span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;contact_info&quot;</span><span class="p">],</span>
<span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
<span class="p">)</span>
<span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s1">&#39;policies&#39;</span><span class="p">]:</span>
<span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;policy_strings&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;mx_host_patterns&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">]</span>
<span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_domain&quot;</span><span class="p">],</span>
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_type&quot;</span><span class="p">],</span>
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span>
<span class="p">)</span>
<span class="k">if</span> <span class="s2">&quot;failure_details&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
<span class="n">failure_details</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failure_details&quot;</span><span class="p">]</span>
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;receiving_mx_hostname&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;additional_information_uri&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span>
<span class="s2">&quot;additional_information_uri&quot;</span><span class="p">]</span>
<span class="k">if</span> <span class="s2">&quot;failure_reason_code&quot;</span> <span class="ow">in</span> <span class="n">failure_details</span><span class="p">:</span>
<span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span>
<span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
<span class="n">result_type</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;result_type&quot;</span><span class="p">],</span>
<span class="n">ip_address</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
<span class="n">receiving_ip</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_ip&quot;</span><span class="p">],</span>
<span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">],</span>
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_details</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">],</span>
<span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
<span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
<span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span>
<span class="p">)</span>
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
<span class="s2">&quot;Elasticsearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
</pre></div>
</div>
_modules/parsedmarc/splunk.html
+47 -8
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc.splunk &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<title>parsedmarc.splunk &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
@@ -92,7 +94,7 @@
<span class="kn">from</span> <span class="nn">parsedmarc</span> <span class="kn">import</span> <span class="n">__version__</span>
<span class="kn">from</span> <span class="nn">parsedmarc.log</span> <span class="kn">import</span> <span class="n">logger</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_timestamp</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
@@ -189,7 +191,7 @@
<span class="s2">&quot;spf&quot;</span><span class="p">]</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:aggregate&quot;</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_timestamp</span><span class="p">(</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
<span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
@@ -225,12 +227,49 @@
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:forensic&quot;</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_timestamp</span><span class="p">(</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
<span class="n">report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">])</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Skipping certificate verification for Splunk HEC&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span>
<span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
<span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div>
<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk"><a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a> <span class="k">def</span> <span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Saves aggregate DMARC reports to Splunk</span>
<span class="sd"> Args:</span>
<span class="sd"> reports: A list of SMTP TLS report dictionaries</span>
<span class="sd"> to save in Splunk</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving SMTP TLS reports to Splunk&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
<span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="k">return</span>
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;smtp:tls&quot;</span>
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
<span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
<span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Skipping certificate verification for Splunk HEC&quot;</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
_modules/parsedmarc/utils.html
+8 -6
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc.utils &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<title>parsedmarc.utils &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="../../_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
@@ -303,7 +305,7 @@
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
<div class="viewcode-block" id="human_timestamp_to_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp"><a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a><span class="k">def</span> <span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">):</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
_sources/elasticsearch.md.txt
+1 -1
View File
@@ -227,7 +227,7 @@ Kibana index patterns with versions that match the upgraded indexes:
Starting in version 5.0.0, `parsedmarc` stores data in a separate
index for each day to make it easy to comply with records
retention regulations such as GDPR. For fore information,
retention regulations such as GDPR. For more information,
check out the Elastic guide to [managing time-based indexes efficiently](https://www.elastic.co/blog/managing-time-based-indices-efficiently).
[elasticsearch]: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html
_sources/output.md.txt
+43
View File
@@ -187,3 +187,46 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized
feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
```
### JSON SMTP TLS report
```json
[
{
"organization_name": "Example Inc.",
"begin_date": "2024-01-09T00:00:00Z",
"end_date": "2024-01-09T23:59:59Z",
"report_id": "2024-01-09T00:00:00Z_example.com",
"policies": [
{
"policy_domain": "example.com",
"policy_type": "sts",
"policy_strings": [
"version: STSv1",
"mode: testing",
"mx: example.com",
"max_age: 86400"
],
"successful_session_count": 0,
"failed_session_count": 3,
"failure_details": [
{
"result_type": "validation-failure",
"failed_session_count": 2,
"sending_mta_ip": "209.85.222.201",
"receiving_ip": "173.212.201.41",
"receiving_mx_hostname": "example.com"
},
{
"result_type": "validation-failure",
"failed_session_count": 1,
"sending_mta_ip": "209.85.208.176",
"receiving_ip": "173.212.201.41",
"receiving_mx_hostname": "example.com"
}
]
}
]
}
]
```
_sources/usage.md.txt
+9 -3
View File
@@ -137,9 +137,9 @@ The full set of configuration options are:
- `archive_folder` - str: The mailbox folder (or label for
Gmail) to sort processed emails into (Default: `Archive`)
- `watch` - bool: Use the IMAP `IDLE` command to process
- messages as they arrive or poll MS Graph for new messages
messages as they arrive or poll MS Graph for new messages
- `delete` - bool: Delete messages after processing them,
- instead of archiving them
instead of archiving them
- `test` - bool: Do not move or delete messages
- `batch_size` - int: Number of messages to read and process
before saving. Default `10`. Use `0` for no limit.
@@ -225,9 +225,12 @@ The full set of configuration options are:
Special characters in the username or password must be
[URL encoded].
:::
- `user` - str: Basic auth username
- `password` - str: Basic auth password
- `apiKey` - str: API key
- `ssl` - bool: Use an encrypted SSL/TLS connection
(Default: `True`)
- `timeout` - float: Timeout in seconds (Default: 60)
- `cert_path` - str: Path to a trusted certificates
- `index_suffix` - str: A suffix to apply to the index names
- `monthly_indexes` - bool: Use monthly indexes instead of daily indexes
@@ -292,6 +295,8 @@ The full set of configuration options are:
(Default: `https://www.googleapis.com/auth/gmail.modify`)
- `oauth2_port` - int: The TCP port for the local server to
listen on for the OAuth2 response (Default: `8080`)
- `paginate_messages` - bool: When `True`, fetch all applicable Gmail messages.
When `False`, only fetch up to 100 new messages per run (Default: `True`)
- `log_analytics`
- `client_id` - str: The app registration's client ID
- `client_secret` - str: The app registration's client secret
@@ -300,6 +305,7 @@ The full set of configuration options are:
- `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR)
- `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR
- `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR
- `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR
:::{note}
Information regarding the setup of the Data Collection Rule can be found [here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal).
_static/documentation_options.js
+1 -1
View File
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '8.6.4',
VERSION: '8.7.0',
LANGUAGE: 'en',
COLLAPSE_INDEX: false,
BUILDER: 'html',
api.html
+95 -15
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>API reference &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>API reference &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -35,7 +37,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -61,9 +63,10 @@
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.InvalidAggregateReport"><code class="docutils literal notranslate"><span class="pre">InvalidAggregateReport</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.InvalidDMARCReport"><code class="docutils literal notranslate"><span class="pre">InvalidDMARCReport</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.InvalidForensicReport"><code class="docutils literal notranslate"><span class="pre">InvalidForensicReport</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.InvalidSMTPTLSReport"><code class="docutils literal notranslate"><span class="pre">InvalidSMTPTLSReport</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.ParserError"><code class="docutils literal notranslate"><span class="pre">ParserError</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.email_results"><code class="docutils literal notranslate"><span class="pre">email_results()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.extract_xml"><code class="docutils literal notranslate"><span class="pre">extract_xml()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.extract_report"><code class="docutils literal notranslate"><span class="pre">extract_report()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.get_dmarc_reports_from_mailbox"><code class="docutils literal notranslate"><span class="pre">get_dmarc_reports_from_mailbox()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.get_dmarc_reports_from_mbox"><code class="docutils literal notranslate"><span class="pre">get_dmarc_reports_from_mbox()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.get_report_zip"><code class="docutils literal notranslate"><span class="pre">get_report_zip()</span></code></a></li>
@@ -72,10 +75,13 @@
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parse_forensic_report"><code class="docutils literal notranslate"><span class="pre">parse_forensic_report()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parse_report_email"><code class="docutils literal notranslate"><span class="pre">parse_report_email()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parse_report_file"><code class="docutils literal notranslate"><span class="pre">parse_report_file()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parse_smtp_tls_report_json"><code class="docutils literal notranslate"><span class="pre">parse_smtp_tls_report_json()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_aggregate_reports_to_csv"><code class="docutils literal notranslate"><span class="pre">parsed_aggregate_reports_to_csv()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_aggregate_reports_to_csv_rows"><code class="docutils literal notranslate"><span class="pre">parsed_aggregate_reports_to_csv_rows()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_forensic_reports_to_csv"><code class="docutils literal notranslate"><span class="pre">parsed_forensic_reports_to_csv()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_forensic_reports_to_csv_rows"><code class="docutils literal notranslate"><span class="pre">parsed_forensic_reports_to_csv_rows()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_smtp_tls_reports_to_csv"><code class="docutils literal notranslate"><span class="pre">parsed_smtp_tls_reports_to_csv()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.parsed_smtp_tls_reports_to_csv_rows"><code class="docutils literal notranslate"><span class="pre">parsed_smtp_tls_reports_to_csv_rows()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.save_output"><code class="docutils literal notranslate"><span class="pre">save_output()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.watch_inbox"><code class="docutils literal notranslate"><span class="pre">watch_inbox()</span></code></a></li>
</ul>
@@ -87,6 +93,7 @@
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.elastic.migrate_indexes"><code class="docutils literal notranslate"><span class="pre">migrate_indexes()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.elastic.save_aggregate_report_to_elasticsearch"><code class="docutils literal notranslate"><span class="pre">save_aggregate_report_to_elasticsearch()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.elastic.save_forensic_report_to_elasticsearch"><code class="docutils literal notranslate"><span class="pre">save_forensic_report_to_elasticsearch()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch"><code class="docutils literal notranslate"><span class="pre">save_smtp_tls_report_to_elasticsearch()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.elastic.set_hosts"><code class="docutils literal notranslate"><span class="pre">set_hosts()</span></code></a></li>
</ul>
</li>
@@ -94,6 +101,7 @@
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.splunk.HECClient"><code class="docutils literal notranslate"><span class="pre">HECClient</span></code></a><ul>
<li class="toctree-l4"><a class="reference internal" href="#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk"><code class="docutils literal notranslate"><span class="pre">HECClient.save_aggregate_reports_to_splunk()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk"><code class="docutils literal notranslate"><span class="pre">HECClient.save_forensic_reports_to_splunk()</span></code></a></li>
<li class="toctree-l4"><a class="reference internal" href="#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk"><code class="docutils literal notranslate"><span class="pre">HECClient.save_smtp_tls_reports_to_splunk()</span></code></a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.splunk.SplunkError"><code class="docutils literal notranslate"><span class="pre">SplunkError</span></code></a></li>
@@ -110,7 +118,7 @@
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.get_ip_address_info"><code class="docutils literal notranslate"><span class="pre">get_ip_address_info()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.get_reverse_dns"><code class="docutils literal notranslate"><span class="pre">get_reverse_dns()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.human_timestamp_to_datetime"><code class="docutils literal notranslate"><span class="pre">human_timestamp_to_datetime()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.human_timestamp_to_timestamp"><code class="docutils literal notranslate"><span class="pre">human_timestamp_to_timestamp()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.human_timestamp_to_unix_timestamp"><code class="docutils literal notranslate"><span class="pre">human_timestamp_to_unix_timestamp()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.is_mbox"><code class="docutils literal notranslate"><span class="pre">is_mbox()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.is_outlook_msg"><code class="docutils literal notranslate"><span class="pre">is_outlook_msg()</span></code></a></li>
<li class="toctree-l3"><a class="reference internal" href="#parsedmarc.utils.parse_email"><code class="docutils literal notranslate"><span class="pre">parse_email()</span></code></a></li>
@@ -171,6 +179,12 @@
<dd><p>Raised when an invalid DMARC forensic report is encountered</p>
</dd></dl>
<dl class="py exception">
<dt class="sig sig-object py" id="parsedmarc.InvalidSMTPTLSReport">
<em class="property"><span class="pre">exception</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">InvalidSMTPTLSReport</span></span><a class="reference internal" href="_modules/parsedmarc.html#InvalidSMTPTLSReport"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.InvalidSMTPTLSReport" title="Permalink to this definition"></a></dt>
<dd><p>Raised when an invalid SMTP TLS report is encountered</p>
</dd></dl>
<dl class="py exception">
<dt class="sig sig-object py" id="parsedmarc.ParserError">
<em class="property"><span class="pre">exception</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">ParserError</span></span><a class="reference internal" href="_modules/parsedmarc.html#ParserError"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.ParserError" title="Permalink to this definition"></a></dt>
@@ -204,16 +218,16 @@
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.extract_xml">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">extract_xml</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#extract_xml"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.extract_xml" title="Permalink to this definition"></a></dt>
<dd><p>Extracts xml from a zip or gzip file at the given path, file-like object,
<dt class="sig sig-object py" id="parsedmarc.extract_report">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">extract_report</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">input_</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#extract_report"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.extract_report" title="Permalink to this definition"></a></dt>
<dd><p>Extracts text from a zip or gzip file at the given path, file-like object,
or bytes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>input</strong> – A path to a file, a file like object, or bytes</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>The extracted XML</p>
<dd class="field-even"><p>The extracted text</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>str</p>
@@ -442,6 +456,12 @@ forensic report results</p></li>
</dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.parse_smtp_tls_report_json">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">parse_smtp_tls_report_json</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">report</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parse_smtp_tls_report_json"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.parse_smtp_tls_report_json" title="Permalink to this definition"></a></dt>
<dd><p>Parses and validates an SMTP TLS report</p>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.parsed_aggregate_reports_to_csv">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">parsed_aggregate_reports_to_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reports</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parsed_aggregate_reports_to_csv"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.parsed_aggregate_reports_to_csv" title="Permalink to this definition"></a></dt>
@@ -515,9 +535,34 @@ format</p>
</dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.parsed_smtp_tls_reports_to_csv">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">parsed_smtp_tls_reports_to_csv</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reports</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parsed_smtp_tls_reports_to_csv"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.parsed_smtp_tls_reports_to_csv" title="Permalink to this definition"></a></dt>
<dd><p>Converts one or more parsed SMTP TLS reports to flat CSV format, including
headers</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>reports</strong> – A parsed aggregate report or list of parsed aggregate reports</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Parsed aggregate report data in flat CSV format, including headers</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>str</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.parsed_smtp_tls_reports_to_csv_rows">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">parsed_smtp_tls_reports_to_csv_rows</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reports</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#parsed_smtp_tls_reports_to_csv_rows"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.parsed_smtp_tls_reports_to_csv_rows" title="Permalink to this definition"></a></dt>
<dd><p>Converts one oor more parsed SMTP TLS reports into a list of single
layer OrderedDict objects suitable for use in a CSV</p>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.save_output">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">save_output</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">results</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_directory</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'output'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">aggregate_json_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'aggregate.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">forensic_json_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'forensic.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">aggregate_csv_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'aggregate.csv'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">forensic_csv_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'forensic.csv'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#save_output"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.save_output" title="Permalink to this definition"></a></dt>
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">save_output</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">results</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_directory</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'output'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">aggregate_json_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'aggregate.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">forensic_json_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'forensic.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">smtp_tls_json_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'smtp_tls.json'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">aggregate_csv_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'aggregate.csv'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">forensic_csv_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'forensic.csv'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">smtp_tls_csv_filename</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'smtp_tls.csv'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#save_output"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.save_output" title="Permalink to this definition"></a></dt>
<dd><p>Save report data in the given directory</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
@@ -526,8 +571,10 @@ format</p>
<li><p><strong>output_directory</strong> (<em>str</em>) – The path to the directory to save in</p></li>
<li><p><strong>aggregate_json_filename</strong> (<em>str</em>) – Filename for the aggregate JSON file</p></li>
<li><p><strong>forensic_json_filename</strong> (<em>str</em>) – Filename for the forensic JSON file</p></li>
<li><p><strong>smtp_tls_json_filename</strong> (<em>str</em>) – Filename for the SMTP TLS JSON file</p></li>
<li><p><strong>aggregate_csv_filename</strong> (<em>str</em>) – Filename for the aggregate CSV file</p></li>
<li><p><strong>forensic_csv_filename</strong> (<em>str</em>) – Filename for the forensic CSV file</p></li>
<li><p><strong>smtp_tls_csv_filename</strong> (<em>str</em>) – Filename for the SMTP TLS CSV file</p></li>
</ul>
</dd>
</dl>
@@ -649,9 +696,29 @@ index</p></li>
</dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.elastic.</span></span><span class="sig-name descname"><span class="pre">save_smtp_tls_report_to_elasticsearch</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">report</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">index_suffix</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">monthly_indexes</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">number_of_shards</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">number_of_replicas</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_smtp_tls_report_to_elasticsearch"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch" title="Permalink to this definition"></a></dt>
<dd><p>Saves a parsed SMTP TLS report to elasticSearch</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>report</strong> (<em>OrderedDict</em>) – A parsed SMTP TLS report</p></li>
<li><p><strong>index_suffix</strong> (<em>str</em>) – The suffix of the name of the index to save to</p></li>
<li><p><strong>monthly_indexes</strong> (<em>bool</em>) – Use monthly indexes instead of daily indexes</p></li>
<li><p><strong>number_of_shards</strong> (<em>int</em>) – The number of shards to use in the index</p></li>
<li><p><strong>number_of_replicas</strong> (<em>int</em>) – The number of replicas to use in the index</p></li>
</ul>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><p><a class="reference internal" href="#parsedmarc.elastic.AlreadySaved" title="parsedmarc.elastic.AlreadySaved"><strong>AlreadySaved</strong></a> – </p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.elastic.set_hosts">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.elastic.</span></span><span class="sig-name descname"><span class="pre">set_hosts</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hosts</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_ssl</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ssl_cert_path</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">username</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">password</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">60.0</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#set_hosts"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.elastic.set_hosts" title="Permalink to this definition"></a></dt>
<span class="sig-prename descclassname"><span class="pre">parsedmarc.elastic.</span></span><span class="sig-name descname"><span class="pre">set_hosts</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">hosts</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_ssl</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ssl_cert_path</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">username</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">password</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">apiKey</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">60.0</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#set_hosts"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.elastic.set_hosts" title="Permalink to this definition"></a></dt>
<dd><p>Sets the Elasticsearch hosts to use</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
@@ -661,6 +728,7 @@ index</p></li>
<li><p><strong>ssl_cert_path</strong> (<em>str</em>) – Path to the certificate chain</p></li>
<li><p><strong>username</strong> (<em>str</em>) – The username to use for authentication</p></li>
<li><p><strong>password</strong> (<em>str</em>) – The password to use for authentication</p></li>
<li><p><strong>apiKey</strong> (<em>str</em>) – The Base64 encoded API key to use for authentication</p></li>
<li><p><strong>timeout</strong> (<em>float</em>) – Timeout in seconds</p></li>
</ul>
</dd>
@@ -711,6 +779,18 @@ to save in Splunk</p>
</dl>
</dd></dl>
<dl class="py method">
<dt class="sig sig-object py" id="parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">
<span class="sig-name descname"><span class="pre">save_smtp_tls_reports_to_splunk</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">reports</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/splunk.html#HECClient.save_smtp_tls_reports_to_splunk"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk" title="Permalink to this definition"></a></dt>
<dd><p>Saves aggregate DMARC reports to Splunk</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>reports</strong> – A list of SMTP TLS report dictionaries
to save in Splunk</p>
</dd>
</dl>
</dd></dl>
</dd></dl>
<dl class="py exception">
@@ -897,8 +977,8 @@ with the given IPv4 or IPv6 address</p>
</dd></dl>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.utils.human_timestamp_to_timestamp">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.utils.</span></span><span class="sig-name descname"><span class="pre">human_timestamp_to_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">human_timestamp</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#human_timestamp_to_timestamp"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.utils.human_timestamp_to_timestamp" title="Permalink to this definition"></a></dt>
<dt class="sig sig-object py" id="parsedmarc.utils.human_timestamp_to_unix_timestamp">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.utils.</span></span><span class="sig-name descname"><span class="pre">human_timestamp_to_unix_timestamp</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">human_timestamp</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/utils.html#human_timestamp_to_unix_timestamp"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.utils.human_timestamp_to_unix_timestamp" title="Permalink to this definition"></a></dt>
<dd><p>Converts a human-readable timestamp into a UNIX timestamp</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
contributing.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Contributing to parsedmarc &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Contributing to parsedmarc &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
davmail.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Accessing an inbox using OWA/EWS &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Accessing an inbox using OWA/EWS &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
dmarc.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Understanding DMARC &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Understanding DMARC &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
elasticsearch.html
+8 -6
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Elasticsearch and Kibana &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Elasticsearch and Kibana &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -266,7 +268,7 @@ Saved Objects page</p></li>
<h2>Records retention<a class="headerlink" href="#records-retention" title="Permalink to this heading"></a></h2>
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
index for each day to make it easy to comply with records
retention regulations such as GDPR. For fore information,
retention regulations such as GDPR. For more information,
check out the Elastic guide to <a class="reference external" href="https://www.elastic.co/blog/managing-time-based-indices-efficiently">managing time-based indexes efficiently</a>.</p>
</section>
</section>
genindex.html
+25 -11
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Index &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Index &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -33,7 +35,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -142,7 +144,7 @@
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.utils.EmailParserError">EmailParserError</a>
</li>
<li><a href="api.html#parsedmarc.extract_xml">extract_xml() (in module parsedmarc)</a>
<li><a href="api.html#parsedmarc.extract_report">extract_report() (in module parsedmarc)</a>
</li>
</ul></td>
</tr></table>
@@ -180,7 +182,7 @@
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.utils.human_timestamp_to_datetime">human_timestamp_to_datetime() (in module parsedmarc.utils)</a>
</li>
<li><a href="api.html#parsedmarc.utils.human_timestamp_to_timestamp">human_timestamp_to_timestamp() (in module parsedmarc.utils)</a>
<li><a href="api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">human_timestamp_to_unix_timestamp() (in module parsedmarc.utils)</a>
</li>
</ul></td>
</tr></table>
@@ -191,10 +193,12 @@
<li><a href="api.html#parsedmarc.InvalidAggregateReport">InvalidAggregateReport</a>
</li>
<li><a href="api.html#parsedmarc.InvalidDMARCReport">InvalidDMARCReport</a>
</li>
<li><a href="api.html#parsedmarc.InvalidForensicReport">InvalidForensicReport</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.InvalidForensicReport">InvalidForensicReport</a>
<li><a href="api.html#parsedmarc.InvalidSMTPTLSReport">InvalidSMTPTLSReport</a>
</li>
<li><a href="api.html#parsedmarc.utils.is_mbox">is_mbox() (in module parsedmarc.utils)</a>
</li>
@@ -238,16 +242,22 @@
<li><a href="api.html#parsedmarc.parse_report_email">parse_report_email() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parse_report_file">parse_report_file() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parse_smtp_tls_report_json">parse_smtp_tls_report_json() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parsed_aggregate_reports_to_csv">parsed_aggregate_reports_to_csv() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parsed_aggregate_reports_to_csv_rows">parsed_aggregate_reports_to_csv_rows() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parsed_forensic_reports_to_csv">parsed_forensic_reports_to_csv() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">parsed_forensic_reports_to_csv_rows() (in module parsedmarc)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.parsed_forensic_reports_to_csv_rows">parsed_forensic_reports_to_csv_rows() (in module parsedmarc)</a>
<li><a href="api.html#parsedmarc.parsed_smtp_tls_reports_to_csv">parsed_smtp_tls_reports_to_csv() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.parsed_smtp_tls_reports_to_csv_rows">parsed_smtp_tls_reports_to_csv_rows() (in module parsedmarc)</a>
</li>
<li>
parsedmarc
@@ -299,11 +309,15 @@
</li>
<li><a href="api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">save_forensic_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">save_forensic_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api.html#parsedmarc.save_output">save_output() (in module parsedmarc)</a>
</li>
<li><a href="api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">save_smtp_tls_report_to_elasticsearch() (in module parsedmarc.elastic)</a>
</li>
<li><a href="api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">save_smtp_tls_reports_to_splunk() (parsedmarc.splunk.HECClient method)</a>
</li>
<li><a href="api.html#parsedmarc.elastic.set_hosts">set_hosts() (in module parsedmarc.elastic)</a>
</li>
index.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -35,7 +37,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
installation.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Installation &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Installation &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
kibana.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Using the Kibana dashboards &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Using the Kibana dashboards &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
mailing-lists.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>What about mailing lists? &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>What about mailing lists? &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -34,7 +36,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
objects.inv
BIN
View File
Binary file not shown.
output.html
+51 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Sample outputs &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Sample outputs &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -59,6 +61,7 @@
<li class="toctree-l2"><a class="reference internal" href="#sample-forensic-report-output">Sample forensic report output</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#json-forensic-report">JSON forensic report</a></li>
<li class="toctree-l3"><a class="reference internal" href="#csv-forensic-report">CSV forensic report</a></li>
<li class="toctree-l3"><a class="reference internal" href="#json-smtp-tls-report">JSON SMTP TLS report</a></li>
</ul>
</li>
</ul>
@@ -284,6 +287,49 @@ auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,&quot;Mon, 01
</pre></div>
</div>
</section>
<section id="json-smtp-tls-report">
<h3>JSON SMTP TLS report<a class="headerlink" href="#json-smtp-tls-report" title="Permalink to this heading"></a></h3>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;organization_name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;Example Inc.&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;begin_date&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2024-01-09T00:00:00Z&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;end_date&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2024-01-09T23:59:59Z&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;report_id&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;2024-01-09T00:00:00Z_example.com&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;policies&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;policy_domain&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;example.com&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;policy_type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;sts&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;policy_strings&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">&quot;version: STSv1&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="s2">&quot;mode: testing&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="s2">&quot;mx: example.com&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="s2">&quot;max_age: 86400&quot;</span>
<span class="w"> </span><span class="p">],</span>
<span class="w"> </span><span class="nt">&quot;successful_session_count&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">0</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;failed_session_count&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;failure_details&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;result_type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;validation-failure&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;failed_session_count&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">2</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;sending_mta_ip&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;209.85.222.201&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;receiving_ip&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;173.212.201.41&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;receiving_mx_hostname&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;example.com&quot;</span>
<span class="w"> </span><span class="p">},</span>
<span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;result_type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;validation-failure&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;failed_session_count&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;sending_mta_ip&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;209.85.208.176&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;receiving_ip&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;173.212.201.41&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;receiving_mx_hostname&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;example.com&quot;</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">]</span>
</pre></div>
</div>
</section>
</section>
</section>
py-modindex.html
+7 -5
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Python Module Index &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Python Module Index &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
search.html
+7 -5
View File
@@ -1,11 +1,13 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Search &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Search &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="#" method="get">
searchindex.js
+1 -1
View File
File diff suppressed because one or more lines are too long
splunk.html
+7 -5
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Splunk &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Splunk &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
usage.html
+18 -9
View File
@@ -1,12 +1,14 @@
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<html class="writer-html5" lang="en">
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Using parsedmarc &mdash; parsedmarc 8.6.4 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<title>Using parsedmarc &mdash; parsedmarc 8.7.0 documentation</title>
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" type="text/css" href="_static/css/theme.css" />
<!--[if lt IE 9]>
<script src="_static/js/html5shiv.min.js"></script>
<![endif]-->
@@ -36,7 +38,7 @@
parsedmarc
</a>
<div class="version">
8.6.4
8.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -227,10 +229,10 @@ Gmail) where the incoming reports can be found
(Default: <code class="docutils literal notranslate"><span class="pre">INBOX</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">archive_folder</span></code> - str: The mailbox folder (or label for
Gmail) to sort processed emails into (Default: <code class="docutils literal notranslate"><span class="pre">Archive</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">watch</span></code> - bool: Use the IMAP <code class="docutils literal notranslate"><span class="pre">IDLE</span></code> command to process</p></li>
<li><p>messages as they arrive or poll MS Graph for new messages</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">delete</span></code> - bool: Delete messages after processing them,</p></li>
<li><p>instead of archiving them</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">watch</span></code> - bool: Use the IMAP <code class="docutils literal notranslate"><span class="pre">IDLE</span></code> command to process
messages as they arrive or poll MS Graph for new messages</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">delete</span></code> - bool: Delete messages after processing them,
instead of archiving them</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">test</span></code> - bool: Do not move or delete messages</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code> - int: Number of messages to read and process
before saving. Default <code class="docutils literal notranslate"><span class="pre">10</span></code>. Use <code class="docutils literal notranslate"><span class="pre">0</span></code> for no limit.</p></li>
@@ -325,8 +327,12 @@ or URLs (e.g. <code class="docutils literal notranslate"><span class="pre">127.0
<a class="reference external" href="https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters">URL encoded</a>.</p>
</div>
</li>
<li><p><code class="docutils literal notranslate"><span class="pre">user</span></code> - str: Basic auth username</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">password</span></code> - str: Basic auth password</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">apiKey</span></code> - str: API key</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">ssl</span></code> - bool: Use an encrypted SSL/TLS connection
(Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">timeout</span></code> - float: Timeout in seconds (Default: 60)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">cert_path</span></code> - str: Path to a trusted certificates</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">index_suffix</span></code> - str: A suffix to apply to the index names</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">monthly_indexes</span></code> - bool: Use monthly indexes instead of daily indexes</p></li>
@@ -410,6 +416,8 @@ acquiring credentials
(Default: <code class="docutils literal notranslate"><span class="pre">https://www.googleapis.com/auth/gmail.modify</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">oauth2_port</span></code> - int: The TCP port for the local server to
listen on for the OAuth2 response (Default: <code class="docutils literal notranslate"><span class="pre">8080</span></code>)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">paginate_messages</span></code> - bool: When <code class="docutils literal notranslate"><span class="pre">True</span></code>, fetch all applicable Gmail messages.
When <code class="docutils literal notranslate"><span class="pre">False</span></code>, only fetch up to 100 new messages per run (Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
</ul>
</li>
<li><p><code class="docutils literal notranslate"><span class="pre">log_analytics</span></code></p>
@@ -421,6 +429,7 @@ listen on for the OAuth2 response (Default: <code class="docutils literal notran
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_immutable_id</span></code> - str: The immutable ID of the Data Collection Rule (DCR)</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_aggregate_stream</span></code> - str: The stream name for aggregate reports in the DCR</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_forensic_stream</span></code> - str: The stream name for the forensic reports in the DCR</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">dcr_smtp_tls_stream</span></code> - str: The stream name for the SMTP TLS reports in the DCR</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>