This commit is contained in:
Sean Whalen
2022-05-10 21:07:04 -04:00
parent a2cae44500
commit 38f5cc983e
9 changed files with 40 additions and 22 deletions
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Overview: module code &mdash; parsedmarc 8.1.1 documentation</title>
<title>Overview: module code &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -27,7 +27,7 @@
<a href="../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+23 -6
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc &mdash; parsedmarc 8.1.1 documentation</title>
<title>parsedmarc &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -27,7 +27,7 @@
<a href="../index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
@@ -100,7 +100,7 @@
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">parse_email</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="kn">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.1.1&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;8.2.0&quot;</span>
<span class="n">formatter</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span>
<span class="n">fmt</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">%(levelname)8s</span><span class="s1">:</span><span class="si">%(filename)s</span><span class="s1">:</span><span class="si">%(lineno)d</span><span class="s1">:</span><span class="si">%(message)s</span><span class="s1">&#39;</span><span class="p">,</span>
@@ -115,6 +115,7 @@
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^([\w\-]+): (.+)$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^&lt;\?xml .*?&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;&lt;/??xs:schema.*&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">text_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;\s*([a-zA-Z\s]+):\s(.+)&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">&quot;</span>
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x1F\x8B</span><span class="s2">&quot;</span>
@@ -674,7 +675,7 @@
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;arrival_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">msg_date</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<span class="k">if</span> <span class="s2">&quot;version&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="s2">&quot;user_agent&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_report</span><span class="p">:</span>
<span class="n">parsed_report</span><span class="p">[</span><span class="s2">&quot;user_agent&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
@@ -895,6 +896,21 @@
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;message/rfc822&quot;</span><span class="p">:</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">payload</span>
<span class="k">elif</span> <span class="n">content_type</span> <span class="o">==</span> <span class="s2">&quot;text/plain&quot;</span><span class="p">:</span>
<span class="k">if</span> <span class="s2">&quot;A message claiming to be from you has failed&quot;</span> <span class="ow">in</span> <span class="n">payload</span><span class="p">:</span>
<span class="n">parts</span> <span class="o">=</span> <span class="n">payload</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;detected.&quot;</span><span class="p">)</span>
<span class="n">field_matches</span> <span class="o">=</span> <span class="n">text_report_regex</span><span class="o">.</span><span class="n">findall</span><span class="p">(</span><span class="n">parts</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="n">fields</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
<span class="k">for</span> <span class="n">match</span> <span class="ow">in</span> <span class="n">field_matches</span><span class="p">:</span>
<span class="n">field_name</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;-&quot;</span><span class="p">)</span>
<span class="n">fields</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
<span class="n">feedback_report</span> <span class="o">=</span> <span class="s2">&quot;Arrival-Date: </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span> \
<span class="s2">&quot;Source-IP: </span><span class="si">{}</span><span class="s2">&quot;</span> \
<span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">fields</span><span class="p">[</span><span class="s2">&quot;received-date&quot;</span><span class="p">],</span>
<span class="n">fields</span><span class="p">[</span><span class="s2">&quot;sender-ip-address&quot;</span><span class="p">])</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">parts</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lstrip</span><span class="p">()</span>
<span class="n">sample</span> <span class="o">=</span> <span class="n">sample</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;=</span><span class="se">\r\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">sample</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">payload</span> <span class="o">=</span> <span class="n">b64decode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
@@ -923,7 +939,7 @@
<span class="s1">&#39;aggregate DMARC report: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidAggregateReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">error</span> <span class="o">=</span> <span class="s1">&#39;Unable to parse message with &#39;</span> \
<span class="s1">&#39;subject &quot;</span><span class="si">{0}</span><span class="s1">&quot;: </span><span class="si">{1}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">subject</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">InvalidDMARCReport</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
@@ -1084,7 +1100,7 @@
<span class="n">dns_timeout</span><span class="o">=</span><span class="mf">6.0</span><span class="p">,</span>
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="n">results</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">batch_size</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
<span class="n">create_folders</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Fetches and parses DMARC reports from a mailbox</span>
@@ -1103,6 +1119,7 @@
<span class="sd"> forensic report results</span>
<span class="sd"> results (dict): Results from the previous run</span>
<span class="sd"> batch_size (int): Number of messages to read and process before saving</span>
<span class="sd"> (use 0 for no limit)</span>
<span class="sd"> create_folders (bool): Whether to create the destination folders</span>
<span class="sd"> (not used in watch)</span>
+1 -1
View File
@@ -195,7 +195,7 @@ The full set of configuration options are:
- ``watch`` - bool: Use the IMAP ``IDLE`` command to process messages as they arrive or poll MS Graph for new messages
- ``delete`` - bool: Delete messages after processing them, instead of archiving them
- ``test`` - bool: Do not move or delete messages
- ``batch_size`` - int: Number of messages to read and process before saving. Defaults to all messages if not set.
- ``batch_size`` - int: Number of messages to read and process before saving. Default 10. Use 0 for no limit.
- ``imap``
- ``host`` - str: The IMAP server hostname or IP address
+1 -1
View File
@@ -1,6 +1,6 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '8.1.1',
VERSION: '8.2.0',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
BUILDER: 'html',
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Index &mdash; parsedmarc 8.1.1 documentation</title>
<title>Index &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -27,7 +27,7 @@
<a href="index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+6 -5
View File
@@ -4,7 +4,7 @@
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 8.1.1 documentation</title>
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -28,7 +28,7 @@
<a href="#" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
@@ -314,7 +314,7 @@ lookalike domain monitoring, check out <a class="reference external" href="https
<li><p><code class="docutils literal notranslate"><span class="pre">watch</span></code> - bool: Use the IMAP <code class="docutils literal notranslate"><span class="pre">IDLE</span></code> command to process messages as they arrive or poll MS Graph for new messages</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">delete</span></code> - bool: Delete messages after processing them, instead of archiving them</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">test</span></code> - bool: Do not move or delete messages</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code> - int: Number of messages to read and process before saving. Defaults to all messages if not set.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">batch_size</span></code> - int: Number of messages to read and process before saving. Default 10. Use 0 for no limit.</p></li>
</ul>
</dd>
</dl>
@@ -1772,7 +1772,7 @@ or bytes.</p>
<dl class="py function">
<dt class="sig sig-object py" id="parsedmarc.get_dmarc_reports_from_mailbox">
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">get_dmarc_reports_from_mailbox</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">connection</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">parsedmarc.mail.mailbox_connection.MailboxConnection</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">reports_folder</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'INBOX'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">archive_folder</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'Archive'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">delete</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">test</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ip_db_path</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">offline</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">nameservers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dns_timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">6.0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">strip_attachment_payloads</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">results</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">batch_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">create_folders</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#get_dmarc_reports_from_mailbox"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.get_dmarc_reports_from_mailbox" title="Permalink to this definition"></a></dt>
<span class="sig-prename descclassname"><span class="pre">parsedmarc.</span></span><span class="sig-name descname"><span class="pre">get_dmarc_reports_from_mailbox</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">connection</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">parsedmarc.mail.mailbox_connection.MailboxConnection</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">reports_folder</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'INBOX'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">archive_folder</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'Archive'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">delete</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">test</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">ip_db_path</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">offline</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">nameservers</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">dns_timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">6.0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">strip_attachment_payloads</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">results</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">batch_size</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">10</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">create_folders</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc.html#get_dmarc_reports_from_mailbox"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#parsedmarc.get_dmarc_reports_from_mailbox" title="Permalink to this definition"></a></dt>
<dd><p>Fetches and parses DMARC reports from a mailbox</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
@@ -1789,7 +1789,8 @@ or bytes.</p>
<li><p><strong>strip_attachment_payloads</strong> (<em>bool</em>) Remove attachment payloads from
forensic report results</p></li>
<li><p><strong>results</strong> (<em>dict</em>) Results from the previous run</p></li>
<li><p><strong>batch_size</strong> (<em>int</em>) Number of messages to read and process before saving</p></li>
<li><p><strong>batch_size</strong> (<em>int</em>) Number of messages to read and process before saving
(use 0 for no limit)</p></li>
<li><p><strong>create_folders</strong> (<em>bool</em>) Whether to create the destination folders
(not used in watch)</p></li>
</ul>
+1 -1
View File
@@ -1,6 +1,6 @@
# Sphinx inventory version 2
# Project: parsedmarc
# Version: 8.1.1
# Version: 8.2.0
# The remainder of this file is compressed using zlib.
xÚ­—ÍN1Çïy
Kí5«"U=pCªP<€eìÉÆÂ[²<}ý±@€ Ö—Äkû÷Ÿñx<Þ˜Cš9N†ñT[D»nãµúVú–ßÉr1<Oî.Í–))ÎúÞAÏ<ÜÀ`O°ã0xi
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Python Module Index &mdash; parsedmarc 8.1.1 documentation</title>
<title>Python Module Index &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<!--[if lt IE 9]>
@@ -30,7 +30,7 @@
<a href="index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
+2 -2
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Search &mdash; parsedmarc 8.1.1 documentation</title>
<title>Search &mdash; parsedmarc 8.2.0 documentation</title>
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
@@ -30,7 +30,7 @@
<a href="index.html" class="icon icon-home"> parsedmarc
</a>
<div class="version">
8.1.1
8.2.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="#" method="get">