mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-04-04 12:48:53 +00:00
5.2.0
This commit is contained in:
Sean Whalen
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Overview: module code — parsedmarc 5.1.3 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>parsedmarc — parsedmarc 5.1.3 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
@@ -183,7 +183,12 @@
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
|
||||
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">parse_email</span>
|
||||
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"5.1.3"</span>
|
||||
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">"5.2.0"</span>
|
||||
|
||||
<span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span>
|
||||
<span class="nb">format</span><span class="o">=</span><span class="s1">'</span><span class="si">%(levelname)8s</span><span class="s1">:</span><span class="si">%(filename)s</span><span class="s1">:</span><span class="si">%(lineno)d</span><span class="s1">:'</span>
|
||||
<span class="s1">'</span><span class="si">%(message)s</span><span class="s1">'</span><span class="p">,</span>
|
||||
<span class="n">datefmt</span><span class="o">=</span><span class="s1">'%Y-%m-</span><span class="si">%d</span><span class="s1">:%H:%M:%S'</span><span class="p">)</span>
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"parsedmarc"</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"parsedmarc v</span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
|
||||
@@ -1317,11 +1322,12 @@
|
||||
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
||||
<span class="k">if</span> <span class="s2">"Server Unavailable"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Received Server Unavailable response"</span>
|
||||
<span class="s2">"Waiting </span><span class="si">{0}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
||||
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">error</span><span class="p">,</span>
|
||||
<span class="n">sleep_minutes</span><span class="p">))</span>
|
||||
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">sleep_minutes</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span>
|
||||
@@ -1351,7 +1357,31 @@
|
||||
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection refused"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">ConnectionResetError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection reset"</span><span class="p">)</span>
|
||||
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"Connection reset. "</span>
|
||||
<span class="s2">"Waiting </span><span class="si">{0}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">sleep_minutes</span><span class="p">))</span>
|
||||
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">sleep_minutes</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)</span>
|
||||
<span class="n">results</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span>
|
||||
<span class="n">host</span><span class="o">=</span><span class="n">host</span><span class="p">,</span>
|
||||
<span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span>
|
||||
<span class="n">password</span><span class="o">=</span><span class="n">password</span><span class="p">,</span>
|
||||
<span class="n">connection</span><span class="o">=</span><span class="n">connection</span><span class="p">,</span>
|
||||
<span class="n">port</span><span class="o">=</span><span class="n">port</span><span class="p">,</span>
|
||||
<span class="n">ssl</span><span class="o">=</span><span class="n">ssl</span><span class="p">,</span>
|
||||
<span class="n">ssl_context</span><span class="o">=</span><span class="n">ssl_context</span><span class="p">,</span>
|
||||
<span class="n">move_supported</span><span class="o">=</span><span class="n">move_supported</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="n">reports_folder</span><span class="p">,</span>
|
||||
<span class="n">archive_folder</span><span class="o">=</span><span class="n">archive_folder</span><span class="p">,</span>
|
||||
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
||||
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dns_timeout</span><span class="p">,</span>
|
||||
<span class="n">strip_attachment_payloads</span><span class="o">=</span><span class="n">strip_attachment_payloads</span><span class="p">,</span>
|
||||
<span class="n">results</span><span class="o">=</span><span class="n">results</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">results</span>
|
||||
<span class="k">except</span> <span class="ne">ConnectionAbortedError</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"Connection aborted"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">TimeoutError</span><span class="p">:</span>
|
||||
@@ -1607,8 +1637,13 @@
|
||||
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
||||
<span class="k">if</span> <span class="s2">"Server Unavailable"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"IMAP error: </span><span class="si">{0}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
||||
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">error</span><span class="p">,</span>
|
||||
<span class="n">sleep_minutes</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
||||
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
||||
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
||||
@@ -1736,8 +1771,33 @@
|
||||
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
||||
<span class="k">break</span>
|
||||
<span class="k">except</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">IMAPClientError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"'"</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="n">error</span> <span class="o">=</span> <span class="n">error</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"b'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"'"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span>
|
||||
<span class="c1"># Workaround for random Exchange/Office365 IMAP errors</span>
|
||||
<span class="k">if</span> <span class="s2">"unexpected response"</span> <span class="ow">in</span> <span class="n">error</span> <span class="ow">or</span> <span class="s2">"BAD"</span> <span class="ow">in</span> <span class="n">error</span><span class="p">:</span>
|
||||
<span class="n">sleep_minutes</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
|
||||
<span class="s2">"</span><span class="si">{0}</span><span class="s2">. "</span>
|
||||
<span class="s2">"Waiting </span><span class="si">{1}</span><span class="s2"> minutes before trying again"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
|
||||
<span class="n">error</span><span class="p">,</span>
|
||||
<span class="n">sleep_minutes</span><span class="p">))</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"Reconnecting watcher"</span><span class="p">)</span>
|
||||
<span class="n">server</span> <span class="o">=</span> <span class="n">imapclient</span><span class="o">.</span><span class="n">IMAPClient</span><span class="p">(</span><span class="n">host</span><span class="p">)</span>
|
||||
<span class="n">server</span><span class="o">.</span><span class="n">login</span><span class="p">(</span><span class="n">username</span><span class="p">,</span> <span class="n">password</span><span class="p">)</span>
|
||||
<span class="n">server</span><span class="o">.</span><span class="n">select_folder</span><span class="p">(</span><span class="n">rf</span><span class="p">)</span>
|
||||
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
|
||||
<span class="n">ms</span> <span class="o">=</span> <span class="s2">"MOVE"</span> <span class="ow">in</span> <span class="n">get_imap_capabilities</span><span class="p">(</span><span class="n">server</span><span class="p">)</span>
|
||||
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
|
||||
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
|
||||
<span class="n">reports_folder</span><span class="o">=</span><span class="n">rf</span><span class="p">,</span>
|
||||
<span class="n">archive_folder</span><span class="o">=</span><span class="n">af</span><span class="p">,</span>
|
||||
<span class="n">delete</span><span class="o">=</span><span class="n">delete</span><span class="p">,</span>
|
||||
<span class="n">test</span><span class="o">=</span><span class="n">test</span><span class="p">,</span>
|
||||
<span class="n">nameservers</span><span class="o">=</span><span class="n">ns</span><span class="p">,</span>
|
||||
<span class="n">dns_timeout</span><span class="o">=</span><span class="n">dt</span><span class="p">)</span>
|
||||
<span class="n">callback</span><span class="p">(</span><span class="n">res</span><span class="p">)</span>
|
||||
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="n">error</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">gaierror</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">IMAPError</span><span class="p">(</span><span class="s2">"DNS resolution failed"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="ne">ConnectionRefusedError</span><span class="p">:</span>
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>parsedmarc.elastic — parsedmarc 5.1.3 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
@@ -314,16 +314,29 @@
|
||||
<span class="sd">"""Raised when a report to be saved matches an existing report"""</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">):</span>
|
||||
<div class="viewcode-block" id="set_hosts"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.set_hosts">[docs]</a><span class="k">def</span> <span class="nf">set_hosts</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="n">use_ssl</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ssl_cert_path</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Sets the Elasticsearch hosts to use</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> hosts: A single hostname or URL, or list of hostnames or URLs</span>
|
||||
<span class="sd"> hosts (str): A single hostname or URL, or list of hostnames or URLs</span>
|
||||
<span class="sd"> use_ssl (bool): Use a HTTPS connection to the server</span>
|
||||
<span class="sd"> ssl_cert_path (str): Path to the certificate chain</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">hosts</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">list</span><span class="p">:</span>
|
||||
<span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="n">hosts</span><span class="o">=</span><span class="n">hosts</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="mi">20</span><span class="p">)</span></div>
|
||||
<span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"hosts"</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span>
|
||||
<span class="s2">"timeout"</span><span class="p">:</span> <span class="mi">20</span>
|
||||
<span class="p">}</span>
|
||||
<span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'use_ssl'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'ca_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">conn_params</span><span class="p">[</span><span class="s1">'verify_certs'</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="create_indexes"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.create_indexes">[docs]</a><span class="k">def</span> <span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">,</span> <span class="n">settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
@@ -399,13 +412,15 @@
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span><span class="n">aggregate_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC aggregate report to ElasticSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
@@ -420,7 +435,10 @@
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">])</span>
|
||||
<span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
||||
<span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S"</span><span class="p">)</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
|
||||
<span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"end_date"</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
|
||||
<span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"begin_date"</span><span class="p">],</span>
|
||||
@@ -509,13 +527,16 @@
|
||||
|
||||
|
||||
<div class="viewcode-block" id="save_forensic_report_to_elasticsearch"><a class="viewcode-back" href="../../index.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a><span class="k">def</span> <span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">,</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">index_suffix</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">monthly_indexes</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Saves a parsed DMARC forensic report to ElasticSearch</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> forensic_report (OrderedDict): A parsed forensic report</span>
|
||||
<span class="sd"> index_suffix (str): The suffix of the name of the index to save to</span>
|
||||
<span class="sd"> monthly_indexes (bool): Use monthly indexes instead of daily</span>
|
||||
<span class="sd"> indexes</span>
|
||||
|
||||
<span class="sd"> Raises:</span>
|
||||
<span class="sd"> AlreadySaved</span>
|
||||
@@ -619,7 +640,10 @@
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"dmarc_forensic"</span>
|
||||
<span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">"%Y-%m-</span><span class="si">%d</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">index</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
|
||||
<span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">])</span>
|
||||
<span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>parsedmarc.splunk — parsedmarc 5.1.3 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>parsedmarc.utils — parsedmarc 5.1.3 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@@ -62,27 +62,30 @@ CLI help
|
||||
::
|
||||
|
||||
usage: parsedmarc [-h] [--strip-attachment-payloads] [-o OUTPUT]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] [-H HOST]
|
||||
[-u USER] [-p PASSWORD] [--imap-port IMAP_PORT]
|
||||
[--imap-skip-certificate-verification] [--imap-no-ssl]
|
||||
[-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d]
|
||||
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
|
||||
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
|
||||
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
|
||||
[--hec-skip-certificate-verification]
|
||||
[-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]]
|
||||
[--kafka-username KAFKA_USERNAME]
|
||||
[--kafka-password KAFKA_PASSWORD] [--kafka-use-ssl]
|
||||
[--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC]
|
||||
[--kafka-forensic_topic KAFKA_FORENSIC_TOPIC]
|
||||
[--save-aggregate] [--save-forensic] [-O OUTGOING_HOST]
|
||||
[-U OUTGOING_USER] [-P OUTGOING_PASSWORD]
|
||||
[--outgoing-port OUTGOING_PORT]
|
||||
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
|
||||
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
|
||||
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
|
||||
[-s] [--debug] [-v]
|
||||
[file_path [file_path ...]]
|
||||
[-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] [-H HOST]
|
||||
[-u USER] [-p PASSWORD] [--imap-port IMAP_PORT]
|
||||
[--imap-skip-certificate-verification] [--imap-no-ssl]
|
||||
[-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d]
|
||||
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
|
||||
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
|
||||
[--elasticsearch-use-ssl]
|
||||
[--elasticsearch-ssl-cert-path ELASTICSEARCH_SSL_CERT_PATH]
|
||||
[--elasticsearch-monthly-indexes] [--hec HEC]
|
||||
[--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
|
||||
[--hec-skip-certificate-verification]
|
||||
[-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]]
|
||||
[--kafka-username KAFKA_USERNAME]
|
||||
[--kafka-password KAFKA_PASSWORD] [--kafka-use-ssl]
|
||||
[--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC]
|
||||
[--kafka-forensic_topic KAFKA_FORENSIC_TOPIC]
|
||||
[--save-aggregate] [--save-forensic] [-O OUTGOING_HOST]
|
||||
[-U OUTGOING_USER] [-P OUTGOING_PASSWORD]
|
||||
[--outgoing-port OUTGOING_PORT]
|
||||
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
|
||||
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
|
||||
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
|
||||
[-s] [--debug] [--log-file LOG_FILE] [-v]
|
||||
[file_path [file_path ...]]
|
||||
|
||||
Parses DMARC reports
|
||||
|
||||
@@ -124,6 +127,13 @@ CLI help
|
||||
--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX
|
||||
append this suffix to the dmarc_aggregate and
|
||||
dmarc_forensic Elasticsearch index names, joined by _
|
||||
--elasticsearch-use-ssl
|
||||
Use SSL when connecting to Elasticsearch
|
||||
--elasticsearch-ssl-cert-path ELASTICSEARCH_SSL_CERT_PATH
|
||||
Path to the Elasticsearch SSL certificate
|
||||
--elasticsearch-monthly-indexes
|
||||
Use monthly Elasticsearch indexes instead of daily
|
||||
indexes
|
||||
--hec HEC the URL to a Splunk HTTP Event Collector (HEC)
|
||||
--hec-token HEC_TOKEN
|
||||
the authorization token for a Splunk HTTP Event
|
||||
@@ -134,7 +144,7 @@ CLI help
|
||||
--hec-skip-certificate-verification
|
||||
skip certificate verification for Splunk HEC
|
||||
-K [KAFKA_HOSTS [KAFKA_HOSTS ...]], --kafka-hosts [KAFKA_HOSTS [KAFKA_HOSTS ...]]
|
||||
s list of one or more Kafka hostnames
|
||||
a list of one or more Kafka hostnames
|
||||
--kafka-username KAFKA_USERNAME
|
||||
an optional Kafka username
|
||||
--kafka-password KAFKA_PASSWORD
|
||||
@@ -175,6 +185,7 @@ CLI help
|
||||
--test do not move or delete IMAP messages
|
||||
-s, --silent only print errors and warnings
|
||||
--debug print debugging information
|
||||
--log-file LOG_FILE output logging to a file
|
||||
-v, --version show program's version number and exit
|
||||
|
||||
Sample aggregate report output
|
||||
@@ -461,12 +472,18 @@ Installation using pypy3
|
||||
------------------------
|
||||
|
||||
For the best possible processing speed, consider using ``parsedmarc`` inside a ``pypy3``
|
||||
virtualenv. First, `download the latest version of pypy3`_. Extract it to
|
||||
virtualenv. First, `download the latest portable Linux version of pypy3`_. Extract it to
|
||||
``/opt/pypy3`` (``sudo mkdir /opt`` if ``/opt`` does not exist), then create a
|
||||
symlink:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
tar -jxf pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
rm pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
sudo chown -R root:root pypy3.5-6.0.0-linux_x86_64-portable
|
||||
sudo mv pypy3.5-6.0.0-linux_x86_64-portable /opt/pypy3
|
||||
sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3
|
||||
|
||||
Install ``virtualenv`` on your system:
|
||||
@@ -1232,7 +1249,7 @@ Indices and tables
|
||||
|
||||
.. _Demystifying DMARC: https://seanthegeek.net/459/demystifying-dmarc/
|
||||
|
||||
.. _download the latest version of pypy3: https://pypy.org/download.html#default-with-a-jit-compiler
|
||||
.. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
|
||||
|
||||
.. _Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
var DOCUMENTATION_OPTIONS = {
|
||||
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
|
||||
VERSION: '5.1.3',
|
||||
VERSION: '5.2.0',
|
||||
LANGUAGE: 'None',
|
||||
COLLAPSE_INDEX: false,
|
||||
FILE_SUFFIX: '.html',
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Index — parsedmarc 5.1.3 documentation</title>
|
||||
<title>Index — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
84
index.html
84
index.html
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 5.1.3 documentation</title>
|
||||
<title>parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
@@ -245,27 +245,30 @@ lookalike domain monitoring, check out <a class="reference external" href="https
|
||||
<div class="section" id="cli-help">
|
||||
<h2>CLI help<a class="headerlink" href="#cli-help" title="Permalink to this headline">¶</a></h2>
|
||||
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">usage</span><span class="p">:</span> <span class="n">parsedmarc</span> <span class="p">[</span><span class="o">-</span><span class="n">h</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">strip</span><span class="o">-</span><span class="n">attachment</span><span class="o">-</span><span class="n">payloads</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">o</span> <span class="n">OUTPUT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">TIMEOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">H</span> <span class="n">HOST</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">u</span> <span class="n">USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PASSWORD</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">port</span> <span class="n">IMAP_PORT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">no</span><span class="o">-</span><span class="n">ssl</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">r</span> <span class="n">REPORTS_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">a</span> <span class="n">ARCHIVE_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">E</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="o">...</span><span class="p">]]]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">suffix</span> <span class="n">ELASTICSEARCH_INDEX_SUFFIX</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">hec</span> <span class="n">HEC</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">token</span> <span class="n">HEC_TOKEN</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">index</span> <span class="n">HEC_INDEX</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">K</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]]]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">username</span> <span class="n">KAFKA_USERNAME</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">password</span> <span class="n">KAFKA_PASSWORD</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">use</span><span class="o">-</span><span class="n">ssl</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">aggregate</span><span class="o">-</span><span class="n">topic</span> <span class="n">KAFKA_AGGREGATE_TOPIC</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">forensic_topic</span> <span class="n">KAFKA_FORENSIC_TOPIC</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">aggregate</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">forensic</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">O</span> <span class="n">OUTGOING_HOST</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">U</span> <span class="n">OUTGOING_USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">P</span> <span class="n">OUTGOING_PASSWORD</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">port</span> <span class="n">OUTGOING_PORT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">ssl</span> <span class="n">OUTGOING_SSL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">F</span> <span class="n">OUTGOING_FROM</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">T</span> <span class="n">OUTGOING_TO</span> <span class="p">[</span><span class="n">OUTGOING_TO</span> <span class="o">...</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">S</span> <span class="n">OUTGOING_SUBJECT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">A</span> <span class="n">OUTGOING_ATTACHMENT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">M</span> <span class="n">OUTGOING_MESSAGE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">w</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">test</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">s</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">debug</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">v</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="n">file_path</span> <span class="p">[</span><span class="n">file_path</span> <span class="o">...</span><span class="p">]]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">n</span> <span class="n">NAMESERVERS</span> <span class="p">[</span><span class="n">NAMESERVERS</span> <span class="o">...</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">t</span> <span class="n">TIMEOUT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">H</span> <span class="n">HOST</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">u</span> <span class="n">USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">p</span> <span class="n">PASSWORD</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">port</span> <span class="n">IMAP_PORT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">imap</span><span class="o">-</span><span class="n">no</span><span class="o">-</span><span class="n">ssl</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">r</span> <span class="n">REPORTS_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">a</span> <span class="n">ARCHIVE_FOLDER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">d</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">E</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="p">[</span><span class="n">ELASTICSEARCH_HOST</span> <span class="o">...</span><span class="p">]]]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">suffix</span> <span class="n">ELASTICSEARCH_INDEX_SUFFIX</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">use</span><span class="o">-</span><span class="n">ssl</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">ssl</span><span class="o">-</span><span class="n">cert</span><span class="o">-</span><span class="n">path</span> <span class="n">ELASTICSEARCH_SSL_CERT_PATH</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">monthly</span><span class="o">-</span><span class="n">indexes</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span> <span class="n">HEC</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">token</span> <span class="n">HEC_TOKEN</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">index</span> <span class="n">HEC_INDEX</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">K</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]]]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">username</span> <span class="n">KAFKA_USERNAME</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">password</span> <span class="n">KAFKA_PASSWORD</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">use</span><span class="o">-</span><span class="n">ssl</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">aggregate</span><span class="o">-</span><span class="n">topic</span> <span class="n">KAFKA_AGGREGATE_TOPIC</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">forensic_topic</span> <span class="n">KAFKA_FORENSIC_TOPIC</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">aggregate</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">save</span><span class="o">-</span><span class="n">forensic</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">O</span> <span class="n">OUTGOING_HOST</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">U</span> <span class="n">OUTGOING_USER</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">P</span> <span class="n">OUTGOING_PASSWORD</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">port</span> <span class="n">OUTGOING_PORT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">--</span><span class="n">outgoing</span><span class="o">-</span><span class="n">ssl</span> <span class="n">OUTGOING_SSL</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">F</span> <span class="n">OUTGOING_FROM</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">T</span> <span class="n">OUTGOING_TO</span> <span class="p">[</span><span class="n">OUTGOING_TO</span> <span class="o">...</span><span class="p">]]</span> <span class="p">[</span><span class="o">-</span><span class="n">S</span> <span class="n">OUTGOING_SUBJECT</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">A</span> <span class="n">OUTGOING_ATTACHMENT</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">M</span> <span class="n">OUTGOING_MESSAGE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">w</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">test</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="o">-</span><span class="n">s</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">debug</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">log</span><span class="o">-</span><span class="n">file</span> <span class="n">LOG_FILE</span><span class="p">]</span> <span class="p">[</span><span class="o">-</span><span class="n">v</span><span class="p">]</span>
|
||||
<span class="p">[</span><span class="n">file_path</span> <span class="p">[</span><span class="n">file_path</span> <span class="o">...</span><span class="p">]]</span>
|
||||
|
||||
<span class="n">Parses</span> <span class="n">DMARC</span> <span class="n">reports</span>
|
||||
|
||||
@@ -307,6 +310,13 @@ lookalike domain monitoring, check out <a class="reference external" href="https
|
||||
<span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">index</span><span class="o">-</span><span class="n">suffix</span> <span class="n">ELASTICSEARCH_INDEX_SUFFIX</span>
|
||||
<span class="n">append</span> <span class="n">this</span> <span class="n">suffix</span> <span class="n">to</span> <span class="n">the</span> <span class="n">dmarc_aggregate</span> <span class="ow">and</span>
|
||||
<span class="n">dmarc_forensic</span> <span class="n">Elasticsearch</span> <span class="n">index</span> <span class="n">names</span><span class="p">,</span> <span class="n">joined</span> <span class="n">by</span> <span class="n">_</span>
|
||||
<span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">use</span><span class="o">-</span><span class="n">ssl</span>
|
||||
<span class="n">Use</span> <span class="n">SSL</span> <span class="n">when</span> <span class="n">connecting</span> <span class="n">to</span> <span class="n">Elasticsearch</span>
|
||||
<span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">ssl</span><span class="o">-</span><span class="n">cert</span><span class="o">-</span><span class="n">path</span> <span class="n">ELASTICSEARCH_SSL_CERT_PATH</span>
|
||||
<span class="n">Path</span> <span class="n">to</span> <span class="n">the</span> <span class="n">Elasticsearch</span> <span class="n">SSL</span> <span class="n">certificate</span>
|
||||
<span class="o">--</span><span class="n">elasticsearch</span><span class="o">-</span><span class="n">monthly</span><span class="o">-</span><span class="n">indexes</span>
|
||||
<span class="n">Use</span> <span class="n">monthly</span> <span class="n">Elasticsearch</span> <span class="n">indexes</span> <span class="n">instead</span> <span class="n">of</span> <span class="n">daily</span>
|
||||
<span class="n">indexes</span>
|
||||
<span class="o">--</span><span class="n">hec</span> <span class="n">HEC</span> <span class="n">the</span> <span class="n">URL</span> <span class="n">to</span> <span class="n">a</span> <span class="n">Splunk</span> <span class="n">HTTP</span> <span class="n">Event</span> <span class="n">Collector</span> <span class="p">(</span><span class="n">HEC</span><span class="p">)</span>
|
||||
<span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">token</span> <span class="n">HEC_TOKEN</span>
|
||||
<span class="n">the</span> <span class="n">authorization</span> <span class="n">token</span> <span class="k">for</span> <span class="n">a</span> <span class="n">Splunk</span> <span class="n">HTTP</span> <span class="n">Event</span>
|
||||
@@ -317,7 +327,7 @@ lookalike domain monitoring, check out <a class="reference external" href="https
|
||||
<span class="o">--</span><span class="n">hec</span><span class="o">-</span><span class="n">skip</span><span class="o">-</span><span class="n">certificate</span><span class="o">-</span><span class="n">verification</span>
|
||||
<span class="n">skip</span> <span class="n">certificate</span> <span class="n">verification</span> <span class="k">for</span> <span class="n">Splunk</span> <span class="n">HEC</span>
|
||||
<span class="o">-</span><span class="n">K</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]],</span> <span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">hosts</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="p">[</span><span class="n">KAFKA_HOSTS</span> <span class="o">...</span><span class="p">]]</span>
|
||||
<span class="n">s</span> <span class="nb">list</span> <span class="n">of</span> <span class="n">one</span> <span class="ow">or</span> <span class="n">more</span> <span class="n">Kafka</span> <span class="n">hostnames</span>
|
||||
<span class="n">a</span> <span class="nb">list</span> <span class="n">of</span> <span class="n">one</span> <span class="ow">or</span> <span class="n">more</span> <span class="n">Kafka</span> <span class="n">hostnames</span>
|
||||
<span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">username</span> <span class="n">KAFKA_USERNAME</span>
|
||||
<span class="n">an</span> <span class="n">optional</span> <span class="n">Kafka</span> <span class="n">username</span>
|
||||
<span class="o">--</span><span class="n">kafka</span><span class="o">-</span><span class="n">password</span> <span class="n">KAFKA_PASSWORD</span>
|
||||
@@ -358,6 +368,7 @@ lookalike domain monitoring, check out <a class="reference external" href="https
|
||||
<span class="o">--</span><span class="n">test</span> <span class="n">do</span> <span class="ow">not</span> <span class="n">move</span> <span class="ow">or</span> <span class="n">delete</span> <span class="n">IMAP</span> <span class="n">messages</span>
|
||||
<span class="o">-</span><span class="n">s</span><span class="p">,</span> <span class="o">--</span><span class="n">silent</span> <span class="n">only</span> <span class="nb">print</span> <span class="n">errors</span> <span class="ow">and</span> <span class="n">warnings</span>
|
||||
<span class="o">--</span><span class="n">debug</span> <span class="nb">print</span> <span class="n">debugging</span> <span class="n">information</span>
|
||||
<span class="o">--</span><span class="n">log</span><span class="o">-</span><span class="n">file</span> <span class="n">LOG_FILE</span> <span class="n">output</span> <span class="n">logging</span> <span class="n">to</span> <span class="n">a</span> <span class="n">file</span>
|
||||
<span class="o">-</span><span class="n">v</span><span class="p">,</span> <span class="o">--</span><span class="n">version</span> <span class="n">show</span> <span class="n">program</span><span class="s1">'s version number and exit</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
@@ -615,10 +626,15 @@ above commands.</p>
|
||||
<div class="section" id="installation-using-pypy3">
|
||||
<h3>Installation using pypy3<a class="headerlink" href="#installation-using-pypy3" title="Permalink to this headline">¶</a></h3>
|
||||
<p>For the best possible processing speed, consider using <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> inside a <code class="docutils literal notranslate"><span class="pre">pypy3</span></code>
|
||||
virtualenv. First, <a class="reference external" href="https://pypy.org/download.html#default-with-a-jit-compiler">download the latest version of pypy3</a>. Extract it to
|
||||
virtualenv. First, <a class="reference external" href="https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux">download the latest portable Linux version of pypy3</a>. Extract it to
|
||||
<code class="docutils literal notranslate"><span class="pre">/opt/pypy3</span></code> (<code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">mkdir</span> <span class="pre">/opt</span></code> if <code class="docutils literal notranslate"><span class="pre">/opt</span></code> does not exist), then create a
|
||||
symlink:</p>
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3
|
||||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
tar -jxf pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
rm pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2
|
||||
sudo chown -R root:root pypy3.5-6.0.0-linux_x86_64-portable
|
||||
sudo mv pypy3.5-6.0.0-linux_x86_64-portable /opt/pypy3
|
||||
sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3
|
||||
</pre></div>
|
||||
</div>
|
||||
<p>Install <code class="docutils literal notranslate"><span class="pre">virtualenv</span></code> on your system:</p>
|
||||
@@ -1673,7 +1689,7 @@ to a callback function</p>
|
||||
|
||||
<dl class="function">
|
||||
<dt id="parsedmarc.elastic.save_aggregate_report_to_elasticsearch">
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_aggregate_report_to_elasticsearch</code><span class="sig-paren">(</span><em>aggregate_report</em>, <em>index_suffix=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_aggregate_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_aggregate_report_to_elasticsearch" title="Permalink to this definition">¶</a></dt>
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_aggregate_report_to_elasticsearch</code><span class="sig-paren">(</span><em>aggregate_report</em>, <em>index_suffix=None</em>, <em>monthly_indexes=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_aggregate_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_aggregate_report_to_elasticsearch" title="Permalink to this definition">¶</a></dt>
|
||||
<dd><p>Saves a parsed DMARC aggregate report to ElasticSearch</p>
|
||||
<table class="docutils field-list" frame="void" rules="none">
|
||||
<col class="field-name" />
|
||||
@@ -1682,6 +1698,7 @@ to a callback function</p>
|
||||
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
|
||||
<li><strong>aggregate_report</strong> (<em>OrderedDict</em>) – A parsed forensic report</li>
|
||||
<li><strong>index_suffix</strong> (<em>str</em>) – The suffix of the name of the index to save to</li>
|
||||
<li><strong>monthly_indexes</strong> (<em>bool</em>) – Use monthly indexes instead of daily indexes</li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -1694,7 +1711,7 @@ to a callback function</p>
|
||||
|
||||
<dl class="function">
|
||||
<dt id="parsedmarc.elastic.save_forensic_report_to_elasticsearch">
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_forensic_report_to_elasticsearch</code><span class="sig-paren">(</span><em>forensic_report</em>, <em>index_suffix=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_forensic_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_forensic_report_to_elasticsearch" title="Permalink to this definition">¶</a></dt>
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">save_forensic_report_to_elasticsearch</code><span class="sig-paren">(</span><em>forensic_report</em>, <em>index_suffix=None</em>, <em>monthly_indexes=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#save_forensic_report_to_elasticsearch"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.save_forensic_report_to_elasticsearch" title="Permalink to this definition">¶</a></dt>
|
||||
<dd><p>Saves a parsed DMARC forensic report to ElasticSearch</p>
|
||||
<table class="docutils field-list" frame="void" rules="none">
|
||||
<col class="field-name" />
|
||||
@@ -1703,6 +1720,8 @@ to a callback function</p>
|
||||
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
|
||||
<li><strong>forensic_report</strong> (<em>OrderedDict</em>) – A parsed forensic report</li>
|
||||
<li><strong>index_suffix</strong> (<em>str</em>) – The suffix of the name of the index to save to</li>
|
||||
<li><strong>monthly_indexes</strong> (<em>bool</em>) – Use monthly indexes instead of daily
|
||||
indexes</li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
@@ -1715,13 +1734,18 @@ to a callback function</p>
|
||||
|
||||
<dl class="function">
|
||||
<dt id="parsedmarc.elastic.set_hosts">
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">set_hosts</code><span class="sig-paren">(</span><em>hosts</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#set_hosts"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.set_hosts" title="Permalink to this definition">¶</a></dt>
|
||||
<code class="descclassname">parsedmarc.elastic.</code><code class="descname">set_hosts</code><span class="sig-paren">(</span><em>hosts</em>, <em>use_ssl=False</em>, <em>ssl_cert_path=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/parsedmarc/elastic.html#set_hosts"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#parsedmarc.elastic.set_hosts" title="Permalink to this definition">¶</a></dt>
|
||||
<dd><p>Sets the Elasticsearch hosts to use</p>
|
||||
<table class="docutils field-list" frame="void" rules="none">
|
||||
<col class="field-name" />
|
||||
<col class="field-body" />
|
||||
<tbody valign="top">
|
||||
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hosts</strong> – A single hostname or URL, or list of hostnames or URLs</td>
|
||||
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
|
||||
<li><strong>hosts</strong> (<em>str</em>) – A single hostname or URL, or list of hostnames or URLs</li>
|
||||
<li><strong>use_ssl</strong> (<em>bool</em>) – Use a HTTPS connection to the server</li>
|
||||
<li><strong>ssl_cert_path</strong> (<em>str</em>) – Path to the certificate chain</li>
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
BIN
objects.inv
BIN
objects.inv
Binary file not shown.
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Python Module Index — parsedmarc 5.1.3 documentation</title>
|
||||
<title>Python Module Index — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Search — parsedmarc 5.1.3 documentation</title>
|
||||
<title>Search — parsedmarc 5.2.0 documentation</title>
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@
|
||||
|
||||
|
||||
<div class="version">
|
||||
5.1.3
|
||||
5.2.0
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user