mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-08-31 11:17:15 +00:00
Update docs
This commit is contained in:
Sean Whalen
+5
-5
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Overview: module code — parsedmarc 10.2.0 documentation</title>
|
||||
<title>Overview: module code — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../genindex.html" />
|
||||
<link rel="search" title="Search" href="../search.html" />
|
||||
@@ -864,7 +864,7 @@
|
||||
<div class="viewcode-block" id="parse_aggregate_report_xml">
|
||||
<a class="viewcode-back" href="../api.html#parsedmarc.parse_aggregate_report_xml">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">parse_aggregate_report_xml</span><span class="p">(</span>
|
||||
<span class="n">xml</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">xml</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span>
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">ip_db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
@@ -880,7 +880,8 @@
|
||||
<span class="w"> </span><span class="sd">"""Parses a DMARC XML report string and returns a consistent dict</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> xml (str): A string of DMARC aggregate report XML</span>
|
||||
<span class="sd"> xml (str | bytes): DMARC aggregate report XML (bytes are decoded</span>
|
||||
<span class="sd"> with errors ignored)</span>
|
||||
<span class="sd"> ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP</span>
|
||||
<span class="sd"> always_use_local_files (bool): Do not download files</span>
|
||||
<span class="sd"> reverse_dns_map_path (str): Path to a reverse DNS map file</span>
|
||||
@@ -1223,9 +1224,6 @@
|
||||
<span class="n">remainder</span> <span class="o">=</span> <span class="n">stream</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
|
||||
<span class="n">file_object</span> <span class="o">=</span> <span class="n">BytesIO</span><span class="p">(</span><span class="n">header</span> <span class="o">+</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">remainder</span><span class="p">))</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">file_object</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="n">ParserError</span><span class="p">(</span><span class="s2">"Invalid report content"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">header</span><span class="p">[:</span> <span class="nb">len</span><span class="p">(</span><span class="n">MAGIC_ZIP</span><span class="p">)]</span> <span class="o">==</span> <span class="n">MAGIC_ZIP</span><span class="p">:</span>
|
||||
<span class="n">_zip</span> <span class="o">=</span> <span class="n">zipfile</span><span class="o">.</span><span class="n">ZipFile</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
|
||||
<span class="n">report</span> <span class="o">=</span> <span class="n">_zip</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">_zip</span><span class="o">.</span><span class="n">namelist</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">errors</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.elastic — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc.elastic — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -375,27 +375,6 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="c1"># pyright: ignore[reportCallIssue]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
|
||||
@@ -505,12 +484,12 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; no migrations are</span>
|
||||
<span class="sd"> currently needed for failure indexes)</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -538,10 +517,7 @@
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span> <span class="c1"># pyright: ignore[reportArgumentType]</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -633,8 +609,12 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"normalized_timespan"</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -762,7 +742,12 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="c1"># arrival_date_utc is a UTC wall-clock string; without assume_utc the</span>
|
||||
<span class="c1"># naive .timestamp() below would interpret it as local time and skew</span>
|
||||
<span class="c1"># the epoch by the host's UTC offset.</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">],</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc.opensearch — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -366,27 +366,6 @@
|
||||
<span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
|
||||
<span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
|
||||
<span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
|
||||
<span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
|
||||
<span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
|
||||
<span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
|
||||
<span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
|
||||
<span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
|
||||
<span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AlreadySaved">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
|
||||
@@ -505,12 +484,12 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> aggregate_indexes (list): A list of aggregate index names</span>
|
||||
<span class="sd"> failure_indexes (list): A list of failure index names</span>
|
||||
<span class="sd"> (accepted for API compatibility; no migrations are</span>
|
||||
<span class="sd"> currently needed for failure indexes)</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">if</span> <span class="n">failure_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">failure_indexes</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||
<span class="k">continue</span>
|
||||
@@ -538,10 +517,7 @@
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
|
||||
<span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">failure_index</span> <span class="ow">in</span> <span class="n">failure_indexes</span><span class="p">:</span>
|
||||
<span class="k">pass</span></div>
|
||||
<span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span></div>
|
||||
|
||||
|
||||
|
||||
@@ -633,8 +609,12 @@
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">"records"</span><span class="p">]:</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">record</span><span class="p">[</span><span class="s2">"interval_end"</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"normalized_timespan"</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
|
||||
@@ -762,7 +742,12 @@
|
||||
<span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
|
||||
<span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
|
||||
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="c1"># arrival_date_utc is a UTC wall-clock string; without assume_utc the</span>
|
||||
<span class="c1"># naive .timestamp() below would interpret it as local time and skew</span>
|
||||
<span class="c1"># the epoch by the host's UTC offset.</span>
|
||||
<span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">failure_report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">],</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.splunk — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc.splunk — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -211,8 +211,10 @@
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"spf_results"</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">"auth_results"</span><span class="p">][</span><span class="s2">"spf"</span><span class="p">]</span>
|
||||
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:aggregate"</span>
|
||||
<span class="c1"># interval_begin is a UTC wall-clock string; assume_utc keeps</span>
|
||||
<span class="c1"># it from being re-interpreted as local time on non-UTC hosts.</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">]</span>
|
||||
<span class="n">new_report</span><span class="p">[</span><span class="s2">"interval_begin"</span><span class="p">],</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
@@ -255,7 +257,11 @@
|
||||
<span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">failure_reports</span><span class="p">:</span>
|
||||
<span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"sourcetype"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"dmarc:failure"</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">])</span>
|
||||
<span class="c1"># arrival_date_utc is a UTC wall-clock string; assume_utc keeps</span>
|
||||
<span class="c1"># it from being re-interpreted as local time on non-UTC hosts.</span>
|
||||
<span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">report</span><span class="p">[</span><span class="s2">"arrival_date_utc"</span><span class="p">],</span> <span class="n">assume_utc</span><span class="o">=</span><span class="kc">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"time"</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
|
||||
<span class="n">data</span><span class="p">[</span><span class="s2">"event"</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||||
<span class="n">json_str</span> <span class="o">+=</span> <span class="s2">"</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.types — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc.types — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
|
||||
@@ -5,16 +5,16 @@
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>parsedmarc.utils — parsedmarc 10.2.0 documentation</title>
|
||||
<title>parsedmarc.utils — parsedmarc 10.2.2 documentation</title>
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=e59714d7" />
|
||||
<link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
|
||||
|
||||
|
||||
<script src="../../_static/jquery.js?v=5d32c60e"></script>
|
||||
<script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=d0f31c56"></script>
|
||||
<script src="../../_static/doctools.js?v=9bcbadda"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../_static/documentation_options.js?v=43731225"></script>
|
||||
<script src="../../_static/doctools.js?v=fd6eb6e6"></script>
|
||||
<script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||||
<script src="../../_static/js/theme.js"></script>
|
||||
<link rel="index" title="Index" href="../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../search.html" />
|
||||
@@ -104,11 +104,7 @@
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
||||
<span class="c1"># Try backported to PY<3 `importlib_resources`</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
|
||||
|
||||
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">dns.exception</span>
|
||||
@@ -476,7 +472,7 @@
|
||||
<div class="viewcode-block" id="human_timestamp_to_datetime">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_datetime</span><span class="p">(</span>
|
||||
<span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">to_utc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">to_utc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span> <span class="n">assume_utc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">datetime</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a Python ``datetime`` object</span>
|
||||
@@ -484,6 +480,11 @@
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> human_timestamp (str): A timestamp string</span>
|
||||
<span class="sd"> to_utc (bool): Convert the timestamp to UTC</span>
|
||||
<span class="sd"> assume_utc (bool): Treat a timestamp that carries no UTC offset as</span>
|
||||
<span class="sd"> UTC wall-clock time instead of local time. Pass this when the</span>
|
||||
<span class="sd"> string is known to be UTC (e.g. an ``arrival_date_utc`` value);</span>
|
||||
<span class="sd"> otherwise naive results are interpreted as local time by</span>
|
||||
<span class="sd"> ``datetime.astimezone()`` / ``datetime.timestamp()``.</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> datetime: The converted timestamp</span>
|
||||
@@ -493,29 +494,41 @@
|
||||
<span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">parenthesis_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">""</span><span class="p">,</span> <span class="n">human_timestamp</span><span class="p">)</span>
|
||||
|
||||
<span class="n">dt</span> <span class="o">=</span> <span class="n">parse_date</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">assume_utc</span> <span class="ow">and</span> <span class="n">dt</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">dt</span> <span class="o">=</span> <span class="n">dt</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">tzinfo</span><span class="o">=</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
|
||||
<span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">assume_utc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Converts a human-readable timestamp into a UNIX timestamp</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
|
||||
<span class="sd"> assume_utc (bool): Treat a timestamp that carries no UTC offset as</span>
|
||||
<span class="sd"> UTC wall-clock time instead of local time</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> float: The converted timestamp</span>
|
||||
<span class="sd"> int: The converted timestamp</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">human_timestamp</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span></div>
|
||||
<span class="k">return</span> <span class="nb">int</span><span class="p">(</span>
|
||||
<span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">,</span> <span class="n">assume_utc</span><span class="o">=</span><span class="n">assume_utc</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
<span class="n">_IP_DB_PATH</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># The last database path logged by _get_ip_database_path(), so the</span>
|
||||
<span class="c1"># selection is logged when it changes rather than on every lookup.</span>
|
||||
<span class="n">_LAST_LOGGED_IP_DB_PATH</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="load_ip_db">
|
||||
<a class="viewcode-back" href="../../api.html#parsedmarc.utils.load_ip_db">[docs]</a>
|
||||
@@ -639,12 +652,11 @@
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">_IPINFO_API_TOKEN</span> <span class="ow">or</span> <span class="ow">not</span> <span class="n">probe</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">_ipinfo_api_lookup</span><span class="p">(</span><span class="s2">"1.1.1.1"</span><span class="p">)</span>
|
||||
<span class="k">except</span> <span class="n">InvalidIPinfoAPIKey</span><span class="p">:</span>
|
||||
<span class="k">raise</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">"IPinfo API probe failed (will fall back per-request): </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="c1"># _ipinfo_api_lookup() raises InvalidIPinfoAPIKey on 401/403 (which</span>
|
||||
<span class="c1"># must propagate) and returns None on any other failure — network</span>
|
||||
<span class="c1"># errors, non-2xx responses, malformed bodies.</span>
|
||||
<span class="k">if</span> <span class="n">_ipinfo_api_lookup</span><span class="p">(</span><span class="s2">"1.1.1.1"</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"IPinfo API probe failed (will fall back per-request)"</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"IPinfo API configured"</span><span class="p">)</span></div>
|
||||
|
||||
@@ -740,6 +752,12 @@
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_get_ip_database_path</span><span class="p">(</span><span class="n">db_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="c1"># Last-resort fallbacks for unusual installs where the bundled database</span>
|
||||
<span class="c1"># is missing. Country-only databases (GeoLite2 / DBIP) lack the ASN</span>
|
||||
<span class="c1"># fields source attribution depends on, so an incidental system GeoIP</span>
|
||||
<span class="c1"># file must never shadow the parsedmarc-managed database</span>
|
||||
<span class="c1"># (https://github.com/domainaware/parsedmarc/issues/810). To use</span>
|
||||
<span class="c1"># MaxMind or DBIP data deliberately, set the ip_db_path option.</span>
|
||||
<span class="n">db_paths</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"ipinfo_lite.mmdb"</span><span class="p">,</span>
|
||||
<span class="s2">"GeoLite2-Country.mmdb"</span><span class="p">,</span>
|
||||
@@ -763,19 +781,35 @@
|
||||
<span class="p">)</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">db_paths</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
|
||||
<span class="k">break</span>
|
||||
<span class="c1"># The database parsedmarc manages takes precedence: the one selected by</span>
|
||||
<span class="c1"># load_ip_db() (downloaded, cached, or bundled), or the bundled copy</span>
|
||||
<span class="c1"># directly for library callers that never call load_ip_db().</span>
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">_IP_DB_PATH</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">_IP_DB_PATH</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">_IP_DB_PATH</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">_IP_DB_PATH</span>
|
||||
<span class="n">bundled_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">ipinfo</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"ipinfo_lite.mmdb"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">bundled_path</span><span class="p">):</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">bundled_path</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
|
||||
<span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">ipinfo</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">"ipinfo_lite.mmdb"</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">db_paths</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
|
||||
<span class="k">break</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="c1"># Nothing found anywhere; use the bundled path so the</span>
|
||||
<span class="c1"># os.stat() below raises a FileNotFoundError naming the</span>
|
||||
<span class="c1"># expected install location.</span>
|
||||
<span class="n">db_path</span> <span class="o">=</span> <span class="n">bundled_path</span>
|
||||
|
||||
<span class="k">global</span> <span class="n">_LAST_LOGGED_IP_DB_PATH</span>
|
||||
<span class="k">if</span> <span class="n">db_path</span> <span class="o">!=</span> <span class="n">_LAST_LOGGED_IP_DB_PATH</span><span class="p">:</span>
|
||||
<span class="c1"># Log per selected path, not per lookup — this function runs on</span>
|
||||
<span class="c1"># every uncached IP lookup and would flood --debug output.</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Using IP database at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="n">_LAST_LOGGED_IP_DB_PATH</span> <span class="o">=</span> <span class="n">db_path</span>
|
||||
|
||||
<span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">db_age</span> <span class="o">></span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
|
||||
@@ -1278,10 +1312,11 @@
|
||||
<span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">"date_utc"</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">"T"</span><span class="p">,</span> <span class="s2">" "</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"from"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="s2">"From"</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"Headers"</span><span class="p">][</span><span class="s2">"From"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="c1"># mailparser omits "from" from mail_json when the From header is</span>
|
||||
<span class="c1"># present but unparseable (e.g. an empty "From:"); headers_json may</span>
|
||||
<span class="c1"># still carry a "From" entry, which can be an empty list — treat</span>
|
||||
<span class="c1"># that the same as a missing header.</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"headers"</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"From"</span><span class="p">)</span> <span class="ow">or</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">]</span> <span class="o">=</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">"from"</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>
|
||||
|
||||
Reference in New Issue
Block a user