amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-05-15 16:35:34 +00:00
Code Issues Projects Releases Wiki Activity

6.3.0

Browse Source
This commit is contained in:
Sean Whalen
2019-03-29 16:44:39 -04:00
parent 89dc4e0f1f
commit e7376c7e34
16 changed files with 860 additions and 880 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_modules/index.html
+2 -2
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Overview: module code &mdash; parsedmarc 6.2.2 documentation</title>
<title>Overview: module code &mdash; parsedmarc 6.3.0 documentation</title>
@@ -58,7 +58,7 @@
<div class="version">
6.2.2
6.3.0
</div>
_modules/parsedmarc.html
+9 -8
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc &mdash; parsedmarc 6.2.2 documentation</title>
<title>parsedmarc &mdash; parsedmarc 6.3.0 documentation</title>
@@ -58,7 +58,7 @@
<div class="version">
6.2.2
6.3.0
</div>
@@ -185,7 +185,7 @@
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">timestamp_to_human</span><span class="p">,</span> <span class="n">human_timestamp_to_datetime</span>
<span class="kn">from</span> <span class="nn">parsedmarc.utils</span> <span class="k">import</span> <span class="n">parse_email</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;6.2.2&quot;</span>
<span class="n">__version__</span> <span class="o">=</span> <span class="s2">&quot;6.3.0&quot;</span>
<span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span>
<span class="nb">format</span><span class="o">=</span><span class="s1">&#39;</span><span class="si">%(levelname)8s</span><span class="s1">:</span><span class="si">%(filename)s</span><span class="s1">:</span><span class="si">%(lineno)d</span><span class="s1">:&#39;</span>
@@ -196,8 +196,8 @@
<span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;parsedmarc v</span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">__version__</span><span class="p">))</span>
<span class="n">feedback_report_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^([\w\-]+): (.+)$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^&lt;\?xml .*$&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;&lt;\/?xs:schema.&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_header_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^&lt;\?xml .*?&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">xml_schema_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;&lt;/??xs:schema.*&gt;&quot;</span><span class="p">,</span> <span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
<span class="n">MAGIC_ZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x50\x4B\x03\x04</span><span class="s2">&quot;</span>
<span class="n">MAGIC_GZIP</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\x1F\x8B</span><span class="s2">&quot;</span>
@@ -378,10 +378,10 @@
<span class="k">try</span><span class="p">:</span>
<span class="c1"># Replace XML header (sometimes they are invalid)</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_header_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_header_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">&quot;&lt;?xml version=</span><span class="se">\&quot;</span><span class="s2">1.0</span><span class="se">\&quot;</span><span class="s2">?&gt;&quot;</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
<span class="c1"># Remove invalid schema tags</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_schema_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">&#39;&lt;?xml version=&quot;1.0&quot;?&gt;&#39;</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
<span class="n">xml</span> <span class="o">=</span> <span class="n">xml_schema_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s1">&#39;&#39;</span><span class="p">,</span> <span class="n">xml</span><span class="p">)</span>
<span class="n">report</span> <span class="o">=</span> <span class="n">xmltodict</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">xml</span><span class="p">)[</span><span class="s2">&quot;feedback&quot;</span><span class="p">]</span>
<span class="n">report_metadata</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]</span>
@@ -1822,7 +1822,8 @@
<span class="n">server</span><span class="o">.</span><span class="n">idle</span><span class="p">()</span>
<span class="n">idle_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">monotonic</span><span class="p">()</span>
<span class="k">for</span> <span class="n">response</span> <span class="ow">in</span> <span class="n">responses</span><span class="p">:</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">&#39;RECENT&#39;</span> <span class="ow">and</span> <span class="n">response</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Received response: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="p">))</span>
<span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">response</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">&#39;RECENT&#39;</span><span class="p">:</span>
<span class="n">server</span><span class="o">.</span><span class="n">idle_done</span><span class="p">()</span>
<span class="n">res</span> <span class="o">=</span> <span class="n">get_dmarc_reports_from_inbox</span><span class="p">(</span><span class="n">connection</span><span class="o">=</span><span class="n">server</span><span class="p">,</span>
<span class="n">move_supported</span><span class="o">=</span><span class="n">ms</span><span class="p">,</span>
_modules/parsedmarc/elastic.html
+2 -2
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.elastic &mdash; parsedmarc 6.2.2 documentation</title>
<title>parsedmarc.elastic &mdash; parsedmarc 6.3.0 documentation</title>
@@ -58,7 +58,7 @@
<div class="version">
6.2.2
6.3.0
</div>
_modules/parsedmarc/splunk.html
+2 -2
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.splunk &mdash; parsedmarc 6.2.2 documentation</title>
<title>parsedmarc.splunk &mdash; parsedmarc 6.3.0 documentation</title>
@@ -58,7 +58,7 @@
<div class="version">
6.2.2
6.3.0
</div>
_modules/parsedmarc/utils.html
+7 -4
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>parsedmarc.utils &mdash; parsedmarc 6.2.2 documentation</title>
<title>parsedmarc.utils &mdash; parsedmarc 6.3.0 documentation</title>
@@ -58,7 +58,7 @@
<div class="version">
6.2.2
6.3.0
</div>
@@ -180,8 +180,10 @@
<span class="n">platform</span><span class="o">.</span><span class="n">release</span><span class="p">(),</span>
<span class="p">)</span>
<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;parsedmarc&quot;</span><span class="p">)</span>
<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;mailparser&quot;</span><span class="p">)</span>
<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
<span class="n">tempdir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
@@ -590,7 +592,8 @@
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;sample.msg&quot;</span><span class="p">,</span> <span class="s2">&quot;wb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
<span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">&quot;msgconvert&quot;</span><span class="p">,</span> <span class="s2">&quot;sample.msg&quot;</span><span class="p">])</span>
<span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">([</span><span class="s2">&quot;msgconvert&quot;</span><span class="p">,</span> <span class="s2">&quot;sample.msg&quot;</span><span class="p">],</span>
<span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span><span class="p">)</span>
<span class="n">eml_path</span> <span class="o">=</span> <span class="s2">&quot;sample.eml&quot;</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
<span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
_sources/index.rst.txt
+2
View File
@@ -175,12 +175,14 @@ The full set of configuration options are:
- ``user`` - str: The Kafka user
- ``passsword`` - str: The Kafka password
- ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True)
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``aggregate_topic`` - str: The Kafka topic for aggregate reports
- ``forensic_topic`` - str: The Kafka topic for forensic reports
- ``smtp``
- ``host`` - str: The SMTP hostname
- ``port`` - int: The SMTP port (Default: 25)
- ``ssl`` - bool: Require SSL/TLS instead of using STARTTLS
- ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended)
- ``user`` - str: the SMTP username
- ``password`` - str: the SMTP password
- ``from`` - str: The From header to use in the email
_static/basic.css
+73 -1
View File
@@ -231,6 +231,16 @@ a.headerlink {
visibility: hidden;
}
a.brackets:before,
span.brackets > a:before{
content: "[";
}
a.brackets:after,
span.brackets > a:after {
content: "]";
}
h1:hover > a.headerlink,
h2:hover > a.headerlink,
h3:hover > a.headerlink,
@@ -391,6 +401,16 @@ table.citation td {
border-bottom: none;
}
th > p:first-child,
td > p:first-child {
margin-top: 0px;
}
th > p:last-child,
td > p:last-child {
margin-bottom: 0px;
}
/* -- figures --------------------------------------------------------------- */
div.figure {
@@ -460,11 +480,57 @@ ol.upperroman {
list-style: upper-roman;
}
li > p:first-child {
margin-top: 0px;
}
li > p:last-child {
margin-bottom: 0px;
}
dl.footnote > dt,
dl.citation > dt {
float: left;
}
dl.footnote > dd,
dl.citation > dd {
margin-bottom: 0em;
}
dl.footnote > dd:after,
dl.citation > dd:after {
content: "";
clear: both;
}
dl.field-list {
display: flex;
flex-wrap: wrap;
}
dl.field-list > dt {
flex-basis: 20%;
font-weight: bold;
word-break: break-word;
}
dl.field-list > dt:after {
content: ":";
}
dl.field-list > dd {
flex-basis: 70%;
padding-left: 1em;
margin-left: 0em;
margin-bottom: 0em;
}
dl {
margin-bottom: 15px;
}
dd p {
dd > p:first-child {
margin-top: 0px;
}
@@ -537,6 +603,12 @@ dl.glossary dt {
font-style: oblique;
}
.classifier:before {
font-style: normal;
margin: 0.5em;
content: ":";
}
abbr, acronym {
border-bottom: dotted 1px;
cursor: help;
_static/doctools.js
+2 -3
View File
@@ -87,14 +87,13 @@ jQuery.fn.highlightText = function(text, className) {
node.nextSibling));
node.nodeValue = val.substr(0, pos);
if (isInSVG) {
var bbox = span.getBBox();
var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
rect.x.baseVal.value = bbox.x;
var bbox = node.parentElement.getBBox();
rect.x.baseVal.value = bbox.x;
rect.y.baseVal.value = bbox.y;
rect.width.baseVal.value = bbox.width;
rect.height.baseVal.value = bbox.height;
rect.setAttribute('class', className);
var parentOfText = node.parentNode.parentNode;
addItems.push({
"parent": node.parentNode,
"target": rect});
_static/documentation_options.js
+2 -2
View File
@@ -1,10 +1,10 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
VERSION: '6.2.2',
VERSION: '6.3.0',
LANGUAGE: 'None',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt',
NAVIGATION_WITH_KEYS: false,
NAVIGATION_WITH_KEYS: false
};
_static/searchtools.js
+34 -20
View File
@@ -36,8 +36,10 @@ if (!Scorer) {
// query found in title
title: 15,
partialTitle: 7,
// query found in terms
term: 5
term: 5,
partialTerm: 2
};
}
@@ -56,6 +58,14 @@ var Search = {
_queued_query : null,
_pulse_status : -1,
htmlToText : function(htmlString) {
var htmlElement = document.createElement('span');
htmlElement.innerHTML = htmlString;
$(htmlElement).find('.headerlink').remove();
docContent = $(htmlElement).find('[role=main]')[0];
return docContent.textContent || docContent.innerText;
},
init : function() {
var params = $.getQueryParameters();
if (params.q) {
@@ -65,16 +75,6 @@ var Search = {
}
},
loadIndex : function(url) {
$.ajax({type: "GET", url: url, data: null,
dataType: "script", cache: true,
complete: function(jqxhr, textstatus) {
if (textstatus != "success") {
document.getElementById("searchindexloader").src = url;
}
}});
},
setIndex : function(index) {
var q;
this._index = index;
@@ -120,7 +120,7 @@ var Search = {
this.out = $('#search-results');
this.title = $('<h2>' + _('Searching') + '</h2>').appendTo(this.out);
this.dots = $('<span></span>').appendTo(this.title);
this.status = $('<p style="display: none"></p>').appendTo(this.out);
this.status = $('<p class="search-summary">&nbsp;</p>').appendTo(this.out);
this.output = $('<ul class="search"/>').appendTo(this.out);
$('#search-progress').text(_('Preparing search...'));
@@ -259,11 +259,7 @@ var Search = {
displayNextItem();
});
} else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) {
var suffix = DOCUMENTATION_OPTIONS.SOURCELINK_SUFFIX;
if (suffix === undefined) {
suffix = '.txt';
}
$.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[5] + (item[5].slice(-suffix.length) === suffix ? '' : suffix),
$.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + item[0] + DOCUMENTATION_OPTIONS.FILE_SUFFIX,
dataType: "text",
complete: function(jqxhr, textstatus) {
var data = jqxhr.responseText;
@@ -385,6 +381,19 @@ var Search = {
{files: terms[word], score: Scorer.term},
{files: titleterms[word], score: Scorer.title}
];
// add support for partial matches
if (word.length > 2) {
for (var w in terms) {
if (w.match(word) && !terms[word]) {
_o.push({files: terms[w], score: Scorer.partialTerm})
}
}
for (var w in titleterms) {
if (w.match(word) && !titleterms[word]) {
_o.push({files: titleterms[w], score: Scorer.partialTitle})
}
}
}
// no match but word was a required one
if ($u.every(_o, function(o){return o.files === undefined;})) {
@@ -424,8 +433,12 @@ var Search = {
var valid = true;
// check if all requirements are matched
if (fileMap[file].length != searchterms.length)
continue;
var filteredTermCount = // as search terms with length < 3 are discarded: ignore
searchterms.filter(function(term){return term.length > 2}).length
if (
fileMap[file].length != searchterms.length &&
fileMap[file].length != filteredTermCount
) continue;
// ensure that none of the excluded terms is in the search result
for (i = 0; i < excluded.length; i++) {
@@ -456,7 +469,8 @@ var Search = {
* words. the first one is used to find the occurrence, the
* latter for highlighting it.
*/
makeSearchSummary : function(text, keywords, hlwords) {
makeSearchSummary : function(htmlText, keywords, hlwords) {
var text = Search.htmlToText(htmlText);
var textLower = text.toLowerCase();
var start = 0;
$.each(keywords, function() {
genindex.html
+2 -2
View File
@@ -9,7 +9,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Index &mdash; parsedmarc 6.2.2 documentation</title>
<title>Index &mdash; parsedmarc 6.3.0 documentation</title>
@@ -59,7 +59,7 @@
<div class="version">
6.2.2
6.3.0
</div>
index.html
+709 -826
View File
File diff suppressed because it is too large Load Diff
objects.inv
BIN
View File
Binary file not shown.
py-modindex.html
+11 -5
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Python Module Index &mdash; parsedmarc 6.2.2 documentation</title>
<title>Python Module Index &mdash; parsedmarc 6.3.0 documentation</title>
@@ -61,7 +61,7 @@
<div class="version">
6.2.2
6.3.0
</div>
@@ -162,17 +162,23 @@
<td>
<a href="index.html#module-parsedmarc"><code class="xref">parsedmarc</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<tr>
<td><img src="_static/minus.png" class="toggler"
id="toggle-2" style="display: none" alt="-" /></td>
<td>
<code class="xref">parsedmarc</code></td><td>
<em></em></td></tr>
<tr class="cg-2">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.elastic"><code class="xref">parsedmarc.elastic</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<tr class="cg-2">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.splunk"><code class="xref">parsedmarc.splunk</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<tr class="cg-2">
<td></td>
<td>&#160;&#160;&#160;
<a href="index.html#module-parsedmarc.utils"><code class="xref">parsedmarc.utils</code></a></td><td>
search.html
+2 -2
View File
@@ -8,7 +8,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Search &mdash; parsedmarc 6.2.2 documentation</title>
<title>Search &mdash; parsedmarc 6.3.0 documentation</title>
@@ -59,7 +59,7 @@
<div class="version">
6.2.2
6.3.0
</div>
searchindex.js
+1 -1
View File
File diff suppressed because one or more lines are too long