From 5b85233807ee2598d8098f5b3e82841282ec3901 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 19 Nov 2018 08:43:39 -0500 Subject: [PATCH] 5.0.0 --- _modules/index.html | 16 +- _modules/parsedmarc.html | 94 +++++++-- _modules/parsedmarc/elastic.html | 105 +++++++--- _modules/parsedmarc/splunk.html | 16 +- _modules/parsedmarc/utils.html | 57 +++-- _sources/index.rst.txt | 34 ++- _static/basic.css | 11 + _static/doctools.js | 6 +- _static/documentation_options.js | 293 +++++++++++++++++++++++++- _static/searchtools.js | 347 +++---------------------------- genindex.html | 25 ++- index.html | 89 +++++--- objects.inv | Bin 742 -> 750 bytes py-modindex.html | 16 +- search.html | 16 +- searchindex.js | 2 +- 16 files changed, 635 insertions(+), 492 deletions(-) diff --git a/_modules/index.html b/_modules/index.html index d002f8c..1edd206 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -8,7 +8,7 @@ - Overview: module code — parsedmarc 4.4.1 documentation + Overview: module code — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
- 4.4.1 + 5.0.0
@@ -178,17 +178,7 @@ - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 086c610..83334dc 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -8,7 +8,7 @@ - parsedmarc — parsedmarc 4.4.1 documentation + parsedmarc — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
- 4.4.1 + 5.0.0
@@ -172,6 +172,7 @@ from ssl import SSLError, CertificateError, create_default_context import time +from expiringdict import ExpiringDict import xmltodict import imapclient import imapclient.exceptions @@ -182,7 +183,7 @@ from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "4.4.1" +__version__ = "5.0.0" logger = logging.getLogger("parsedmarc") logger.debug("parsedmarc v{0}".format(__version__)) @@ -195,6 +196,8 @@ MAGIC_GZIP = b"\x1F\x8B" MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20" +IP_ADDRESS_CACHE = ExpiringDict(max_len=10000, max_age_seconds=1800) +
[docs]class ParserError(RuntimeError): """Raised whenever the parser fails for some reason"""
@@ -238,9 +241,11 @@ nameservers = ["8.8.8.8", "4.4.4.4"] record = record.copy() new_record = OrderedDict() - new_record["source"] = get_ip_address_info(record["row"]["source_ip"], - nameservers=nameservers, - timeout=timeout) + new_record_source = get_ip_address_info(record["row"]["source_ip"], + cache=IP_ADDRESS_CACHE, + nameservers=nameservers, + timeout=timeout) + new_record["source"] = new_record_source new_record["count"] = int(record["row"]["count"]) policy_evaluated = record["row"]["policy_evaluated"].copy() new_policy_evaluated = OrderedDict([("disposition", "none"), @@ -667,7 +672,7 @@ parsed_report[key] = report_value[1] if "arrival_date" not in parsed_report: - parsed_report["arrival_date"] = msg_date + parsed_report["arrival_date"] = msg_date.isoformat() if "version" not in parsed_report: parsed_report["version"] = 1 @@ -681,9 +686,10 @@ parsed_report["arrival_date_utc"] = arrival_utc ip_address = parsed_report["source_ip"] - parsed_report["source"] = get_ip_address_info(ip_address, - nameservers=nameservers, - timeout=timeout) + parsed_report_source = get_ip_address_info(ip_address, + nameservers=nameservers, + timeout=timeout) + parsed_report["source"] = parsed_report_source del parsed_report["source_ip"] if "identity_alignment" not in parsed_report: @@ -803,6 +809,8 @@ try: if is_outlook_msg(input_): input_ = convert_outlook_msg(input_) + if type(input_) == bytes: + input_ = input_.decode(encoding="utf8") msg = mailparser.parse_from_string(input_) msg_headers = json.loads(msg.headers_json) date = email.utils.format_datetime(datetime.utcnow()) @@ -1608,6 +1616,24 @@ dns_timeout=dt) callback(res) server.idle() + except KeyError: + logger.debug("IMAP error: Server returned unexpected result") + logger.debug("Reconnecting watcher") + server = imapclient.IMAPClient(host) + server.login(username, password) + server.select_folder(rf) + idle_start_time = time.monotonic() + ms = "MOVE" in get_imap_capabilities(server) + res = get_dmarc_reports_from_inbox(connection=server, + move_supported=ms, + reports_folder=rf, + archive_folder=af, + delete=delete, + test=test, + nameservers=ns, + dns_timeout=dt) + callback(res) + server.idle() except ConnectionAbortedError: raise IMAPError("Connection aborted") except TimeoutError: @@ -1645,6 +1671,20 @@ idle_start_time = time.monotonic() responses = server.idle_check(timeout=wait) if responses is not None: + if len(responses) == 0: + # Gmail/G-Suite does not generate anything in the responses + server.idle_done() + res = get_dmarc_reports_from_inbox(connection=server, + move_supported=ms, + reports_folder=rf, + archive_folder=af, + delete=delete, + test=test, + nameservers=ns, + dns_timeout=dt) + callback(res) + server.idle() + idle_start_time = time.monotonic() for response in responses: if response[1] == b'RECENT' and response[0] > 0: server.idle_done() @@ -1667,7 +1707,7 @@ raise IMAPError("DNS resolution failed") except ConnectionRefusedError: raise IMAPError("Connection refused") - except ConnectionResetError: + except (KeyError, ConnectionResetError): logger.debug("IMAP error: Connection reset") logger.debug("Reconnecting watcher") server = imapclient.IMAPClient(host) @@ -1683,6 +1723,26 @@ test=test, nameservers=ns, dns_timeout=dt) + callback(res) + server.idle() + except KeyError: + logger.debug("IMAP error: Server returned unexpected result") + logger.debug("Reconnecting watcher") + server = imapclient.IMAPClient(host) + server.login(username, password) + server.select_folder(rf) + idle_start_time = time.monotonic() + ms = "MOVE" in get_imap_capabilities(server) + res = get_dmarc_reports_from_inbox(connection=server, + move_supported=ms, + reports_folder=rf, + archive_folder=af, + delete=delete, + test=test, + nameservers=ns, + dns_timeout=dt) + callback(res) + server.idle() except ConnectionAbortedError: raise IMAPError("Connection aborted") except TimeoutError: @@ -1749,17 +1809,7 @@ - + diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 553edc4..48be902 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -8,7 +8,7 @@ - parsedmarc.elastic — parsedmarc 4.4.1 documentation + parsedmarc.elastic — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
- 4.4.1 + 5.0.0
@@ -152,7 +152,9 @@ from elasticsearch_dsl.search import Q from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ - InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date + InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date, Search +from elasticsearch.helpers import reindex + from parsedmarc.utils import human_timestamp_to_datetime @@ -175,7 +177,7 @@ p = Text() sp = Text() pct = Integer() - fo = Integer() + fo = Text() # TODO: Change this to Text (issue #31) class _DKIMResult(InnerDoc): @@ -324,18 +326,15 @@ connections.create_connection(hosts=hosts, timeout=20) -
[docs]def create_indexes(names=None, settings=None): +
[docs]def create_indexes(names, settings=None): """ Create Elasticsearch indexes Args: names (list): A list of index names - ["dmarc_aggregate", "dmarc_forensic"] by default settings (dict): Index settings """ - if names is None: - names = ["dmarc_aggregate", "dmarc_forensic"] for name in names: index = Index(name) try: @@ -349,14 +348,65 @@ "Elasticsearch error: {0}".format(e.__str__()))
+
[docs]def migrate_indexes(aggregate_indexes=None, forensic_indexes=None): + """ + Updates index mappings + + Args: + aggregate_indexes (list): A list of aggregate index names + forensic_indexes (list): A list of forensic index names + """ + version = 2 + if aggregate_indexes is None: + aggregate_indexes = [] + if forensic_indexes is None: + forensic_indexes = [] + for aggregate_index_name in aggregate_indexes: + if not Index(aggregate_index_name).exists(): + continue + aggregate_index = Index(aggregate_index_name) + doc = "doc" + fo_field = "published_policy.fo" + fo = "fo" + fo_mapping = aggregate_index.get_field_mapping(fields=[fo_field]) + fo_mapping = fo_mapping[list(fo_mapping.keys())[0]]["mappings"] + if doc not in fo_mapping: + continue + + fo_mapping = fo_mapping[doc][fo_field]["mapping"][fo] + fo_type = fo_mapping["type"] + if fo_type == "long": + # TODO: Do reindex, delete, and alias here (issue #31) + new_index_name = "{0}-v{1}".format(aggregate_index_name, version) + body = {"properties": {"published_policy.fo": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + Index(new_index_name).create() + Index(new_index_name).put_mapping(doc_type=doc, body=body) + reindex(connections.get_connection(), aggregate_index_name, + new_index_name) + Index(aggregate_index_name).delete() + + for forensic_index in forensic_indexes: + pass
+ +
[docs]def save_aggregate_report_to_elasticsearch(aggregate_report, - index="dmarc_aggregate"): + index_suffix=None): """ Saves a parsed DMARC aggregate report to ElasticSearch Args: aggregate_report (OrderedDict): A parsed forensic report - index (str): The name of the index to save to + index_suffix (str): The suffix of the name of the index to save to Raises: AlreadySaved @@ -371,6 +421,7 @@ end_date = human_timestamp_to_datetime(metadata["end_date"]) begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%S") end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%S") + index_date = begin_date.strftime("%Y-%m-%d") aggregate_report["begin_date"] = begin_date aggregate_report["end_date"] = end_date date_range = (aggregate_report["begin_date"], @@ -378,11 +429,11 @@ org_name_query = Q(dict(match=dict(org_name=org_name))) report_id_query = Q(dict(match=dict(report_id=report_id))) - domain_query = Q(dict(match=dict(domain=domain))) + domain_query = Q(dict(match={"published_policy.domain": domain})) begin_date_query = Q(dict(match=dict(date_range=begin_date))) end_date_query = Q(dict(match=dict(date_range=end_date))) - search = Index(index).search() + search = Search(index="dmarc_aggregate*") search.query = org_name_query & report_id_query & domain_query & \ begin_date_query & end_date_query @@ -443,7 +494,13 @@ scope=spf_result["scope"], result=spf_result["result"]) + index = "dmarc_aggregate" + if index_suffix: + index = "{0}_{1}".format(index, index_suffix) + index = "{0}-{1}".format(index, index_date) + create_indexes([index]) agg_doc.meta.index = index + try: agg_doc.save() except Exception as e: @@ -452,13 +509,13 @@
[docs]def save_forensic_report_to_elasticsearch(forensic_report, - index="dmarc_forensic"): + index_suffix=None): """ Saves a parsed DMARC forensic report to ElasticSearch Args: forensic_report (OrderedDict): A parsed forensic report - index (str): The name of the index to save to + index_suffix (str): The suffix of the name of the index to save to Raises: AlreadySaved @@ -478,7 +535,7 @@ arrival_date_human = forensic_report["arrival_date_utc"] arrival_date = human_timestamp_to_datetime(arrival_date_human) - search = Index(index).search() + search = Search(index="dmarc_forensic*") arrival_query = {"match": {"arrival_date": arrival_date}} q = Q(arrival_query) @@ -559,6 +616,12 @@ sample=sample ) + index = "dmarc_forensic" + if index_suffix: + index = "{0}_{1}".format(index, index_suffix) + index_date = arrival_date.strftime("%Y-%m-%d") + index = "{0}-{1}".format(index, index_date) + create_indexes([index]) forensic_doc.meta.index = index try: forensic_doc.save() @@ -598,17 +661,7 @@ - + diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 0efc62b..b8715d7 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -8,7 +8,7 @@ - parsedmarc.splunk — parsedmarc 4.4.1 documentation + parsedmarc.splunk — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
- 4.4.1 + 5.0.0
@@ -331,17 +331,7 @@ - + diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index 41518c1..5ed92a2 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -8,7 +8,7 @@ - parsedmarc.utils — parsedmarc 4.4.1 documentation + parsedmarc.utils — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
- 4.4.1 + 5.0.0
@@ -251,13 +251,14 @@ return psl.get_public_suffix(domain)
-
[docs]def query_dns(domain, record_type, nameservers=None, timeout=2.0): +
[docs]def query_dns(domain, record_type, cache=None, nameservers=None, timeout=2.0): """ Queries DNS Args: domain (str): The domain or subdomain to query about record_type (str): The record type to query for + cache (ExpiringDict): Cache storage nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds @@ -265,6 +266,14 @@ Returns: list: A list of answers """ + domain = str(domain).lower() + record_type = record_type.upper() + cache_key = "{0}_{1}".format(domain, record_type) + if cache: + records = cache.get(cache_key, None) + if records: + return records + resolver = dns.resolver.Resolver() timeout = float(timeout) if nameservers is None: @@ -281,19 +290,24 @@ _resource_record = [ resource_record[0][:0].join(resource_record) for resource_record in resource_records if resource_record] - return [r.decode() for r in _resource_record] + records = [r.decode() for r in _resource_record] else: - return list(map( + records = list(map( lambda r: r.to_text().replace('"', '').rstrip("."), - resolver.query(domain, record_type, tcp=True)))
+ resolver.query(domain, record_type, tcp=True))) + if cache: + cache[cache_key] = records + + return records
-
[docs]def get_reverse_dns(ip_address, nameservers=None, timeout=2.0): +
[docs]def get_reverse_dns(ip_address, cache=None, nameservers=None, timeout=2.0): """ Resolves an IP address to a hostname using a reverse DNS query Args: ip_address (str): The IP address to resolve + cache (ExpiringDict): Cache storage nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS query timeout in seconds @@ -304,7 +318,7 @@ hostname = None try: address = dns.reversename.from_address(ip_address) - hostname = query_dns(address, "PTR", + hostname = query_dns(address, "PTR", cache=cache, nameservers=nameservers, timeout=timeout)[0] @@ -437,12 +451,13 @@ return country
-
[docs]def get_ip_address_info(ip_address, nameservers=None, timeout=2.0): +
[docs]def get_ip_address_info(ip_address, cache=None, nameservers=None, timeout=2.0): """ Returns reverse DNS and country information for the given IP address Args: ip_address (str): The IP address to check + cache (ExpiringDict): Cache storage nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds @@ -452,6 +467,10 @@ """ ip_address = ip_address.lower() + if cache: + info = cache.get(ip_address, None) + if info: + return info info = OrderedDict() info["ip_address"] = ip_address reverse_dns = get_reverse_dns(ip_address, @@ -573,11 +592,15 @@ headers = json.loads(parsed_email.headers_json).copy() parsed_email = json.loads(parsed_email.mail_json).copy() parsed_email["headers"] = headers + if "received" in parsed_email: for received in parsed_email["received"]: if "date_utc" in received: - received["date_utc"] = received["date_utc"].replace("T", - " ") + if received["date_utc"] is None: + del received["date_utc"] + else: + received["date_utc"] = received["date_utc"].replace("T", + " ") if "from" not in parsed_email: if "From" in parsed_email["headers"]: @@ -687,17 +710,7 @@ - + diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 57c4864..611e152 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt @@ -67,7 +67,6 @@ CLI help [--imap-skip-certificate-verification] [--imap-no-ssl] [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] [--hec-skip-certificate-verification] @@ -100,7 +99,7 @@ CLI help nameservers) -t TIMEOUT, --timeout TIMEOUT number of seconds to wait for an answer from DNS - (Default: 2.0) + (Default: 6.0) -H HOST, --host HOST IMAP hostname or IP address -u USER, --user USER IMAP user -p PASSWORD, --password PASSWORD @@ -120,9 +119,6 @@ CLI help -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] One or more Elasticsearch hostnames or URLs to use (e.g. localhost:9200) - --elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX - Prefix to add in front of the dmarc_aggregate and - dmarc_forensic Elasticsearch index names, joined by _ --elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX Append this suffix to the dmarc_aggregate and dmarc_forensic Elasticsearch index names, joined by _ @@ -608,7 +604,8 @@ Configure Davmail by creating a ``davmail.properties`` file # Enable IDLE support, set polling delay in minutes davmail.imapIdleDelay=1 - # Always reply to IMAP RFC822.SIZE requests with Exchange approximate message size for performance reasons + # Always reply to IMAP RFC822.SIZE requests with Exchange approximate + # message size for performance reasons davmail.imapAlwaysApproxMsgSize=true ############################################################# @@ -849,12 +846,33 @@ the commercial X-Pack_. :align: center :target: _static/screenshots/confirm-overwrite.png +Upgrading Kibana index patterns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``parsedmarc`` 5.0.0 makes some changes to the way data is indexed in +Elasticsearch. if you are upgrading prom a previous release of +``parsedmarc``, you need to complete the following steps to replace the +Kibana index patterns with versions that match the upgraded indexes: + +1. Login in to Kibana, and click on Management +2. Under Kibana, click on Saved Objects +3. Check the checkboxes for the ``dmarc_aggregate`` and ``dmarc_forensic`` + index patterns +4. Click Delete +5. Click Delete on the conformation message +6. Download (right click the link and click save as) + the latest version of kibana_saved_objects.json_ +7. Import ``kibana_saved_objects.json`` by clicking Import from the Kibana + Saved Objects page + Records retention ~~~~~~~~~~~~~~~~~ -To prevent your indexes from growing too large, or to comply with records -retention regulations such as GDPR, you need to use `time-based indexes +Starting in version 5.0.0, ``parsedmarc`` stores data in a separate +index for each day to make it easy to comply with records +retention regulations such as GDPR. For fore information, +check out the Elastic guide to `managing time-based indexes efficiently `_. Splunk diff --git a/_static/basic.css b/_static/basic.css index 19ced10..104f076 100644 --- a/_static/basic.css +++ b/_static/basic.css @@ -81,6 +81,10 @@ div.sphinxsidebar input { font-size: 1em; } +div.sphinxsidebar #searchbox form.search { + overflow: hidden; +} + div.sphinxsidebar #searchbox input[type="text"] { float: left; width: 80%; @@ -427,6 +431,13 @@ table.field-list td, table.field-list th { hyphens: manual; } +/* -- hlist styles ---------------------------------------------------------- */ + +table.hlist td { + vertical-align: top; +} + + /* -- other body styles ----------------------------------------------------- */ ol.arabic { diff --git a/_static/doctools.js b/_static/doctools.js index d892892..ffadbec 100644 --- a/_static/doctools.js +++ b/_static/doctools.js @@ -150,7 +150,9 @@ var Documentation = { this.fixFirefoxAnchorBug(); this.highlightSearchWords(); this.initIndexTable(); - + if (DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) { + this.initOnKeyListeners(); + } }, /** @@ -310,4 +312,4 @@ _ = Documentation.gettext; $(document).ready(function() { Documentation.init(); -}); \ No newline at end of file +}); diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 8e2ab1b..94fcf8a 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,9 +1,296 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '4.4.1', + VERSION: '5.0.0', LANGUAGE: 'None', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true, - SOURCELINK_SUFFIX: '.txt' -}; \ No newline at end of file + SOURCELINK_SUFFIX: '.txt', + NAVIGATION_WITH_KEYS: false, + SEARCH_LANGUAGE_STOP_WORDS: ["a","and","are","as","at","be","but","by","for","if","in","into","is","it","near","no","not","of","on","or","such","that","the","their","then","there","these","they","this","to","was","will","with"] +}; + + + +/* Non-minified version JS is _stemmer.js if file is provided */ +/** + * Porter Stemmer + */ +var Stemmer = function() { + + var step2list = { + ational: 'ate', + tional: 'tion', + enci: 'ence', + anci: 'ance', + izer: 'ize', + bli: 'ble', + alli: 'al', + entli: 'ent', + eli: 'e', + ousli: 'ous', + ization: 'ize', + ation: 'ate', + ator: 'ate', + alism: 'al', + iveness: 'ive', + fulness: 'ful', + ousness: 'ous', + aliti: 'al', + iviti: 'ive', + biliti: 'ble', + logi: 'log' + }; + + var step3list = { + icate: 'ic', + ative: '', + alize: 'al', + iciti: 'ic', + ical: 'ic', + ful: '', + ness: '' + }; + + var c = "[^aeiou]"; // consonant + var v = "[aeiouy]"; // vowel + var C = c + "[^aeiouy]*"; // consonant sequence + var V = v + "[aeiou]*"; // vowel sequence + + var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 + var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 + var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 + var s_v = "^(" + C + ")?" + v; // vowel in stem + + this.stemWord = function (w) { + var stem; + var suffix; + var firstch; + var origword = w; + + if (w.length < 3) + return w; + + var re; + var re2; + var re3; + var re4; + + firstch = w.substr(0,1); + if (firstch == "y") + w = firstch.toUpperCase() + w.substr(1); + + // Step 1a + re = /^(.+?)(ss|i)es$/; + re2 = /^(.+?)([^s])s$/; + + if (re.test(w)) + w = w.replace(re,"$1$2"); + else if (re2.test(w)) + w = w.replace(re2,"$1$2"); + + // Step 1b + re = /^(.+?)eed$/; + re2 = /^(.+?)(ed|ing)$/; + if (re.test(w)) { + var fp = re.exec(w); + re = new RegExp(mgr0); + if (re.test(fp[1])) { + re = /.$/; + w = w.replace(re,""); + } + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = new RegExp(s_v); + if (re2.test(stem)) { + w = stem; + re2 = /(at|bl|iz)$/; + re3 = new RegExp("([^aeiouylsz])\\1$"); + re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re2.test(w)) + w = w + "e"; + else if (re3.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + else if (re4.test(w)) + w = w + "e"; + } + } + + // Step 1c + re = /^(.+?)y$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(s_v); + if (re.test(stem)) + w = stem + "i"; + } + + // Step 2 + re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step2list[suffix]; + } + + // Step 3 + re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step3list[suffix]; + } + + // Step 4 + re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + re2 = /^(.+?)(s|t)(ion)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + if (re.test(stem)) + w = stem; + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = new RegExp(mgr1); + if (re2.test(stem)) + w = stem; + } + + // Step 5 + re = /^(.+?)e$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + re2 = new RegExp(meq1); + re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) + w = stem; + } + re = /ll$/; + re2 = new RegExp(mgr1); + if (re.test(w) && re2.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + + // and turn initial Y back to y + if (firstch == "y") + w = firstch.toLowerCase() + w.substr(1); + return w; + } +} + + + + + +var splitChars = (function() { + var result = {}; + var singles = [96, 180, 187, 191, 215, 247, 749, 885, 903, 907, 909, 930, 1014, 1648, + 1748, 1809, 2416, 2473, 2481, 2526, 2601, 2609, 2612, 2615, 2653, 2702, + 2706, 2729, 2737, 2740, 2857, 2865, 2868, 2910, 2928, 2948, 2961, 2971, + 2973, 3085, 3089, 3113, 3124, 3213, 3217, 3241, 3252, 3295, 3341, 3345, + 3369, 3506, 3516, 3633, 3715, 3721, 3736, 3744, 3748, 3750, 3756, 3761, + 3781, 3912, 4239, 4347, 4681, 4695, 4697, 4745, 4785, 4799, 4801, 4823, + 4881, 5760, 5901, 5997, 6313, 7405, 8024, 8026, 8028, 8030, 8117, 8125, + 8133, 8181, 8468, 8485, 8487, 8489, 8494, 8527, 11311, 11359, 11687, 11695, + 11703, 11711, 11719, 11727, 11735, 12448, 12539, 43010, 43014, 43019, 43587, + 43696, 43713, 64286, 64297, 64311, 64317, 64319, 64322, 64325, 65141]; + var i, j, start, end; + for (i = 0; i < singles.length; i++) { + result[singles[i]] = true; + } + var ranges = [[0, 47], [58, 64], [91, 94], [123, 169], [171, 177], [182, 184], [706, 709], + [722, 735], [741, 747], [751, 879], [888, 889], [894, 901], [1154, 1161], + [1318, 1328], [1367, 1368], [1370, 1376], [1416, 1487], [1515, 1519], [1523, 1568], + [1611, 1631], [1642, 1645], [1750, 1764], [1767, 1773], [1789, 1790], [1792, 1807], + [1840, 1868], [1958, 1968], [1970, 1983], [2027, 2035], [2038, 2041], [2043, 2047], + [2070, 2073], [2075, 2083], [2085, 2087], [2089, 2307], [2362, 2364], [2366, 2383], + [2385, 2391], [2402, 2405], [2419, 2424], [2432, 2436], [2445, 2446], [2449, 2450], + [2483, 2485], [2490, 2492], [2494, 2509], [2511, 2523], [2530, 2533], [2546, 2547], + [2554, 2564], [2571, 2574], [2577, 2578], [2618, 2648], [2655, 2661], [2672, 2673], + [2677, 2692], [2746, 2748], [2750, 2767], [2769, 2783], [2786, 2789], [2800, 2820], + [2829, 2830], [2833, 2834], [2874, 2876], [2878, 2907], [2914, 2917], [2930, 2946], + [2955, 2957], [2966, 2968], [2976, 2978], [2981, 2983], [2987, 2989], [3002, 3023], + [3025, 3045], [3059, 3076], [3130, 3132], [3134, 3159], [3162, 3167], [3170, 3173], + [3184, 3191], [3199, 3204], [3258, 3260], [3262, 3293], [3298, 3301], [3312, 3332], + [3386, 3388], [3390, 3423], [3426, 3429], [3446, 3449], [3456, 3460], [3479, 3481], + [3518, 3519], [3527, 3584], [3636, 3647], [3655, 3663], [3674, 3712], [3717, 3718], + [3723, 3724], [3726, 3731], [3752, 3753], [3764, 3772], [3774, 3775], [3783, 3791], + [3802, 3803], [3806, 3839], [3841, 3871], [3892, 3903], [3949, 3975], [3980, 4095], + [4139, 4158], [4170, 4175], [4182, 4185], [4190, 4192], [4194, 4196], [4199, 4205], + [4209, 4212], [4226, 4237], [4250, 4255], [4294, 4303], [4349, 4351], [4686, 4687], + [4702, 4703], [4750, 4751], [4790, 4791], [4806, 4807], [4886, 4887], [4955, 4968], + [4989, 4991], [5008, 5023], [5109, 5120], [5741, 5742], [5787, 5791], [5867, 5869], + [5873, 5887], [5906, 5919], [5938, 5951], [5970, 5983], [6001, 6015], [6068, 6102], + [6104, 6107], [6109, 6111], [6122, 6127], [6138, 6159], [6170, 6175], [6264, 6271], + [6315, 6319], [6390, 6399], [6429, 6469], [6510, 6511], [6517, 6527], [6572, 6592], + [6600, 6607], [6619, 6655], [6679, 6687], [6741, 6783], [6794, 6799], [6810, 6822], + [6824, 6916], [6964, 6980], [6988, 6991], [7002, 7042], [7073, 7085], [7098, 7167], + [7204, 7231], [7242, 7244], [7294, 7400], [7410, 7423], [7616, 7679], [7958, 7959], + [7966, 7967], [8006, 8007], [8014, 8015], [8062, 8063], [8127, 8129], [8141, 8143], + [8148, 8149], [8156, 8159], [8173, 8177], [8189, 8303], [8306, 8307], [8314, 8318], + [8330, 8335], [8341, 8449], [8451, 8454], [8456, 8457], [8470, 8472], [8478, 8483], + [8506, 8507], [8512, 8516], [8522, 8525], [8586, 9311], [9372, 9449], [9472, 10101], + [10132, 11263], [11493, 11498], [11503, 11516], [11518, 11519], [11558, 11567], + [11622, 11630], [11632, 11647], [11671, 11679], [11743, 11822], [11824, 12292], + [12296, 12320], [12330, 12336], [12342, 12343], [12349, 12352], [12439, 12444], + [12544, 12548], [12590, 12592], [12687, 12689], [12694, 12703], [12728, 12783], + [12800, 12831], [12842, 12880], [12896, 12927], [12938, 12976], [12992, 13311], + [19894, 19967], [40908, 40959], [42125, 42191], [42238, 42239], [42509, 42511], + [42540, 42559], [42592, 42593], [42607, 42622], [42648, 42655], [42736, 42774], + [42784, 42785], [42889, 42890], [42893, 43002], [43043, 43055], [43062, 43071], + [43124, 43137], [43188, 43215], [43226, 43249], [43256, 43258], [43260, 43263], + [43302, 43311], [43335, 43359], [43389, 43395], [43443, 43470], [43482, 43519], + [43561, 43583], [43596, 43599], [43610, 43615], [43639, 43641], [43643, 43647], + [43698, 43700], [43703, 43704], [43710, 43711], [43715, 43738], [43742, 43967], + [44003, 44015], [44026, 44031], [55204, 55215], [55239, 55242], [55292, 55295], + [57344, 63743], [64046, 64047], [64110, 64111], [64218, 64255], [64263, 64274], + [64280, 64284], [64434, 64466], [64830, 64847], [64912, 64913], [64968, 65007], + [65020, 65135], [65277, 65295], [65306, 65312], [65339, 65344], [65371, 65381], + [65471, 65473], [65480, 65481], [65488, 65489], [65496, 65497]]; + for (i = 0; i < ranges.length; i++) { + start = ranges[i][0]; + end = ranges[i][1]; + for (j = start; j <= end; j++) { + result[j] = true; + } + } + return result; +})(); + +function splitQuery(query) { + var result = []; + var start = -1; + for (var i = 0; i < query.length; i++) { + if (splitChars[query.charCodeAt(i)]) { + if (start !== -1) { + result.push(query.slice(start, i)); + start = -1; + } + } else if (start === -1) { + start = i; + } + } + if (start !== -1) { + result.push(query.slice(start)); + } + return result; +} + + diff --git a/_static/searchtools.js b/_static/searchtools.js index 41b8336..7473859 100644 --- a/_static/searchtools.js +++ b/_static/searchtools.js @@ -1,5 +1,5 @@ /* - * searchtools.js_t + * searchtools.js * ~~~~~~~~~~~~~~~~ * * Sphinx JavaScript utilities for the full-text search. @@ -9,323 +9,44 @@ * */ +if (!Scorer) { + /** + * Simple result scoring code. + */ + var Scorer = { + // Implement the following function to further tweak the score for each result + // The function takes a result array [filename, title, anchor, descr, score] + // and returns the new score. + /* + score: function(result) { + return result[4]; + }, + */ -/* Non-minified version JS is _stemmer.js if file is provided */ -/** - * Porter Stemmer - */ -var Stemmer = function() { + // query matches the full name of an object + objNameMatch: 11, + // or matches in the last dotted part of the object name + objPartialMatch: 6, + // Additive scores depending on the priority of the object + objPrio: {0: 15, // used to be importantResults + 1: 5, // used to be objectResults + 2: -5}, // used to be unimportantResults + // Used when the priority is not in the mapping. + objPrioDefault: 0, - var step2list = { - ational: 'ate', - tional: 'tion', - enci: 'ence', - anci: 'ance', - izer: 'ize', - bli: 'ble', - alli: 'al', - entli: 'ent', - eli: 'e', - ousli: 'ous', - ization: 'ize', - ation: 'ate', - ator: 'ate', - alism: 'al', - iveness: 'ive', - fulness: 'ful', - ousness: 'ous', - aliti: 'al', - iviti: 'ive', - biliti: 'ble', - logi: 'log' + // query found in title + title: 15, + // query found in terms + term: 5 }; +} - var step3list = { - icate: 'ic', - ative: '', - alize: 'al', - iciti: 'ic', - ical: 'ic', - ful: '', - ness: '' - }; - - var c = "[^aeiou]"; // consonant - var v = "[aeiouy]"; // vowel - var C = c + "[^aeiouy]*"; // consonant sequence - var V = v + "[aeiou]*"; // vowel sequence - - var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 - var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 - var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 - var s_v = "^(" + C + ")?" + v; // vowel in stem - - this.stemWord = function (w) { - var stem; - var suffix; - var firstch; - var origword = w; - - if (w.length < 3) - return w; - - var re; - var re2; - var re3; - var re4; - - firstch = w.substr(0,1); - if (firstch == "y") - w = firstch.toUpperCase() + w.substr(1); - - // Step 1a - re = /^(.+?)(ss|i)es$/; - re2 = /^(.+?)([^s])s$/; - - if (re.test(w)) - w = w.replace(re,"$1$2"); - else if (re2.test(w)) - w = w.replace(re2,"$1$2"); - - // Step 1b - re = /^(.+?)eed$/; - re2 = /^(.+?)(ed|ing)$/; - if (re.test(w)) { - var fp = re.exec(w); - re = new RegExp(mgr0); - if (re.test(fp[1])) { - re = /.$/; - w = w.replace(re,""); - } - } - else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1]; - re2 = new RegExp(s_v); - if (re2.test(stem)) { - w = stem; - re2 = /(at|bl|iz)$/; - re3 = new RegExp("([^aeiouylsz])\\1$"); - re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); - if (re2.test(w)) - w = w + "e"; - else if (re3.test(w)) { - re = /.$/; - w = w.replace(re,""); - } - else if (re4.test(w)) - w = w + "e"; - } - } - - // Step 1c - re = /^(.+?)y$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(s_v); - if (re.test(stem)) - w = stem + "i"; - } - - // Step 2 - re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) - w = stem + step2list[suffix]; - } - - // Step 3 - re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - suffix = fp[2]; - re = new RegExp(mgr0); - if (re.test(stem)) - w = stem + step3list[suffix]; - } - - // Step 4 - re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; - re2 = /^(.+?)(s|t)(ion)$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - if (re.test(stem)) - w = stem; - } - else if (re2.test(w)) { - var fp = re2.exec(w); - stem = fp[1] + fp[2]; - re2 = new RegExp(mgr1); - if (re2.test(stem)) - w = stem; - } - - // Step 5 - re = /^(.+?)e$/; - if (re.test(w)) { - var fp = re.exec(w); - stem = fp[1]; - re = new RegExp(mgr1); - re2 = new RegExp(meq1); - re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); - if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) - w = stem; - } - re = /ll$/; - re2 = new RegExp(mgr1); - if (re.test(w) && re2.test(w)) { - re = /.$/; - w = w.replace(re,""); - } - - // and turn initial Y back to y - if (firstch == "y") - w = firstch.toLowerCase() + w.substr(1); - return w; +if (!splitQuery) { + function splitQuery(query) { + return query.split(/\s+/); } } - - -/** - * Simple result scoring code. - */ -var Scorer = { - // Implement the following function to further tweak the score for each result - // The function takes a result array [filename, title, anchor, descr, score] - // and returns the new score. - /* - score: function(result) { - return result[4]; - }, - */ - - // query matches the full name of an object - objNameMatch: 11, - // or matches in the last dotted part of the object name - objPartialMatch: 6, - // Additive scores depending on the priority of the object - objPrio: {0: 15, // used to be importantResults - 1: 5, // used to be objectResults - 2: -5}, // used to be unimportantResults - // Used when the priority is not in the mapping. - objPrioDefault: 0, - - // query found in title - title: 15, - // query found in terms - term: 5 -}; - - - - - -var splitChars = (function() { - var result = {}; - var singles = [96, 180, 187, 191, 215, 247, 749, 885, 903, 907, 909, 930, 1014, 1648, - 1748, 1809, 2416, 2473, 2481, 2526, 2601, 2609, 2612, 2615, 2653, 2702, - 2706, 2729, 2737, 2740, 2857, 2865, 2868, 2910, 2928, 2948, 2961, 2971, - 2973, 3085, 3089, 3113, 3124, 3213, 3217, 3241, 3252, 3295, 3341, 3345, - 3369, 3506, 3516, 3633, 3715, 3721, 3736, 3744, 3748, 3750, 3756, 3761, - 3781, 3912, 4239, 4347, 4681, 4695, 4697, 4745, 4785, 4799, 4801, 4823, - 4881, 5760, 5901, 5997, 6313, 7405, 8024, 8026, 8028, 8030, 8117, 8125, - 8133, 8181, 8468, 8485, 8487, 8489, 8494, 8527, 11311, 11359, 11687, 11695, - 11703, 11711, 11719, 11727, 11735, 12448, 12539, 43010, 43014, 43019, 43587, - 43696, 43713, 64286, 64297, 64311, 64317, 64319, 64322, 64325, 65141]; - var i, j, start, end; - for (i = 0; i < singles.length; i++) { - result[singles[i]] = true; - } - var ranges = [[0, 47], [58, 64], [91, 94], [123, 169], [171, 177], [182, 184], [706, 709], - [722, 735], [741, 747], [751, 879], [888, 889], [894, 901], [1154, 1161], - [1318, 1328], [1367, 1368], [1370, 1376], [1416, 1487], [1515, 1519], [1523, 1568], - [1611, 1631], [1642, 1645], [1750, 1764], [1767, 1773], [1789, 1790], [1792, 1807], - [1840, 1868], [1958, 1968], [1970, 1983], [2027, 2035], [2038, 2041], [2043, 2047], - [2070, 2073], [2075, 2083], [2085, 2087], [2089, 2307], [2362, 2364], [2366, 2383], - [2385, 2391], [2402, 2405], [2419, 2424], [2432, 2436], [2445, 2446], [2449, 2450], - [2483, 2485], [2490, 2492], [2494, 2509], [2511, 2523], [2530, 2533], [2546, 2547], - [2554, 2564], [2571, 2574], [2577, 2578], [2618, 2648], [2655, 2661], [2672, 2673], - [2677, 2692], [2746, 2748], [2750, 2767], [2769, 2783], [2786, 2789], [2800, 2820], - [2829, 2830], [2833, 2834], [2874, 2876], [2878, 2907], [2914, 2917], [2930, 2946], - [2955, 2957], [2966, 2968], [2976, 2978], [2981, 2983], [2987, 2989], [3002, 3023], - [3025, 3045], [3059, 3076], [3130, 3132], [3134, 3159], [3162, 3167], [3170, 3173], - [3184, 3191], [3199, 3204], [3258, 3260], [3262, 3293], [3298, 3301], [3312, 3332], - [3386, 3388], [3390, 3423], [3426, 3429], [3446, 3449], [3456, 3460], [3479, 3481], - [3518, 3519], [3527, 3584], [3636, 3647], [3655, 3663], [3674, 3712], [3717, 3718], - [3723, 3724], [3726, 3731], [3752, 3753], [3764, 3772], [3774, 3775], [3783, 3791], - [3802, 3803], [3806, 3839], [3841, 3871], [3892, 3903], [3949, 3975], [3980, 4095], - [4139, 4158], [4170, 4175], [4182, 4185], [4190, 4192], [4194, 4196], [4199, 4205], - [4209, 4212], [4226, 4237], [4250, 4255], [4294, 4303], [4349, 4351], [4686, 4687], - [4702, 4703], [4750, 4751], [4790, 4791], [4806, 4807], [4886, 4887], [4955, 4968], - [4989, 4991], [5008, 5023], [5109, 5120], [5741, 5742], [5787, 5791], [5867, 5869], - [5873, 5887], [5906, 5919], [5938, 5951], [5970, 5983], [6001, 6015], [6068, 6102], - [6104, 6107], [6109, 6111], [6122, 6127], [6138, 6159], [6170, 6175], [6264, 6271], - [6315, 6319], [6390, 6399], [6429, 6469], [6510, 6511], [6517, 6527], [6572, 6592], - [6600, 6607], [6619, 6655], [6679, 6687], [6741, 6783], [6794, 6799], [6810, 6822], - [6824, 6916], [6964, 6980], [6988, 6991], [7002, 7042], [7073, 7085], [7098, 7167], - [7204, 7231], [7242, 7244], [7294, 7400], [7410, 7423], [7616, 7679], [7958, 7959], - [7966, 7967], [8006, 8007], [8014, 8015], [8062, 8063], [8127, 8129], [8141, 8143], - [8148, 8149], [8156, 8159], [8173, 8177], [8189, 8303], [8306, 8307], [8314, 8318], - [8330, 8335], [8341, 8449], [8451, 8454], [8456, 8457], [8470, 8472], [8478, 8483], - [8506, 8507], [8512, 8516], [8522, 8525], [8586, 9311], [9372, 9449], [9472, 10101], - [10132, 11263], [11493, 11498], [11503, 11516], [11518, 11519], [11558, 11567], - [11622, 11630], [11632, 11647], [11671, 11679], [11743, 11822], [11824, 12292], - [12296, 12320], [12330, 12336], [12342, 12343], [12349, 12352], [12439, 12444], - [12544, 12548], [12590, 12592], [12687, 12689], [12694, 12703], [12728, 12783], - [12800, 12831], [12842, 12880], [12896, 12927], [12938, 12976], [12992, 13311], - [19894, 19967], [40908, 40959], [42125, 42191], [42238, 42239], [42509, 42511], - [42540, 42559], [42592, 42593], [42607, 42622], [42648, 42655], [42736, 42774], - [42784, 42785], [42889, 42890], [42893, 43002], [43043, 43055], [43062, 43071], - [43124, 43137], [43188, 43215], [43226, 43249], [43256, 43258], [43260, 43263], - [43302, 43311], [43335, 43359], [43389, 43395], [43443, 43470], [43482, 43519], - [43561, 43583], [43596, 43599], [43610, 43615], [43639, 43641], [43643, 43647], - [43698, 43700], [43703, 43704], [43710, 43711], [43715, 43738], [43742, 43967], - [44003, 44015], [44026, 44031], [55204, 55215], [55239, 55242], [55292, 55295], - [57344, 63743], [64046, 64047], [64110, 64111], [64218, 64255], [64263, 64274], - [64280, 64284], [64434, 64466], [64830, 64847], [64912, 64913], [64968, 65007], - [65020, 65135], [65277, 65295], [65306, 65312], [65339, 65344], [65371, 65381], - [65471, 65473], [65480, 65481], [65488, 65489], [65496, 65497]]; - for (i = 0; i < ranges.length; i++) { - start = ranges[i][0]; - end = ranges[i][1]; - for (j = start; j <= end; j++) { - result[j] = true; - } - } - return result; -})(); - -function splitQuery(query) { - var result = []; - var start = -1; - for (var i = 0; i < query.length; i++) { - if (splitChars[query.charCodeAt(i)]) { - if (start !== -1) { - result.push(query.slice(start, i)); - start = -1; - } - } else if (start === -1) { - start = i; - } - } - if (start !== -1) { - result.push(query.slice(start)); - } - return result; -} - - - - /** * Search Module */ @@ -417,7 +138,7 @@ var Search = { */ query : function(query) { var i; - var stopwords = ["a","and","are","as","at","be","but","by","for","if","in","into","is","it","near","no","not","of","on","or","such","that","the","their","then","there","these","they","this","to","was","will","with"]; + var stopwords = DOCUMENTATION_OPTIONS.SEARCH_LANGUAGE_STOP_WORDS; // stem the searchterms and add them to the correct list var stemmer = new Stemmer(); @@ -758,4 +479,4 @@ var Search = { $(document).ready(function() { Search.init(); -}); \ No newline at end of file +}); diff --git a/genindex.html b/genindex.html index 1d7c713..02e15bc 100644 --- a/genindex.html +++ b/genindex.html @@ -9,7 +9,7 @@ - Index — parsedmarc 4.4.1 documentation + Index — parsedmarc 5.0.0 documentation @@ -57,7 +57,7 @@
- 4.4.1 + 5.0.0
@@ -154,6 +154,7 @@ | G | H | I + | M | P | Q | S @@ -261,6 +262,14 @@ +

M

+ + +
+

P

    @@ -380,17 +389,7 @@ - + diff --git a/index.html b/index.html index 1bea51c..c97d397 100644 --- a/index.html +++ b/index.html @@ -8,7 +8,7 @@ - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 4.4.1 documentation + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
    - 4.4.1 + 5.0.0
    @@ -109,6 +109,7 @@
  • Testing multiple report analyzers
  • Accessing an inbox using OWA/EWS
  • Elasticsearch and Kibana
    • @@ -249,7 +250,6 @@ lookalike domain monitoring, check out [--imap-skip-certificate-verification] [--imap-no-ssl] [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] [--hec-skip-certificate-verification] @@ -282,7 +282,7 @@ lookalike domain monitoring, check out nameservers) -t TIMEOUT, --timeout TIMEOUT number of seconds to wait for an answer from DNS - (Default: 2.0) + (Default: 6.0) -H HOST, --host HOST IMAP hostname or IP address -u USER, --user USER IMAP user -p PASSWORD, --password PASSWORD @@ -302,9 +302,6 @@ lookalike domain monitoring, check out -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] One or more Elasticsearch hostnames or URLs to use (e.g. localhost:9200) - --elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX - Prefix to add in front of the dmarc_aggregate and - dmarc_forensic Elasticsearch index names, joined by _ --elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX Append this suffix to the dmarc_aggregate and dmarc_forensic Elasticsearch index names, joined by _ @@ -733,7 +730,8 @@ as a local EWS/OWA IMAP gateway.

    # Enable IDLE support, set polling delay in minutes davmail.imapIdleDelay=1 -# Always reply to IMAP RFC822.SIZE requests with Exchange approximate message size for performance reasons +# Always reply to IMAP RFC822.SIZE requests with Exchange approximate +# message size for performance reasons davmail.imapAlwaysApproxMsgSize=true ############################################################# @@ -918,10 +916,31 @@ breaks them, as there are no permissions/access controls in Kibana without the commercial     X-Pack.

    A screenshot of setting the Saved Objects management UI in Kibana A screenshot of the overwrite conformation prompt +
    +

    Upgrading Kibana index patterns

    +

    parsedmarc 5.0.0 makes some changes to the way data is indexed in +Elasticsearch. if you are upgrading prom a previous release of +parsedmarc, you need to complete the following steps to replace the +Kibana index patterns with versions that match the upgraded indexes:

    +
      +
    1. Login in to Kibana, and click on Management
      2. +
    2. Under Kibana, click on Saved Objects
      3. +
    3. Check the checkboxes for the dmarc_aggregate and dmarc_forensic +index patterns
      4. +
    4. Click Delete
      5. +
    5. Click Delete on the conformation message
      6. +
    6. Download (right click the link and click save as) +the latest version of kibana_saved_objects.json
      7. +
    7. Import kibana_saved_objects.json by clicking Import from the Kibana +Saved Objects page
      8. +
    +

    Records retention

    -

    To prevent your indexes from growing too large, or to comply with records -retention regulations such as GDPR, you need to use time-based indexes.

    +

    Starting in version 5.0.0, parsedmarc stores data in a separate +index for each day to make it easy to comply with records +retention regulations such as GDPR. For fore information, +check out the Elastic guide to managing time-based indexes efficiently.

    @@ -1595,7 +1614,7 @@ to a callback function

    -parsedmarc.elastic.create_indexes(names=None, settings=None)[source]
    +parsedmarc.elastic.create_indexes(names, settings=None)[source]

    Create Elasticsearch indexes

    @@ -1603,7 +1622,6 @@ to a callback function

    @@ -1612,9 +1630,27 @@ to a callback function

    Parameters:
    • names (list) – A list of index names
      • -
    • "dmarc_forensic"] by default (["dmarc_aggregate",) –
    • settings (dict) – Index settings
    +
    +
    +parsedmarc.elastic.migrate_indexes(aggregate_indexes=None, forensic_indexes=None)[source]
    +

    Updates index mappings

    + +++ + + + +
    Parameters:
      +
    • aggregate_indexes (list) – A list of aggregate index names
      • +
    • forensic_indexes (list) – A list of forensic index names
      • +
    +
    +
    +
    -parsedmarc.elastic.save_aggregate_report_to_elasticsearch(aggregate_report, index='dmarc_aggregate')[source]
    +parsedmarc.elastic.save_aggregate_report_to_elasticsearch(aggregate_report, index_suffix=None)[source]

    Saves a parsed DMARC aggregate report to ElasticSearch

    @@ -1622,7 +1658,7 @@ to a callback function

    @@ -1635,7 +1671,7 @@ to a callback function

    -parsedmarc.elastic.save_forensic_report_to_elasticsearch(forensic_report, index='dmarc_forensic')[source]
    +parsedmarc.elastic.save_forensic_report_to_elasticsearch(forensic_report, index_suffix=None)[source]

    Saves a parsed DMARC forensic report to ElasticSearch

    Parameters:
    • aggregate_report (OrderedDict) – A parsed forensic report
      • -
    • index (str) – The name of the index to save to
      • +
    • index_suffix (str) – The suffix of the name of the index to save to
    @@ -1643,7 +1679,7 @@ to a callback function

    @@ -1834,7 +1870,7 @@ country associated with the given IPv4 or IPv6 address

    -parsedmarc.utils.get_ip_address_info(ip_address, nameservers=None, timeout=2.0)[source]
    +parsedmarc.utils.get_ip_address_info(ip_address, cache=None, nameservers=None, timeout=2.0)[source]

    Returns reverse DNS and country information for the given IP address

    Parameters:
    • forensic_report (OrderedDict) – A parsed forensic report
      • -
    • index (str) – The name of the index to save to
      • +
    • index_suffix (str) – The suffix of the name of the index to save to
    @@ -1842,6 +1878,7 @@ country associated with the given IPv4 or IPv6 address

    Parameters:
    • ip_address (str) – The IP address to check
      • +
    • cache (ExpiringDict) – Cache storage
    • nameservers (list) – A list of one or more nameservers to use
    • public DNS resolvers by default) ((Cloudflare's) –
    • timeout (float) – Sets the DNS timeout in seconds
      • @@ -1860,7 +1897,7 @@ country associated with the given IPv4 or IPv6 address

      -parsedmarc.utils.get_reverse_dns(ip_address, nameservers=None, timeout=2.0)[source]
      +parsedmarc.utils.get_reverse_dns(ip_address, cache=None, nameservers=None, timeout=2.0)[source]

      Resolves an IP address to a hostname using a reverse DNS query

      @@ -1868,6 +1905,7 @@ country associated with the given IPv4 or IPv6 address

      Parameters:
      • ip_address (str) – The IP address to resolve
        • +
      • cache (ExpiringDict) – Cache storage
      • nameservers (list) – A list of one or more nameservers to use
      • public DNS resolvers by default) ((Cloudflare's) –
      • timeout (float) – Sets the DNS query timeout in seconds
        • @@ -1965,7 +2003,7 @@ country associated with the given IPv4 or IPv6 address

        -parsedmarc.utils.query_dns(domain, record_type, nameservers=None, timeout=2.0)[source]
        +parsedmarc.utils.query_dns(domain, record_type, cache=None, nameservers=None, timeout=2.0)[source]

        Queries DNS

        @@ -1974,6 +2012,7 @@ country associated with the given IPv4 or IPv6 address

        Parameters:
        • domain (str) – The domain or subdomain to query about
        • record_type (str) – The record type to query for
          • +
        • cache (ExpiringDict) – Cache storage
        • nameservers (list) – A list of one or more nameservers to use
        • public DNS resolvers by default) ((Cloudflare's) –
        • timeout (float) – Sets the DNS timeout in seconds
          • @@ -2072,17 +2111,7 @@ country associated with the given IPv4 or IPv6 address

          - + diff --git a/objects.inv b/objects.inv index 0f6f4e7598858b9264620dd123ca1f4d83e6039a..cb1166172d227ecd386b5025d259f524de839aa4 100644 GIT binary patch delta 644 zcmV-~0(<@D1?~lqJ_R){FfK5WLpy($OK;mS49D;J6auzuj1?Gm=q>0{tcPGlv%A1Z zj3ZR9>5(P<^wYy_(>4do=_QtB{{9pxigqkGk6eLc#2&Ae;ZY*_Mug^g=ypNLFLiGI zmCR@M#kjj$E$)}j8J96UFtkA!O|DMK_lVwu6nxS54togrfY!Ls%>P_19&UdovbV;e z_JXzI_j%|V<6W=*w5A9p_#hg$Vt~4ZbO`*ohCT9Tm)){nJ*LUsv}JcJEg&E*hUhcD zjhc>*>}RmG)(04o!a|LV=|WENp}nK{V`XriWBN2|F4q!;dwv73nBlsL_(Z zD)Fvn8tK7p+4X5?d(`|x*gjV}^ZQnvcd<9iltX7?uyK6eyp>Tuwl&YnbibB2H&UQ3 z&R9C4euSr2>U`@dL7yxN!_M#ty618^MRr{-jlDevA$?m;3(_rf;ZA>gaaNugy~k9= zsR*Pok5u`7bR|>;j?8eR8}Rt!dn23sPX;MBDGoh>&Zkd@YvZgp+L7u_48&Q;@eeSQx;&i(`5!O|i4^FqY{ delta 636 zcmV-?0)zeT1?B~iJ_R%`G%hibLpy&~$!^;)5WV{=2((uV4N&xun~^$b4n~8xcVbMA zWg^_l#kKSG<)S1`;)5vXB2lAxZ&nUz!Fl8g93%F4tqhM6$u}Z2$3wRZN`9%b`ByTZ z*%#&RZnd~yI%iyl;enwI!f0}JihK{zdys-J`rcs=0UyvB7n<=uSBr<6iP(Qz<4}9S zTH*J3=^Eo*ul}^A2qpL+8cQ*Nx`lKI{J4fa@@A9Wvc5eg;clAS9ZL=bq(u>Z#&4r$ zqv1YzPT{#ds4GRH!B9N)Z5r)%sQl-#VxT+YLqnJfgiQ8*lr5-3aKHjRC^;I_C*R6y zh}P4sGb+_(V-8IuLV=|WENp*-6hSo9G^2-9KM6Y;R+b-oU@OvAxR9eIftBK2&orV3 zv!&|OpzTrf4`KV<>CEq2dEUj|Oj9H`RpW+br4x-;%Ow4b zDh)y?_U7cbMNPbkjv*-*c`}@>pg|s=(4_?@d*NfcuBXpgnowsvvd8)Jk7{yNhg - Python Module Index — parsedmarc 4.4.1 documentation + Python Module Index — parsedmarc 5.0.0 documentation @@ -59,7 +59,7 @@
          - 4.4.1 + 5.0.0
          @@ -209,17 +209,7 @@ - + diff --git a/search.html b/search.html index ea34184..6dcd35f 100644 --- a/search.html +++ b/search.html @@ -8,7 +8,7 @@ - Search — parsedmarc 4.4.1 documentation + Search — parsedmarc 5.0.0 documentation @@ -56,7 +56,7 @@
          - 4.4.1 + 5.0.0
          @@ -187,17 +187,7 @@ - + diff --git a/searchindex.js b/searchindex.js index 70e3786..a674324 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["index"],envversion:55,filenames:["index.rst"],objects:{"":{parsedmarc:[0,0,0,"-"]},"parsedmarc.elastic":{AlreadySaved:[0,1,1,""],ElasticsearchError:[0,1,1,""],create_indexes:[0,2,1,""],save_aggregate_report_to_elasticsearch:[0,2,1,""],save_forensic_report_to_elasticsearch:[0,2,1,""],set_hosts:[0,2,1,""]},"parsedmarc.splunk":{HECClient:[0,3,1,""],SplunkError:[0,1,1,""]},"parsedmarc.splunk.HECClient":{save_aggregate_reports_to_splunk:[0,4,1,""],save_forensic_reports_to_splunk:[0,4,1,""]},"parsedmarc.utils":{EmailParserError:[0,1,1,""],convert_outlook_msg:[0,2,1,""],decode_base64:[0,2,1,""],get_base_domain:[0,2,1,""],get_filename_safe_string:[0,2,1,""],get_ip_address_country:[0,2,1,""],get_ip_address_info:[0,2,1,""],get_reverse_dns:[0,2,1,""],human_timestamp_to_datetime:[0,2,1,""],human_timestamp_to_timestamp:[0,2,1,""],is_outlook_msg:[0,2,1,""],parse_email:[0,2,1,""],query_dns:[0,2,1,""],timestamp_to_datetime:[0,2,1,""],timestamp_to_human:[0,2,1,""]},parsedmarc:{IMAPError:[0,1,1,""],InvalidAggregateReport:[0,1,1,""],InvalidDMARCReport:[0,1,1,""],InvalidForensicReport:[0,1,1,""],ParserError:[0,1,1,""],SMTPError:[0,1,1,""],elastic:[0,0,0,"-"],email_results:[0,2,1,""],extract_xml:[0,2,1,""],get_dmarc_reports_from_inbox:[0,2,1,""],get_imap_capabilities:[0,2,1,""],get_report_zip:[0,2,1,""],parse_aggregate_report_file:[0,2,1,""],parse_aggregate_report_xml:[0,2,1,""],parse_forensic_report:[0,2,1,""],parse_report_email:[0,2,1,""],parse_report_file:[0,2,1,""],parsed_aggregate_reports_to_csv:[0,2,1,""],parsed_forensic_reports_to_csv:[0,2,1,""],save_output:[0,2,1,""],splunk:[0,0,0,"-"],utils:[0,0,0,"-"],watch_inbox:[0,2,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"],"3":["py","class","Python class"],"4":["py","method","Python method"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:function","3":"py:class","4":"py:method"},terms:{"50m":0,"\u00fcbersicht":0,"break":0,"byte":0,"case":0,"class":0,"default":0,"float":0,"function":0,"import":0,"int":0,"long":0,"new":0,"null":0,"public":0,"return":0,"switch":0,"true":0,"while":0,And:0,For:0,OLE:0,One:0,TLS:0,That:0,The:0,Then:0,These:0,Use:0,Uses:0,With:0,_input:0,abl:0,about:0,abov:0,access:[],access_token:0,account:0,acm:0,across:0,action:0,actual:0,add:0,add_head:0,address:0,addresse:0,adkim:0,administr:0,adsl:0,aes128:0,aes256:0,after:0,against:0,agari:0,age:0,aggregate_report:0,all:0,allow:0,allowremot:0,along:0,alreadysav:0,also:0,alter:0,altern:0,although:0,alwai:0,ani:0,anonym:0,anoth:0,answer:0,apach:0,apache2:0,appear:0,append:0,appendix:0,approach:0,approxim:0,apt:0,archiv:0,archive_fold:0,argument:0,arriv:0,arrival_d:0,arrival_date_utc:0,artifact:0,ask:0,asmx:0,aspf:0,associ:0,attach:0,attachment_filenam:0,auth:0,auth_bas:0,auth_basic_user_fil:0,auth_failur:0,auth_result:0,authent:0,authentication_mechan:0,authentication_result:0,author:0,auto:0,autodetect:0,avail:0,avoid:0,b2c:0,backward:0,base64:0,base:0,base_domain:0,basic:0,bcc:0,bd6e1bb5:0,becaus:0,been:0,begin_d:0,behind:0,being:0,bellsouth:0,below:[],best:0,between:0,bin:0,binari:0,bind:0,bindaddress:0,bodi:0,bool:0,brand:0,busi:0,cach:0,call:0,callback:0,can:0,capabl:0,caus:0,center:0,cert:0,certif:0,cest:0,chacha20:0,chang:0,charact:0,charset:0,chart:0,check:0,checkdmarc:0,chines:0,chmod:0,chown:0,click:0,client:0,cloudflar:0,code:0,collect:0,collector:0,com:0,come:0,comma:0,command:0,commerci:0,common:0,compat:0,complet:0,compli:0,compress:0,configur:0,connect:0,consid:0,consist:0,consolid:0,consum:0,contact:0,contain:0,content:0,context:0,control:0,convert:0,convert_outlook_msg:0,copi:0,correctli:0,could:0,count:0,countri:0,crash:0,creat:0,create_index:0,credenti:0,crt:0,csr:0,cumul:0,current:0,custom:0,daemon:0,dai:0,daili:0,dat:0,data:0,databas:0,date:0,date_rang:[],date_utc:0,datetim:0,davmail:0,deb:0,debian:0,debug:0,decod:0,decode_base64:0,defens:0,delai:0,delet:0,delivery_result:0,demystifi:0,descript:0,detail:0,develop:0,dict:0,dictionari:0,differ:0,directli:0,directori:0,dis:0,disabl:0,displai:0,display_nam:0,disposit:0,dkim_align:0,dkim_domain:0,dkim_result:0,dkim_selector:0,dkm:0,dmarc_aggreg:0,dmarc_forens:0,dmarcian:0,dns_timeout:0,doctyp:0,doe:0,domain:[],domainawar:0,don:0,done:0,down:0,download:0,draft:0,dtd:0,dure:0,each:0,earlier:0,easier:0,ecdh:0,ecdsa:0,echo:0,edit:0,editor:0,elasticsearch_host:0,elasticsearch_index_prefix:0,elasticsearch_index_suffix:0,elasticsearcherror:0,els:0,email:0,email_result:0,emailparsererror:0,enabl:0,enableew:0,enablekeepal:0,enableproxi:0,encod:0,encount:0,end:0,end_dat:0,ensur:0,entir:0,envelop:0,envelope_from:0,envelope_to:0,environ:0,error:0,especi:0,etc:0,even:0,event:0,everi:0,exampl:0,exampleus:0,except:0,exchang:0,exclud:0,execstart:0,exist:0,exit:0,extract:0,extract_xml:0,fail:0,failur:0,fals:0,feedback:0,feedback_report:0,feedback_typ:0,fetch:0,few:0,field:0,file:0,file_path:0,filenam:0,filename_safe_subject:0,fill:0,filter:0,financ:0,find:0,first:0,fix:0,flag:0,flat:0,flexibl:0,folder:0,foldersizelimit:0,follow:0,foobar:0,forensic_report:0,forensic_top:0,format:0,forward:0,found:0,foundat:0,fqdn:0,frame:0,fraud:0,friendli:0,from:0,front:0,ftp_proxi:0,full:0,further:0,gatewai:0,gcm:0,gdpr:0,gener:0,geolite2:0,get:0,get_base_domain:0,get_dmarc_reports_from_inbox:0,get_filename_safe_str:0,get_imap_cap:0,get_ip_address_countri:0,get_ip_address_info:0,get_report_zip:0,get_reverse_dn:0,git:0,github:0,give:0,given:0,glass:0,global:0,gmail:0,googl:0,gpg:0,graph:0,group:0,grow:0,gzip:0,handl:0,has:0,has_defect:0,have:0,head:0,header:0,header_from:0,headless:0,healthcar:0,heap:0,heavi:0,hec:0,hec_index:0,hec_token:0,hecclient:0,here:0,high:0,highli:0,hop:0,host:0,hostnam:0,hour:0,hover:0,href:0,html:0,htpasswd:0,http2:0,http:0,http_proxi:0,httpasswd:0,https_proxi:0,human:0,human_timestamp:0,human_timestamp_to_datetim:0,human_timestamp_to_timestamp:0,icon:0,identifi:0,idl:0,imap:0,imap_port:0,imapalwaysapproxmsgs:0,imapautoexpung:0,imapcli:0,imaperror:0,imapidledelai:0,imapport:0,immedi:0,impli:0,improv:0,inbox:[],includ:0,includesubdomain:0,incom:0,increas:0,index:0,industri:0,inform:0,inlin:[],input:0,input_:0,insid:0,instanc:0,instead:0,interact:0,interakt:0,invalid:0,invalidaggregatereport:0,invaliddmarcreport:0,invalidforensicreport:0,ip_address:0,ipv4:0,ipv6:0,is_outlook_msg:0,iso:0,issu:0,its:0,java:0,join:0,journalctl:0,jre:0,just:0,jvm:0,kafka:0,kafka_aggregate_top:0,kafka_forensic_top:0,kafka_host:0,kb4099855:0,kb4134118:0,kb4295699:0,keepal:0,kei:0,keyout:0,kibana_saved_object:0,kind:0,know:0,known:0,larg:0,later:0,latest:0,layout:0,leak:0,least:0,leav:0,left:0,legitim:0,level:0,libemail:0,like:0,limit:0,line:0,link:0,linux:0,list:0,listen:0,load:0,local:0,localhost:0,locat:0,log:0,login:0,look:0,loopback:0,lot:0,lua:0,maco:0,magnifi:0,mai:0,mail:0,mail_from:0,mail_to:0,mailbox:0,mailer:0,mailrelai:0,mailto:0,main:0,maintain:0,make:0,malici:0,manag:0,manual:0,market:0,match:0,max:0,maximum:0,maxmind:0,mechan:0,mention:0,menu:0,messag:0,message_id:0,meta:0,mfrom:0,microsoft:0,might:0,mime:0,minimum:0,minut:0,mkdir:0,mode:0,modern:0,modul:0,mon:0,monitor:0,more:0,most:0,mous:0,move:0,move_support:0,msg:0,msg_byte:0,msg_date:0,msgconvert:0,multi:0,must:0,name:0,nameserv:0,nano:0,ncontent:0,ndate:0,need:0,neeed:0,net:0,network:0,newest:0,newkei:0,newli:[],next:0,nfrom:0,nginx:0,nmessag:0,nmime:0,node:0,non:0,none:0,noproxyfor:0,norepli:0,normal:0,nosecureimap:0,nosniff:0,notabl:0,now:0,nreceiv:[],nsubject:0,nto:0,number:0,nwettbewerb:0,object:0,observ:0,occur:0,occurr:0,oct:0,off:0,office365:0,often:0,oing_from:[],old:0,older:0,oldest:0,onc:[],ondmarc:0,one:0,onli:0,onlin:0,openjdk:[],openssl:0,opt:0,ordereddict:0,org:0,org_email:0,org_extra_contact_info:0,org_nam:0,organ:0,organis:0,origin:0,original_envelope_id:0,original_mail_from:0,original_rcpt_to:0,other:0,our:0,out:0,outdat:0,outg:[],outgo:0,outgoing_attach:0,outgoing_from:0,outgoing_host:0,outgoing_messag:0,outgoing_password:0,outgoing_port:0,outgoing_ssl:0,outgoing_subject:0,outgoing_to:0,outgoing_us:0,outlook:0,output_directori:0,over:0,overrid:0,overwrit:0,own:0,pack:0,packag:0,pad:0,page:0,pan:0,param:0,paramet:0,parent:0,pars:0,parse_aggregate_report_fil:0,parse_aggregate_report_xml:0,parse_email:0,parse_forensic_report:0,parse_report_email:0,parse_report_fil:0,parsed_aggregate_reports_to_csv:0,parsed_forensic_reports_to_csv:0,parsed_sampl:0,parser:0,parsererror:0,part:0,particular:0,particularli:0,pass:0,passag:0,password:0,past:0,patch:0,path:0,pattern:0,payload:0,pct:0,percentag:0,perl:0,permiss:0,peter:0,pie:0,pip3:0,pip:0,place:0,plain:0,pleas:0,plu:0,polici:0,policy_evalu:0,policy_override_com:0,policy_override_reason:0,policy_publish:0,poll:0,poly1305:0,port:0,posit:0,possibl:0,prefix:0,preload:0,premad:0,prevent:0,previou:0,previous:0,print:0,printabl:0,privaci:0,process:0,produc:0,program:0,project:0,prompt:0,proofpoint:0,properti:0,protect:0,provid:0,prox:0,proxi:0,proxy_add_x_forwarded_for:0,proxy_pass:0,proxy_set_head:0,proxyhost:0,proxypassword:0,proxyport:0,proxyus:0,public_suffix_list:0,publicli:[],publicsuffix:0,publish:0,python3:0,python:0,queri:0,query_dn:0,quot:0,rais:0,readabl:0,real:0,realli:0,reason:0,receiv:0,recipi:0,recogn:0,record_typ:0,refer:0,regardless:0,regul:0,regulatori:0,relai:0,relat:0,releas:0,reli:0,reload:0,remain:0,remot:0,remote_addr:0,remov:0,replac:0,repli:0,reply_to:0,report_id:0,report_metadata:0,report_typ:0,reported_domain:0,reports_fold:0,repositori:0,req:0,request:0,request_uri:0,requir:0,resolv:0,respons:0,restart:0,restartsec:0,restor:0,result:0,retriev:0,reus:0,revers:0,reverse_dn:0,review:0,rfc822:0,rfc:0,right:0,rollup:0,root:0,rsa:0,rua:0,ruf:0,rule:0,safe:0,same:0,sameorigin:0,sample_headers_onli:0,save:0,save_aggregate_report_to_elasticsearch:0,save_aggregate_reports_to_splunk:0,save_forensic_report_to_elasticsearch:0,save_forensic_reports_to_splunk:0,save_output:0,schema:0,scope:0,search:0,second:0,secur:0,see:0,segment:0,select:[],selector:0,self:0,send:0,sensit:0,sent:0,separ:0,server:0,servernameon:0,session:0,set:0,set_host:0,sha256:0,sha384:0,share:0,sharepoint:0,should:0,show:0,shown:[],shv:0,side:0,sign:0,signatur:0,silent:0,similar:0,simpl:0,simpli:0,simplifi:0,singl:0,sister:0,site:0,situat:0,size:0,skip:0,slightli:0,small:0,smg:0,smtp:0,smtperror:0,socket:0,solut:0,some:0,someon:0,sometim:0,sort:0,source_base_domain:0,source_countri:0,source_ip_address:0,source_reverse_dn:0,sourceforg:0,specif:0,specifi:0,speed:0,spf_align:0,spf_domain:0,spf_result:0,spf_scope:0,splunk:[],splunkerror:0,spoof:0,ssl:0,ssl_certif:0,ssl_certificate_kei:0,ssl_cipher:0,ssl_context:0,ssl_prefer_server_ciph:0,ssl_protocol:0,ssl_session_cach:0,ssl_session_ticket:0,ssl_session_timeout:0,sslcontext:0,stabl:0,standard:0,start:0,starttl:0,statu:0,still:0,store:0,str:0,strict:0,string:0,strip:0,strip_attachment_payload:0,structur:0,subdomain:0,subject:0,subsidiari:0,substitut:0,sudo:0,suffix:0,suggest:0,suit:0,suppli:0,sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq:0,symlink:0,system:0,systemctl:0,tab:0,tag:0,target:0,tby:0,tee:0,tell:0,temporari:0,text:0,tgs:[],thank:0,thei:0,theirs:0,them:0,thi:0,those:0,three:0,through:0,time:0,timeout:0,timestamp:0,timestamp_to_datetim:0,timestamp_to_human:0,timezon:0,tld:0,tlsv1:0,to_domain:0,to_utc:0,token:0,too:0,top:0,topic:0,tracker:0,transfer:0,transpar:0,transport:0,trust:0,tweak:0,two:0,type:0,ubuntu:0,uncom:0,under:0,underneath:0,understand:0,uninstal:0,unit:0,unix:0,unzip:0,updat:0,upgrad:0,upper:0,uri:0,url:0,usag:0,use:0,use_ssl:0,used:0,useful:0,user:0,user_ag:0,usernam:0,usesystemproxi:0,usr:0,utc:0,utf:0,util:[],valu:0,vendor:0,venv:0,veri:0,verif:0,verifi:0,version:0,vew:0,view:0,virtualenv:0,visit:[],volum:0,vulner:0,w3c:0,wai:0,wait:0,want:0,wantedbi:0,warn:0,watch:0,watch_inbox:0,watcher:0,web:0,webdav:0,webmail:0,well:0,were:0,wettbewerb:0,wget:0,when:0,whenev:0,where:0,wherea:0,which:0,who:0,why:0,wide:0,wiki:0,window:0,without:0,work:0,workstat:0,worst:0,would:0,write:0,www:0,x509:0,xennn:0,xml:0,xml_schema:0,xms4g:0,xmx4g:0,yahoo:0,yet:0,you:0,your:0,yyyi:0,zip:0},titles:["parsedmarc documentation - Open source DMARC report analyzer and visualizer"],titleterms:{DNS:0,EWS:0,Using:0,access:0,aggreg:0,align:0,analyz:0,api:0,bug:0,cli:0,csv:0,dashboard:0,depend:0,dkim:0,dmarc:0,document:0,domain:0,elast:0,elasticsearch:0,featur:0,forens:0,guid:0,help:0,inbox:0,indic:0,instal:0,json:0,kibana:0,lookalik:0,multipl:0,open:0,option:0,output:0,owa:0,parsedmarc:0,perform:0,pypy3:0,record:0,report:0,resourc:0,retent:0,run:0,sampl:0,sender:0,servic:0,sourc:0,spf:0,splunk:0,summari:0,support:0,systemd:0,tabl:0,test:0,using:0,util:0,valid:0,visual:0,what:0,won:0}}) \ No newline at end of file +Search.setIndex({docnames:["index"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.cpp":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,"sphinx.ext.todo":1,"sphinx.ext.viewcode":1,sphinx:55},filenames:["index.rst"],objects:{"":{parsedmarc:[0,0,0,"-"]},"parsedmarc.elastic":{AlreadySaved:[0,1,1,""],ElasticsearchError:[0,1,1,""],create_indexes:[0,2,1,""],migrate_indexes:[0,2,1,""],save_aggregate_report_to_elasticsearch:[0,2,1,""],save_forensic_report_to_elasticsearch:[0,2,1,""],set_hosts:[0,2,1,""]},"parsedmarc.splunk":{HECClient:[0,3,1,""],SplunkError:[0,1,1,""]},"parsedmarc.splunk.HECClient":{save_aggregate_reports_to_splunk:[0,4,1,""],save_forensic_reports_to_splunk:[0,4,1,""]},"parsedmarc.utils":{EmailParserError:[0,1,1,""],convert_outlook_msg:[0,2,1,""],decode_base64:[0,2,1,""],get_base_domain:[0,2,1,""],get_filename_safe_string:[0,2,1,""],get_ip_address_country:[0,2,1,""],get_ip_address_info:[0,2,1,""],get_reverse_dns:[0,2,1,""],human_timestamp_to_datetime:[0,2,1,""],human_timestamp_to_timestamp:[0,2,1,""],is_outlook_msg:[0,2,1,""],parse_email:[0,2,1,""],query_dns:[0,2,1,""],timestamp_to_datetime:[0,2,1,""],timestamp_to_human:[0,2,1,""]},parsedmarc:{IMAPError:[0,1,1,""],InvalidAggregateReport:[0,1,1,""],InvalidDMARCReport:[0,1,1,""],InvalidForensicReport:[0,1,1,""],ParserError:[0,1,1,""],SMTPError:[0,1,1,""],elastic:[0,0,0,"-"],email_results:[0,2,1,""],extract_xml:[0,2,1,""],get_dmarc_reports_from_inbox:[0,2,1,""],get_imap_capabilities:[0,2,1,""],get_report_zip:[0,2,1,""],parse_aggregate_report_file:[0,2,1,""],parse_aggregate_report_xml:[0,2,1,""],parse_forensic_report:[0,2,1,""],parse_report_email:[0,2,1,""],parse_report_file:[0,2,1,""],parsed_aggregate_reports_to_csv:[0,2,1,""],parsed_forensic_reports_to_csv:[0,2,1,""],save_output:[0,2,1,""],splunk:[0,0,0,"-"],utils:[0,0,0,"-"],watch_inbox:[0,2,1,""]}},objnames:{"0":["py","module","Python module"],"1":["py","exception","Python exception"],"2":["py","function","Python function"],"3":["py","class","Python class"],"4":["py","method","Python method"]},objtypes:{"0":"py:module","1":"py:exception","2":"py:function","3":"py:class","4":"py:method"},terms:{"50m":0,"\u00fcbersicht":0,"break":0,"byte":0,"case":0,"class":0,"default":0,"float":0,"function":0,"import":0,"int":0,"long":0,"new":0,"null":0,"public":0,"return":0,"switch":0,"true":0,"while":0,And:0,For:0,OLE:0,One:0,TLS:0,That:0,The:0,Then:0,These:0,Use:0,Uses:0,With:0,_input:0,abl:0,about:0,abov:0,access_token:0,account:0,acm:0,across:0,action:0,actual:0,add:0,add_head:0,address:0,addresse:0,adkim:0,administr:0,adsl:0,aes128:0,aes256:0,after:0,against:0,agari:0,age:0,aggregate_index:0,aggregate_report:0,all:0,allow:0,allowremot:0,along:0,alreadysav:0,also:0,alter:0,altern:0,although:0,alwai:0,ani:0,anonym:0,anoth:0,answer:0,apach:0,apache2:0,appear:0,append:0,appendix:0,approach:0,approxim:0,apt:0,archiv:0,archive_fold:0,argument:0,arriv:0,arrival_d:0,arrival_date_utc:0,artifact:0,ask:0,asmx:0,aspf:0,associ:0,attach:0,attachment_filenam:0,auth:0,auth_bas:0,auth_basic_user_fil:0,auth_failur:0,auth_result:0,authent:0,authentication_mechan:0,authentication_result:0,author:0,auto:0,autodetect:0,avail:0,avoid:0,b2c:0,backward:0,base64:0,base:0,base_domain:0,basic:0,bcc:0,bd6e1bb5:0,becaus:0,been:0,begin_d:0,behind:0,being:0,bellsouth:0,best:0,between:0,bin:0,binari:0,bind:0,bindaddress:0,bodi:0,bool:0,brand:0,busi:0,cach:0,call:0,callback:0,can:0,capabl:0,caus:0,center:0,cert:0,certif:0,cest:0,chacha20:0,chang:0,charact:0,charset:0,chart:0,check:0,checkbox:0,checkdmarc:0,chines:0,chmod:0,chown:0,click:0,client:0,cloudflar:0,code:0,collect:0,collector:0,com:0,come:0,comma:0,command:0,commerci:0,common:0,compat:0,complet:0,compli:0,compress:0,configur:0,conform:0,connect:0,consid:0,consist:0,consolid:0,consum:0,contact:0,contain:0,content:0,context:0,control:0,convert:0,convert_outlook_msg:0,copi:0,correctli:0,could:0,count:0,countri:0,crash:0,creat:0,create_index:0,credenti:0,crt:0,csr:0,cumul:0,current:0,custom:0,daemon:0,dai:0,daili:0,dat:0,data:0,databas:0,date:0,date_utc:0,datetim:0,davmail:0,deb:0,debian:0,debug:0,decod:0,decode_base64:0,defens:0,delai:0,delet:0,delivery_result:0,demystifi:0,descript:0,detail:0,develop:0,dict:0,dictionari:0,differ:0,directli:0,directori:0,dis:0,disabl:0,displai:0,display_nam:0,disposit:0,dkim_align:0,dkim_domain:0,dkim_result:0,dkim_selector:0,dkm:0,dmarc_aggreg:0,dmarc_forens:0,dmarcian:0,dns_timeout:0,doctyp:0,doe:0,domainawar:0,don:0,done:0,down:0,download:0,draft:0,dtd:0,dure:0,each:0,earlier:0,easi:0,easier:0,ecdh:0,ecdsa:0,echo:0,edit:0,editor:0,effici:0,elasticsearch_host:0,elasticsearch_index_suffix:0,elasticsearcherror:0,els:0,email:0,email_result:0,emailparsererror:0,enabl:0,enableew:0,enablekeepal:0,enableproxi:0,encod:0,encount:0,end:0,end_dat:0,ensur:0,entir:0,envelop:0,envelope_from:0,envelope_to:0,environ:0,error:0,especi:0,etc:0,even:0,event:0,everi:0,exampl:0,exampleus:0,except:0,exchang:0,exclud:0,execstart:0,exist:0,exit:0,expiringdict:0,extract:0,extract_xml:0,fail:0,failur:0,fals:0,feedback:0,feedback_report:0,feedback_typ:0,fetch:0,few:0,field:0,file:0,file_path:0,filenam:0,filename_safe_subject:0,fill:0,filter:0,financ:0,find:0,first:0,fix:0,flag:0,flat:0,flexibl:0,folder:0,foldersizelimit:0,follow:0,foobar:0,fore:0,forensic_index:0,forensic_report:0,forensic_top:0,format:0,forward:0,found:0,foundat:0,fqdn:0,frame:0,fraud:0,friendli:0,from:0,ftp_proxi:0,full:0,further:0,gatewai:0,gcm:0,gdpr:0,gener:0,geolite2:0,get:0,get_base_domain:0,get_dmarc_reports_from_inbox:0,get_filename_safe_str:0,get_imap_cap:0,get_ip_address_countri:0,get_ip_address_info:0,get_report_zip:0,get_reverse_dn:0,git:0,github:0,give:0,given:0,glass:0,global:0,gmail:0,googl:0,gpg:0,graph:0,group:0,gzip:0,handl:0,has:0,has_defect:0,have:0,head:0,header:0,header_from:0,headless:0,healthcar:0,heap:0,heavi:0,hec:0,hec_index:0,hec_token:0,hecclient:0,here:0,high:0,highli:0,hop:0,host:0,hostnam:0,hour:0,hover:0,href:0,html:0,htpasswd:0,http2:0,http:0,http_proxi:0,httpasswd:0,https_proxi:0,human:0,human_timestamp:0,human_timestamp_to_datetim:0,human_timestamp_to_timestamp:0,icon:0,identifi:0,idl:0,imap:0,imap_port:0,imapalwaysapproxmsgs:0,imapautoexpung:0,imapcli:0,imaperror:0,imapidledelai:0,imapport:0,immedi:0,impli:0,improv:0,includ:0,includesubdomain:0,incom:0,increas:0,index_suffix:0,industri:0,inform:0,input:0,input_:0,insid:0,instanc:0,instead:0,interact:0,interakt:0,invalid:0,invalidaggregatereport:0,invaliddmarcreport:0,invalidforensicreport:0,ip_address:0,ipv4:0,ipv6:0,is_outlook_msg:0,iso:0,issu:0,its:0,java:0,join:0,journalctl:0,jre:0,just:0,jvm:0,kafka:0,kafka_aggregate_top:0,kafka_forensic_top:0,kafka_host:0,kb4099855:0,kb4134118:0,kb4295699:0,keepal:0,kei:0,keyout:0,kibana_saved_object:0,kind:0,know:0,known:0,larg:0,later:0,latest:0,layout:0,leak:0,least:0,leav:0,left:0,legitim:0,level:0,libemail:0,like:0,limit:0,line:0,link:0,linux:0,list:0,listen:0,load:0,local:0,localhost:0,locat:0,log:0,login:0,look:0,loopback:0,lot:0,lua:0,maco:0,magnifi:0,mai:0,mail:0,mail_from:0,mail_to:0,mailbox:0,mailer:0,mailrelai:0,mailto:0,main:0,maintain:0,make:0,malici:0,manag:0,manual:0,map:0,market:0,match:0,max:0,maximum:0,maxmind:0,mechan:0,mention:0,menu:0,messag:0,message_id:0,meta:0,mfrom:0,microsoft:0,might:0,migrate_index:0,mime:0,minimum:0,minut:0,mkdir:0,mode:0,modern:0,modul:0,mon:0,monitor:0,more:0,most:0,mous:0,move:0,move_support:0,msg:0,msg_byte:0,msg_date:0,msgconvert:0,multi:0,must:0,name:0,nameserv:0,nano:0,ncontent:0,ndate:0,need:0,neeed:0,net:0,network:0,newest:0,newkei:0,next:0,nfrom:0,nginx:0,nmessag:0,nmime:0,node:0,non:0,none:0,noproxyfor:0,norepli:0,normal:0,nosecureimap:0,nosniff:0,notabl:0,now:0,nsubject:0,nto:0,number:0,nwettbewerb:0,object:0,observ:0,occur:0,occurr:0,oct:0,off:0,office365:0,often:0,old:0,older:0,oldest:0,ondmarc:0,one:0,onli:0,onlin:0,openssl:0,opt:0,ordereddict:0,org:0,org_email:0,org_extra_contact_info:0,org_nam:0,organ:0,organis:0,origin:0,original_envelope_id:0,original_mail_from:0,original_rcpt_to:0,other:0,our:0,out:0,outdat:0,outgo:0,outgoing_attach:0,outgoing_from:0,outgoing_host:0,outgoing_messag:0,outgoing_password:0,outgoing_port:0,outgoing_ssl:0,outgoing_subject:0,outgoing_to:0,outgoing_us:0,outlook:0,output_directori:0,over:0,overrid:0,overwrit:0,own:0,pack:0,packag:0,pad:0,page:0,pan:0,param:0,paramet:0,parent:0,pars:0,parse_aggregate_report_fil:0,parse_aggregate_report_xml:0,parse_email:0,parse_forensic_report:0,parse_report_email:0,parse_report_fil:0,parsed_aggregate_reports_to_csv:0,parsed_forensic_reports_to_csv:0,parsed_sampl:0,parser:0,parsererror:0,part:0,particular:0,particularli:0,pass:0,passag:0,password:0,past:0,patch:0,path:0,payload:0,pct:0,percentag:0,perl:0,permiss:0,peter:0,pie:0,pip3:0,pip:0,place:0,plain:0,pleas:0,plu:0,polici:0,policy_evalu:0,policy_override_com:0,policy_override_reason:0,policy_publish:0,poll:0,poly1305:0,port:0,posit:0,possibl:0,preload:0,premad:0,previou:0,previous:0,print:0,printabl:0,privaci:0,process:0,produc:0,program:0,project:0,prom:0,prompt:0,proofpoint:0,properti:0,protect:0,provid:0,prox:0,proxi:0,proxy_add_x_forwarded_for:0,proxy_pass:0,proxy_set_head:0,proxyhost:0,proxypassword:0,proxyport:0,proxyus:0,public_suffix_list:0,publicsuffix:0,publish:0,python3:0,python:0,queri:0,query_dn:0,quot:0,rais:0,readabl:0,real:0,realli:0,reason:0,receiv:0,recipi:0,recogn:0,record_typ:0,refer:0,regardless:0,regul:0,regulatori:0,relai:0,relat:0,releas:0,reli:0,reload:0,remain:0,remot:0,remote_addr:0,remov:0,replac:0,repli:0,reply_to:0,report_id:0,report_metadata:0,report_typ:0,reported_domain:0,reports_fold:0,repositori:0,req:0,request:0,request_uri:0,requir:0,resolv:0,respons:0,restart:0,restartsec:0,restor:0,result:0,retriev:0,reus:0,revers:0,reverse_dn:0,review:0,rfc822:0,rfc:0,right:0,rollup:0,root:0,rsa:0,rua:0,ruf:0,rule:0,safe:0,same:0,sameorigin:0,sample_headers_onli:0,save:0,save_aggregate_report_to_elasticsearch:0,save_aggregate_reports_to_splunk:0,save_forensic_report_to_elasticsearch:0,save_forensic_reports_to_splunk:0,save_output:0,schema:0,scope:0,search:0,second:0,secur:0,see:0,segment:0,selector:0,self:0,send:0,sensit:0,sent:0,separ:0,server:0,servernameon:0,session:0,set:0,set_host:0,sha256:0,sha384:0,share:0,sharepoint:0,should:0,show:0,shv:0,side:0,sign:0,signatur:0,silent:0,similar:0,simpl:0,simpli:0,simplifi:0,singl:0,sister:0,site:0,situat:0,size:0,skip:0,slightli:0,small:0,smg:0,smtp:0,smtperror:0,socket:0,solut:0,some:0,someon:0,sometim:0,sort:0,source_base_domain:0,source_countri:0,source_ip_address:0,source_reverse_dn:0,sourceforg:0,specif:0,specifi:0,speed:0,spf_align:0,spf_domain:0,spf_result:0,spf_scope:0,splunkerror:0,spoof:0,ssl:0,ssl_certif:0,ssl_certificate_kei:0,ssl_cipher:0,ssl_context:0,ssl_prefer_server_ciph:0,ssl_protocol:0,ssl_session_cach:0,ssl_session_ticket:0,ssl_session_timeout:0,sslcontext:0,stabl:0,standard:0,start:0,starttl:0,statu:0,step:0,still:0,storag:0,store:0,str:0,strict:0,string:0,strip:0,strip_attachment_payload:0,structur:0,subdomain:0,subject:0,subsidiari:0,substitut:0,sudo:0,suffix:0,suggest:0,suit:0,suppli:0,sw50zxjha3rpdmugv2v0dgjld2vyymvylcocymvyc2ljahq:0,symlink:0,system:0,systemctl:0,tab:0,tag:0,target:0,tby:0,tee:0,tell:0,temporari:0,text:0,thank:0,thei:0,theirs:0,them:0,thi:0,those:0,three:0,through:0,time:0,timeout:0,timestamp:0,timestamp_to_datetim:0,timestamp_to_human:0,timezon:0,tld:0,tlsv1:0,to_domain:0,to_utc:0,token:0,top:0,topic:0,tracker:0,transfer:0,transpar:0,transport:0,trust:0,tweak:0,two:0,type:0,ubuntu:0,uncom:0,under:0,underneath:0,understand:0,uninstal:0,unit:0,unix:0,unzip:0,updat:0,upper:0,uri:0,url:0,usag:0,use:0,use_ssl:0,used:0,useful:0,user:0,user_ag:0,usernam:0,usesystemproxi:0,usr:0,utc:0,utf:0,valu:0,vendor:0,venv:0,veri:0,verif:0,verifi:0,version:0,vew:0,view:0,virtualenv:0,volum:0,vulner:0,w3c:0,wai:0,wait:0,want:0,wantedbi:0,warn:0,watch:0,watch_inbox:0,watcher:0,web:0,webdav:0,webmail:0,well:0,were:0,wettbewerb:0,wget:0,when:0,whenev:0,where:0,wherea:0,which:0,who:0,why:0,wide:0,wiki:0,window:0,without:0,work:0,workstat:0,worst:0,would:0,write:0,www:0,x509:0,xennn:0,xml:0,xml_schema:0,xms4g:0,xmx4g:0,yahoo:0,yet:0,you:0,your:0,yyyi:0,zip:0},titles:["parsedmarc documentation - Open source DMARC report analyzer and visualizer"],titleterms:{DNS:0,EWS:0,Using:0,access:0,aggreg:0,align:0,analyz:0,api:0,bug:0,cli:0,csv:0,dashboard:0,depend:0,dkim:0,dmarc:0,document:0,domain:0,elast:0,elasticsearch:0,featur:0,forens:0,guid:0,help:0,inbox:0,index:0,indic:0,instal:0,json:0,kibana:0,lookalik:0,multipl:0,open:0,option:0,output:0,owa:0,parsedmarc:0,pattern:0,perform:0,pypy3:0,record:0,report:0,resourc:0,retent:0,run:0,sampl:0,sender:0,servic:0,sourc:0,spf:0,splunk:0,summari:0,support:0,systemd:0,tabl:0,test:0,upgrad:0,using:0,util:0,valid:0,visual:0,what:0,won:0}}) \ No newline at end of file