From 1ee957d94b51d351ccafb9c40ac165f8bdf8bfe7 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Thu, 23 Apr 2026 02:18:47 -0400 Subject: [PATCH] Update docs --- _modules/index.html | 4 +- _modules/parsedmarc.html | 16 ++- _modules/parsedmarc/elastic.html | 16 ++- _modules/parsedmarc/opensearch.html | 16 ++- _modules/parsedmarc/splunk.html | 7 +- _modules/parsedmarc/types.html | 9 +- _modules/parsedmarc/utils.html | 165 +++++++++++++++++++++------- _sources/output.md.txt | 18 ++- _static/documentation_options.js | 2 +- api.html | 16 ++- contributing.html | 4 +- davmail.html | 4 +- dmarc.html | 4 +- elasticsearch.html | 4 +- genindex.html | 8 +- index.html | 4 +- installation.html | 4 +- kibana.html | 4 +- mailing-lists.html | 4 +- objects.inv | Bin 1350 -> 1360 bytes opensearch.html | 4 +- output.html | 20 +++- py-modindex.html | 4 +- search.html | 4 +- searchindex.js | 2 +- splunk.html | 4 +- usage.html | 4 +- 27 files changed, 255 insertions(+), 96 deletions(-) diff --git a/_modules/index.html b/_modules/index.html index 8a93a14..0028af1 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 9.8.0 documentation + Overview: module code — parsedmarc 9.9.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 2e9e709..9c7847d 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 9.8.0 documentation + parsedmarc — parsedmarc 9.9.0 documentation - + @@ -1233,6 +1233,9 @@ row["source_base_domain"] = record["source"]["base_domain"] row["source_name"] = record["source"]["name"] row["source_type"] = record["source"]["type"] + row["source_asn"] = record["source"]["asn"] + row["source_asn_name"] = record["source"]["asn_name"] + row["source_asn_domain"] = record["source"]["asn_domain"] row["count"] = record["count"] row["spf_aligned"] = record["alignment"]["spf"] row["dkim_aligned"] = record["alignment"]["dkim"] @@ -1327,6 +1330,9 @@ "source_base_domain", "source_name", "source_type", + "source_asn", + "source_asn_name", + "source_asn_domain", "count", "spf_aligned", "dkim_aligned", @@ -1534,6 +1540,9 @@ row["source_base_domain"] = report["source"]["base_domain"] row["source_name"] = report["source"]["name"] row["source_type"] = report["source"]["type"] + row["source_asn"] = report["source"]["asn"] + row["source_asn_name"] = report["source"]["asn_name"] + row["source_asn_domain"] = report["source"]["asn_domain"] row["source_country"] = report["source"]["country"] del row["source"] row["subject"] = report["parsed_sample"].get("subject") @@ -1582,6 +1591,9 @@ "source_base_domain", "source_name", "source_type", + "source_asn", + "source_asn_name", + "source_asn_domain", "delivery_result", "auth_failure", "reported_domain", diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 9f19679..35af78d 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 9.8.0 documentation + parsedmarc.elastic — parsedmarc 9.9.0 documentation - + @@ -164,6 +164,9 @@ source_base_domain = Text() source_type = Text() source_name = Text() + source_asn = Integer() + source_asn_name = Text() + source_asn_domain = Text() message_count = Integer disposition = Text() dkim_aligned = Boolean() @@ -258,6 +261,9 @@ source_ip_address = Ip() source_country = Text() source_reverse_dns = Text() + source_asn = Integer() + source_asn_name = Text() + source_asn_domain = Text() source_authentication_mechanisms = Text() source_auth_failures = Text() dkim_domain = Text() @@ -588,6 +594,9 @@ source_base_domain=record["source"]["base_domain"], source_type=record["source"]["type"], source_name=record["source"]["name"], + source_asn=record["source"]["asn"], + source_asn_name=record["source"]["asn_name"], + source_asn_domain=record["source"]["asn_domain"], message_count=record["count"], disposition=record["policy_evaluated"]["disposition"], dkim_aligned=record["policy_evaluated"]["dkim"] is not None @@ -775,6 +784,9 @@ source_country=forensic_report["source"]["country"], source_reverse_dns=forensic_report["source"]["reverse_dns"], source_base_domain=forensic_report["source"]["base_domain"], + source_asn=forensic_report["source"]["asn"], + source_asn_name=forensic_report["source"]["asn_name"], + source_asn_domain=forensic_report["source"]["asn_domain"], authentication_mechanisms=forensic_report["authentication_mechanisms"], auth_failure=forensic_report["auth_failure"], dkim_domain=forensic_report["dkim_domain"], diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 0bf04c2..1cb0b58 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 9.8.0 documentation + parsedmarc.opensearch — parsedmarc 9.9.0 documentation - + @@ -167,6 +167,9 @@ source_base_domain = Text() source_type = Text() source_name = Text() + source_asn = Integer() + source_asn_name = Text() + source_asn_domain = Text() message_count = Integer disposition = Text() dkim_aligned = Boolean() @@ -261,6 +264,9 @@ source_ip_address = Ip() source_country = Text() source_reverse_dns = Text() + source_asn = Integer() + source_asn_name = Text() + source_asn_domain = Text() source_authentication_mechanisms = Text() source_auth_failures = Text() dkim_domain = Text() @@ -618,6 +624,9 @@ source_base_domain=record["source"]["base_domain"], source_type=record["source"]["type"], source_name=record["source"]["name"], + source_asn=record["source"]["asn"], + source_asn_name=record["source"]["asn_name"], + source_asn_domain=record["source"]["asn_domain"], message_count=record["count"], disposition=record["policy_evaluated"]["disposition"], dkim_aligned=record["policy_evaluated"]["dkim"] is not None @@ -805,6 +814,9 @@ source_country=forensic_report["source"]["country"], source_reverse_dns=forensic_report["source"]["reverse_dns"], source_base_domain=forensic_report["source"]["base_domain"], + source_asn=forensic_report["source"]["asn"], + source_asn_name=forensic_report["source"]["asn_name"], + source_asn_domain=forensic_report["source"]["asn_domain"], authentication_mechanisms=forensic_report["authentication_mechanisms"], auth_failure=forensic_report["auth_failure"], dkim_domain=forensic_report["dkim_domain"], diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 9609c51..40cbb2e 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 9.8.0 documentation + parsedmarc.splunk — parsedmarc 9.9.0 documentation - + @@ -193,6 +193,9 @@ new_report["source_base_domain"] = record["source"]["base_domain"] new_report["source_type"] = record["source"]["type"] new_report["source_name"] = record["source"]["name"] + new_report["source_asn"] = record["source"]["asn"] + new_report["source_asn_name"] = record["source"]["asn_name"] + new_report["source_asn_domain"] = record["source"]["asn_domain"] new_report["message_count"] = record["count"] new_report["disposition"] = record["policy_evaluated"]["disposition"] new_report["spf_aligned"] = record["alignment"]["spf"] diff --git a/_modules/parsedmarc/types.html b/_modules/parsedmarc/types.html index 5ce947e..816ae73 100644 --- a/_modules/parsedmarc/types.html +++ b/_modules/parsedmarc/types.html @@ -5,14 +5,14 @@ - parsedmarc.types — parsedmarc 9.8.0 documentation + parsedmarc.types — parsedmarc 9.9.0 documentation - + @@ -129,7 +129,10 @@ reverse_dns: Optional[str] base_domain: Optional[str] name: Optional[str] - type: Optional[str] + type: Optional[str] + asn: Optional[int] + asn_name: Optional[str] + asn_domain: Optional[str] diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index df777f4..ed0935e 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 9.8.0 documentation + parsedmarc.utils — parsedmarc 9.9.0 documentation - + @@ -246,7 +246,10 @@ country: Optional[str] base_domain: Optional[str] name: Optional[str] - type: Optional[str] + type: Optional[str] + asn: Optional[int] + asn_name: Optional[str] + asn_domain: Optional[str] @@ -581,22 +584,7 @@ -
-[docs] -def get_ip_address_country( - ip_address: str, *, db_path: Optional[str] = None -) -> Optional[str]: - """ - Returns the ISO code for the country associated - with the given IPv4 or IPv6 address - - Args: - ip_address (str): The IP address to query for - db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP - - Returns: - str: And ISO country code associated with the given IP address - """ +def _get_ip_database_path(db_path: Optional[str]) -> str: db_paths = [ "ipinfo_lite.mmdb", "GeoLite2-Country.mmdb", @@ -612,14 +600,13 @@ "dbip-country.mmdb", ] - if db_path is not None: - if not os.path.isfile(db_path): - logger.warning( - f"No file exists at {db_path}. Falling back to an " - "included copy of the IPinfo IP to Country " - "Lite database." - ) - db_path = None + if db_path is not None and not os.path.isfile(db_path): + logger.warning( + f"No file exists at {db_path}. Falling back to an " + "included copy of the IPinfo IP to Country " + "Lite database." + ) + db_path = None if db_path is None: for system_path in db_paths: @@ -639,14 +626,39 @@ if db_age > timedelta(days=30): logger.warning("IP database is more than a month old") - db_reader = maxminddb.open_database(db_path) + return db_path + + +class _IPDatabaseRecord(TypedDict): + country: Optional[str] + asn: Optional[int] + asn_name: Optional[str] + asn_domain: Optional[str] + + +
+[docs] +def get_ip_address_db_record( + ip_address: str, *, db_path: Optional[str] = None +) -> _IPDatabaseRecord: + """Look up an IP in the configured MMDB and return country + ASN fields. + + IPinfo Lite carries ``country_code``, ``as_name``, and ``as_domain`` on + every record. MaxMind/DBIP country-only databases carry only country, so + ``asn_name`` / ``asn_domain`` come back None for those users. + """ + resolved_path = _get_ip_database_path(db_path) + db_reader = maxminddb.open_database(resolved_path) record = db_reader.get(ip_address) - # Support both the IPinfo schema (flat top-level ``country_code``) and the - # MaxMind/DBIP schema (nested ``country.iso_code``) so users dropping in - # their own MMDB from any of these providers keeps working. country: Optional[str] = None + asn: Optional[int] = None + asn_name: Optional[str] = None + asn_domain: Optional[str] = None if isinstance(record, dict): + # Support both the IPinfo schema (flat top-level ``country_code``) and + # the MaxMind/DBIP schema (nested ``country.iso_code``) so users + # dropping in their own MMDB from any of these providers keeps working. code = record.get("country_code") if code is None: nested = record.get("country") @@ -655,7 +667,55 @@ if isinstance(code, str): country = code - return country
+ # Normalize ASN to a plain integer. IPinfo stores it as a string like + # "AS15169"; MaxMind's ASN DB uses ``autonomous_system_number`` as an + # int. Integer form lets consumers do range queries and sort + # numerically; display-time formatting with an "AS" prefix is trivial. + raw_asn = record.get("asn") + if isinstance(raw_asn, int): + asn = raw_asn + elif isinstance(raw_asn, str) and raw_asn: + digits = raw_asn.removeprefix("AS").removeprefix("as") + if digits.isdigit(): + asn = int(digits) + if asn is None: + mm_asn = record.get("autonomous_system_number") + if isinstance(mm_asn, int): + asn = mm_asn + + name = record.get("as_name") or record.get("autonomous_system_organization") + if isinstance(name, str) and name: + asn_name = name + domain = record.get("as_domain") + if isinstance(domain, str) and domain: + asn_domain = domain.lower() + + return { + "country": country, + "asn": asn, + "asn_name": asn_name, + "asn_domain": asn_domain, + }
+ + + +
+[docs] +def get_ip_address_country( + ip_address: str, *, db_path: Optional[str] = None +) -> Optional[str]: + """ + Returns the ISO code for the country associated + with the given IPv4 or IPv6 address. + + Args: + ip_address (str): The IP address to query for + db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP + + Returns: + str: And ISO country code associated with the given IP address + """ + return get_ip_address_db_record(ip_address, db_path=db_path)["country"]
@@ -858,6 +918,9 @@ "base_domain": None, "name": None, "type": None, + "asn": None, + "asn_name": None, + "asn_domain": None, } if offline: reverse_dns = None @@ -868,9 +931,13 @@ timeout=timeout, retries=retries, ) - country = get_ip_address_country(ip_address, db_path=ip_db_path) - info["country"] = country + db_record = get_ip_address_db_record(ip_address, db_path=ip_db_path) + info["country"] = db_record["country"] + info["asn"] = db_record["asn"] + info["asn_name"] = db_record["asn_name"] + info["asn_domain"] = db_record["asn_domain"] info["reverse_dns"] = reverse_dns + if reverse_dns is not None: base_domain = get_base_domain(reverse_dns) if base_domain is not None: @@ -885,12 +952,34 @@ info["base_domain"] = base_domain info["type"] = service["type"] info["name"] = service["name"] - - if cache is not None: - cache[ip_address] = info - logger.debug(f"IP address {ip_address} added to cache") else: logger.debug(f"IP address {ip_address} reverse_dns not found") + # Fall back to ASN data for source attribution. ``reverse_dns`` and + # ``base_domain`` are left null so consumers can still tell an + # ASN-derived row apart from one resolved via a real PTR. + map_value: ReverseDNSMap = ( + reverse_dns_map if reverse_dns_map is not None else {} + ) + if len(map_value) == 0: + load_reverse_dns_map( + map_value, + always_use_local_file=always_use_local_files, + local_file_path=reverse_dns_map_path, + url=reverse_dns_map_url, + offline=offline, + ) + if info["asn_domain"] and info["asn_domain"] in map_value: + service = map_value[info["asn_domain"]] + info["name"] = service["name"] + info["type"] = service["type"] + elif info["asn_name"]: + # ASN-domain not in the map: surface the raw AS name with no + # classification. Better than leaving the row unattributed. + info["name"] = info["asn_name"] + + if cache is not None: + cache[ip_address] = info + logger.debug(f"IP address {ip_address} added to cache") return info diff --git a/_sources/output.md.txt b/_sources/output.md.txt index a8d19e4..bc73403 100644 --- a/_sources/output.md.txt +++ b/_sources/output.md.txt @@ -44,7 +44,10 @@ of the report schema. "reverse_dns": null, "base_domain": null, "name": null, - "type": null + "type": null, + "asn": 7018, + "asn_name": "AT&T Services, Inc.", + "asn_domain": "att.com" }, "count": 2, "alignment": { @@ -90,7 +93,7 @@ of the report schema. ### CSV aggregate report ```text -xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,normalized_timespan,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results +xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,normalized_timespan,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_asn_name,source_asn_domain,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass @@ -123,7 +126,12 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized "ip_address": "10.10.10.10", "country": null, "reverse_dns": null, - "base_domain": null + "base_domain": null, + "name": null, + "type": null, + "asn": null, + "asn_name": null, + "asn_domain": null }, "authentication_mechanisms": [], "original_envelope_id": null, @@ -193,7 +201,7 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized ### CSV forensic report ```text -feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only +feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_asn_name,source_asn_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False ``` @@ -238,4 +246,4 @@ auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct ] } ] -``` \ No newline at end of file +``` diff --git a/_static/documentation_options.js b/_static/documentation_options.js index a795c44..ec09df0 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '9.8.0', + VERSION: '9.9.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 42d054a..96fd838 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 9.8.0 documentation + API reference — parsedmarc 9.9.0 documentation - + @@ -153,6 +153,7 @@
  • get_base_domain()
  • get_filename_safe_string()
  • get_ip_address_country()
    • +
  • get_ip_address_db_record()
  • get_ip_address_info()
  • get_reverse_dns()
  • get_service_from_reverse_dns_base_domain()
    • @@ -1267,7 +1268,7 @@ parsedmarc.resources.maps.psl_overrides.txt

    parsedmarc.utils.get_ip_address_country(ip_address: str, *, db_path: str | None = None) → str | None[source]

    Returns the ISO code for the country associated -with the given IPv4 or IPv6 address

    +with the given IPv4 or IPv6 address.

    Parameters:
      @@ -1284,6 +1285,15 @@ with the given IPv4 or IPv6 address

    +
    +
    +parsedmarc.utils.get_ip_address_db_record(ip_address: str, *, db_path: str | None = None) → _IPDatabaseRecord[source]
    +

    Look up an IP in the configured MMDB and return country + ASN fields.

    +

    IPinfo Lite carries country_code, as_name, and as_domain on +every record. MaxMind/DBIP country-only databases carry only country, so +asn_name / asn_domain come back None for those users.

    +
    +
    parsedmarc.utils.get_ip_address_info(ip_address, *, ip_db_path: str | None = None, reverse_dns_map_path: str | None = None, always_use_local_files: bool = False, reverse_dns_map_url: str | None = None, cache: ExpiringDict | None = None, reverse_dns_map: dict[str, ReverseDNSService] | None = None, offline: bool = False, nameservers: list[str] | None = None, timeout: float = 2.0, retries: int = 0) → IPAddressInfo[source]
    diff --git a/contributing.html b/contributing.html index c866ebf..be441d2 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 9.8.0 documentation + Contributing to parsedmarc — parsedmarc 9.9.0 documentation - + diff --git a/davmail.html b/davmail.html index 4281229..9dbe383 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 9.8.0 documentation + Accessing an inbox using OWA/EWS — parsedmarc 9.9.0 documentation - + diff --git a/dmarc.html b/dmarc.html index 44bbeec..296ecf0 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 9.8.0 documentation + Understanding DMARC — parsedmarc 9.9.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index ca6ceb9..d1900ae 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 9.8.0 documentation + Elasticsearch and Kibana — parsedmarc 9.9.0 documentation - + diff --git a/genindex.html b/genindex.html index aeeef99..32c671a 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 9.8.0 documentation + Index — parsedmarc 9.9.0 documentation - + @@ -208,10 +208,12 @@
  • get_dmarc_reports_from_mbox() (in module parsedmarc)
  • get_filename_safe_string() (in module parsedmarc.utils) +
    • +
  • get_ip_address_country() (in module parsedmarc.utils)