diff --git a/_modules/index.html b/_modules/index.html index 0028af1..db3fc82 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 9.9.0 documentation + Overview: module code — parsedmarc 9.10.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 9c7847d..46d9a3b 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 9.9.0 documentation + parsedmarc — parsedmarc 9.10.0 documentation - + @@ -1234,8 +1234,8 @@ row["source_name"] = record["source"]["name"] row["source_type"] = record["source"]["type"] row["source_asn"] = record["source"]["asn"] - row["source_asn_name"] = record["source"]["asn_name"] - row["source_asn_domain"] = record["source"]["asn_domain"] + row["source_as_name"] = record["source"]["as_name"] + row["source_as_domain"] = record["source"]["as_domain"] row["count"] = record["count"] row["spf_aligned"] = record["alignment"]["spf"] row["dkim_aligned"] = record["alignment"]["dkim"] @@ -1331,8 +1331,8 @@ "source_name", "source_type", "source_asn", - "source_asn_name", - "source_asn_domain", + "source_as_name", + "source_as_domain", "count", "spf_aligned", "dkim_aligned", @@ -1541,8 +1541,8 @@ row["source_name"] = report["source"]["name"] row["source_type"] = report["source"]["type"] row["source_asn"] = report["source"]["asn"] - row["source_asn_name"] = report["source"]["asn_name"] - row["source_asn_domain"] = report["source"]["asn_domain"] + row["source_as_name"] = report["source"]["as_name"] + row["source_as_domain"] = report["source"]["as_domain"] row["source_country"] = report["source"]["country"] del row["source"] row["subject"] = report["parsed_sample"].get("subject") @@ -1592,8 +1592,8 @@ "source_name", "source_type", "source_asn", - "source_asn_name", - "source_asn_domain", + "source_as_name", + "source_as_domain", "delivery_result", "auth_failure", "reported_domain", diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 35af78d..8b509e4 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 9.9.0 documentation + parsedmarc.elastic — parsedmarc 9.10.0 documentation - + @@ -165,8 +165,8 @@ source_type = Text() source_name = Text() source_asn = Integer() - source_asn_name = Text() - source_asn_domain = Text() + source_as_name = Text() + source_as_domain = Text() message_count = Integer disposition = Text() dkim_aligned = Boolean() @@ -262,8 +262,8 @@ source_country = Text() source_reverse_dns = Text() source_asn = Integer() - source_asn_name = Text() - source_asn_domain = Text() + source_as_name = Text() + source_as_domain = Text() source_authentication_mechanisms = Text() source_auth_failures = Text() dkim_domain = Text() @@ -595,8 +595,8 @@ source_type=record["source"]["type"], source_name=record["source"]["name"], source_asn=record["source"]["asn"], - source_asn_name=record["source"]["asn_name"], - source_asn_domain=record["source"]["asn_domain"], + source_as_name=record["source"]["as_name"], + source_as_domain=record["source"]["as_domain"], message_count=record["count"], disposition=record["policy_evaluated"]["disposition"], dkim_aligned=record["policy_evaluated"]["dkim"] is not None @@ -785,8 +785,8 @@ source_reverse_dns=forensic_report["source"]["reverse_dns"], source_base_domain=forensic_report["source"]["base_domain"], source_asn=forensic_report["source"]["asn"], - source_asn_name=forensic_report["source"]["asn_name"], - source_asn_domain=forensic_report["source"]["asn_domain"], + source_as_name=forensic_report["source"]["as_name"], + source_as_domain=forensic_report["source"]["as_domain"], authentication_mechanisms=forensic_report["authentication_mechanisms"], auth_failure=forensic_report["auth_failure"], dkim_domain=forensic_report["dkim_domain"], diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 1cb0b58..fce2af7 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 9.9.0 documentation + parsedmarc.opensearch — parsedmarc 9.10.0 documentation - + @@ -168,8 +168,8 @@ source_type = Text() source_name = Text() source_asn = Integer() - source_asn_name = Text() - source_asn_domain = Text() + source_as_name = Text() + source_as_domain = Text() message_count = Integer disposition = Text() dkim_aligned = Boolean() @@ -265,8 +265,8 @@ source_country = Text() source_reverse_dns = Text() source_asn = Integer() - source_asn_name = Text() - source_asn_domain = Text() + source_as_name = Text() + source_as_domain = Text() source_authentication_mechanisms = Text() source_auth_failures = Text() dkim_domain = Text() @@ -625,8 +625,8 @@ source_type=record["source"]["type"], source_name=record["source"]["name"], source_asn=record["source"]["asn"], - source_asn_name=record["source"]["asn_name"], - source_asn_domain=record["source"]["asn_domain"], + source_as_name=record["source"]["as_name"], + source_as_domain=record["source"]["as_domain"], message_count=record["count"], disposition=record["policy_evaluated"]["disposition"], dkim_aligned=record["policy_evaluated"]["dkim"] is not None @@ -815,8 +815,8 @@ source_reverse_dns=forensic_report["source"]["reverse_dns"], source_base_domain=forensic_report["source"]["base_domain"], source_asn=forensic_report["source"]["asn"], - source_asn_name=forensic_report["source"]["asn_name"], - source_asn_domain=forensic_report["source"]["asn_domain"], + source_as_name=forensic_report["source"]["as_name"], + source_as_domain=forensic_report["source"]["as_domain"], authentication_mechanisms=forensic_report["authentication_mechanisms"], auth_failure=forensic_report["auth_failure"], dkim_domain=forensic_report["dkim_domain"], diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 40cbb2e..2ffd2dc 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 9.9.0 documentation + parsedmarc.splunk — parsedmarc 9.10.0 documentation - + @@ -194,8 +194,8 @@ new_report["source_type"] = record["source"]["type"] new_report["source_name"] = record["source"]["name"] new_report["source_asn"] = record["source"]["asn"] - new_report["source_asn_name"] = record["source"]["asn_name"] - new_report["source_asn_domain"] = record["source"]["asn_domain"] + new_report["source_as_name"] = record["source"]["as_name"] + new_report["source_as_domain"] = record["source"]["as_domain"] new_report["message_count"] = record["count"] new_report["disposition"] = record["policy_evaluated"]["disposition"] new_report["spf_aligned"] = record["alignment"]["spf"] diff --git a/_modules/parsedmarc/types.html b/_modules/parsedmarc/types.html index 816ae73..3c96b90 100644 --- a/_modules/parsedmarc/types.html +++ b/_modules/parsedmarc/types.html @@ -5,14 +5,14 @@ - parsedmarc.types — parsedmarc 9.9.0 documentation + parsedmarc.types — parsedmarc 9.10.0 documentation - + @@ -131,8 +131,8 @@ name: Optional[str] type: Optional[str] asn: Optional[int] - asn_name: Optional[str] - asn_domain: Optional[str] + as_name: Optional[str] + as_domain: Optional[str] diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index ed0935e..4dbd491 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 9.9.0 documentation + parsedmarc.utils — parsedmarc 9.10.0 documentation - + @@ -98,6 +98,7 @@ import shutil import subprocess import tempfile +import time from datetime import datetime, timedelta, timezone from typing import Optional, TypedDict, Union, cast @@ -248,8 +249,8 @@ name: Optional[str] type: Optional[str] asn: Optional[int] - asn_name: Optional[str] - asn_domain: Optional[str] + as_name: Optional[str] + as_domain: Optional[str] @@ -584,6 +585,328 @@ +class _IPDatabaseRecord(TypedDict): + country: Optional[str] + asn: Optional[int] + as_name: Optional[str] + as_domain: Optional[str] + + +
+[docs] +class InvalidIPinfoAPIKey(Exception): + """Raised when the IPinfo API rejects the configured token."""
+ + + +# IPinfo Lite REST API. When ``_IPINFO_API_TOKEN`` is set, ``get_ip_address_db_record()`` +# queries the API first and falls through to the bundled/cached MMDB only on +# rate-limit/quota/network errors. A 401/403 on any lookup propagates as +# ``InvalidIPinfoAPIKey`` so the CLI exits fatally; callers of the library +# should catch it. +_IPINFO_API_URL = "https://api.ipinfo.io/lite" +# Account-info / quota endpoint. Separate from the lookup URL because ``/me`` +# lives at the ipinfo.io root, not under ``/lite``. Hitting it at startup +# both validates the token and surfaces plan/usage details; IPinfo documents +# it as a quota-free meta endpoint. +_IPINFO_ACCOUNT_URL = "https://ipinfo.io/me" +_IPINFO_API_TOKEN: Optional[str] = None +_IPINFO_API_TIMEOUT: float = 5.0 +# Default cooldowns when the API returns 429/402 without a ``Retry-After`` +# header. Rate limits are usually short; quota resets (402) are typically at a +# day/month boundary, so we pick a longer default there. +_IPINFO_API_RATE_LIMIT_COOLDOWN_SECONDS: float = 300.0 +_IPINFO_API_QUOTA_COOLDOWN_SECONDS: float = 3600.0 +# Unix timestamp before which lookups skip the API and go straight to the +# MMDB. ``0`` means the API is currently available. +_IPINFO_API_COOLDOWN_UNTIL: float = 0.0 +# Latch for recovery logging: True while the API is in a rate-limited or +# quota-exhausted state, so the next successful lookup can log "recovered" +# exactly once per event. +_IPINFO_API_RATE_LIMITED: bool = False + + +
+[docs] +def configure_ipinfo_api( + token: Optional[str], + *, + probe: bool = True, +) -> None: + """Configure the IPinfo Lite REST API as the primary source for IP lookups. + + When a token is configured, ``get_ip_address_db_record()`` hits the API + first for every lookup and falls back to the MMDB on rate-limit, quota, or + network errors. An invalid token raises ``InvalidIPinfoAPIKey`` — the CLI + catches that and exits fatally. + + Args: + token: IPinfo API token. ``None`` or empty disables the API. + probe: If ``True``, verify the token by hitting ``/me`` (and, if that + is unreachable, by looking up ``1.1.1.1``). A 401/403 raises + ``InvalidIPinfoAPIKey``; other errors are logged and the token is + still accepted so per-request fallback can take over. + """ + global _IPINFO_API_TOKEN + global _IPINFO_API_COOLDOWN_UNTIL, _IPINFO_API_RATE_LIMITED + + _IPINFO_API_TOKEN = token or None + _IPINFO_API_COOLDOWN_UNTIL = 0.0 + _IPINFO_API_RATE_LIMITED = False + + if not _IPINFO_API_TOKEN: + return + + if probe: + # Verify the token. Any network/quota failure here is non-fatal — we + # still accept the token and let per-request fallback handle it — but + # an invalid-key response must fail fast so operators notice + # immediately instead of seeing silent MMDB-only lookups all day. + # + # The /me meta endpoint doubles as a free-of-quota token check and a + # plan/usage lookup, so we try it first. If /me is unreachable, fall + # back to a lookup of 1.1.1.1 to validate the token. + account: Optional[dict] = None + try: + account = _ipinfo_api_account_info() + except InvalidIPinfoAPIKey: + raise + except Exception as e: + logger.debug(f"IPinfo account info fetch failed: {e}") + + if account is not None: + summary = _format_ipinfo_account_summary(account) + if summary: + logger.info(f"IPinfo API configured — {summary}") + else: + logger.info("IPinfo API configured") + return + + try: + _ipinfo_api_lookup("1.1.1.1") + except InvalidIPinfoAPIKey: + raise + except Exception as e: + logger.warning(f"IPinfo API probe failed (will fall back per-request): {e}") + else: + logger.info("IPinfo API configured")
+ + + +def _ipinfo_api_account_info() -> Optional[dict]: + """Fetch the IPinfo ``/me`` account endpoint. + + Returns the parsed JSON dict on success, or ``None`` when the endpoint is + unreachable (network error, non-JSON body, non-2xx other than 401/403). + A 401/403 raises ``InvalidIPinfoAPIKey`` — this endpoint is the best way + to validate a token since it doesn't consume a lookup-quota unit. + """ + if not _IPINFO_API_TOKEN: + return None + headers = { + "User-Agent": USER_AGENT, + "Authorization": f"Bearer {_IPINFO_API_TOKEN}", + "Accept": "application/json", + } + response = requests.get( + _IPINFO_ACCOUNT_URL, headers=headers, timeout=_IPINFO_API_TIMEOUT + ) + if response.status_code in (401, 403): + raise InvalidIPinfoAPIKey( + f"IPinfo API rejected the configured token (HTTP {response.status_code})" + ) + if not response.ok: + logger.debug(f"IPinfo /me returned HTTP {response.status_code}") + return None + try: + payload = response.json() + except ValueError: + return None + return payload if isinstance(payload, dict) else None + + +def _format_ipinfo_account_summary(account: dict) -> Optional[str]: + """Render a short, log-friendly summary of the IPinfo /me response. + + Field names in /me have varied across IPinfo plan generations, so we + probe a few aliases rather than commit to one schema. If nothing + useful is present we return ``None`` and the caller falls back to a + generic "configured" message. + """ + plan = ( + account.get("plan") + or account.get("tier") + or account.get("token_type") + or account.get("type") + ) + limit = account.get("limit") or account.get("monthly_limit") + remaining = account.get("remaining") or account.get("requests_remaining") + used = account.get("month") or account.get("month_requests") or account.get("used") + + parts = [] + if plan: + parts.append(f"plan: {plan}") + if used is not None and limit: + parts.append(f"usage: {used}/{limit} this month") + elif limit: + parts.append(f"monthly limit: {limit}") + if remaining is not None: + parts.append(f"{remaining} remaining") + return ", ".join(parts) if parts else None + + +def _parse_retry_after(response, default_seconds: float) -> float: + """Parse an HTTP ``Retry-After`` header as seconds. + + Supports the delta-seconds form. HTTP-date form is rare enough for an API + client to ignore; we just fall back to the default. + """ + raw = response.headers.get("Retry-After") + if raw: + try: + return max(float(raw.strip()), 1.0) + except ValueError: + pass + return default_seconds + + +def _ipinfo_api_lookup(ip_address: str) -> Optional[_IPDatabaseRecord]: + """Look up an IP via the IPinfo Lite REST API. + + Returns the normalized record on success, or ``None`` when the API is + unavailable for any reason the caller should fall back from (network + error, 429 rate limit, 402 quota exhausted, malformed response). + + On 429/402 the API is put in a cooldown (using ``Retry-After`` when + present) so we stop hammering it, and we log once per event at warning + level. After the cooldown expires the next lookup retries transparently; + a successful retry logs "API recovered" once at info level so operators + can see service came back. + + Raises: + InvalidIPinfoAPIKey: on 401/403. Propagates to abort the run. + """ + global _IPINFO_API_COOLDOWN_UNTIL, _IPINFO_API_RATE_LIMITED + + if not _IPINFO_API_TOKEN: + return None + if _IPINFO_API_COOLDOWN_UNTIL and time.time() < _IPINFO_API_COOLDOWN_UNTIL: + return None + + url = f"{_IPINFO_API_URL}/{ip_address}" + headers = { + "User-Agent": USER_AGENT, + "Authorization": f"Bearer {_IPINFO_API_TOKEN}", + "Accept": "application/json", + } + try: + response = requests.get(url, headers=headers, timeout=_IPINFO_API_TIMEOUT) + except requests.exceptions.RequestException as e: + logger.debug(f"IPinfo API request for {ip_address} failed: {e}") + return None + + if response.status_code in (401, 403): + raise InvalidIPinfoAPIKey( + f"IPinfo API rejected the configured token (HTTP {response.status_code})" + ) + if response.status_code == 429: + cooldown = _parse_retry_after(response, _IPINFO_API_RATE_LIMIT_COOLDOWN_SECONDS) + _IPINFO_API_COOLDOWN_UNTIL = time.time() + cooldown + # First hit of a rate-limit event is visible at warning; subsequent + # 429s after cooldown-and-retry cycles stay at debug so we don't spam + # the log when a run spans a long quota reset. + if not _IPINFO_API_RATE_LIMITED: + logger.warning( + "IPinfo API rate limit hit; falling back to the local MMDB " + f"for {cooldown:.0f}s before retrying" + ) + _IPINFO_API_RATE_LIMITED = True + else: + logger.debug(f"IPinfo API still rate-limited; retry after {cooldown:.0f}s") + return None + if response.status_code == 402: + cooldown = _parse_retry_after(response, _IPINFO_API_QUOTA_COOLDOWN_SECONDS) + _IPINFO_API_COOLDOWN_UNTIL = time.time() + cooldown + if not _IPINFO_API_RATE_LIMITED: + logger.warning( + "IPinfo API quota exhausted; falling back to the local MMDB " + f"for {cooldown:.0f}s before retrying" + ) + _IPINFO_API_RATE_LIMITED = True + else: + logger.debug( + f"IPinfo API quota still exhausted; retry after {cooldown:.0f}s" + ) + return None + if not response.ok: + logger.debug( + f"IPinfo API returned HTTP {response.status_code} for {ip_address}" + ) + return None + + try: + payload = response.json() + except ValueError: + logger.debug(f"IPinfo API returned non-JSON for {ip_address}") + return None + if not isinstance(payload, dict): + return None + + if _IPINFO_API_RATE_LIMITED: + logger.info("IPinfo API recovered; resuming API lookups") + _IPINFO_API_RATE_LIMITED = False + _IPINFO_API_COOLDOWN_UNTIL = 0.0 + + return _normalize_ip_record(payload) + + +def _normalize_ip_record(record: dict) -> _IPDatabaseRecord: + """Normalize an IPinfo / MaxMind record to the internal shape. + + Shared between the API path and the MMDB path so both schemas produce the + same output: country as ISO code, ASN as plain int, as_name string, + as_domain lowercased. + """ + country: Optional[str] = None + asn: Optional[int] = None + as_name: Optional[str] = None + as_domain: Optional[str] = None + + code = record.get("country_code") + if code is None: + nested = record.get("country") + if isinstance(nested, dict): + code = nested.get("iso_code") + if isinstance(code, str): + country = code + + raw_asn = record.get("asn") + if isinstance(raw_asn, int): + asn = raw_asn + elif isinstance(raw_asn, str) and raw_asn: + digits = raw_asn.removeprefix("AS").removeprefix("as") + if digits.isdigit(): + asn = int(digits) + if asn is None: + mm_asn = record.get("autonomous_system_number") + if isinstance(mm_asn, int): + asn = mm_asn + + name = record.get("as_name") or record.get("autonomous_system_organization") + if isinstance(name, str) and name: + as_name = name + domain = record.get("as_domain") + if isinstance(domain, str) and domain: + as_domain = domain.lower() + + return { + "country": country, + "asn": asn, + "as_name": as_name, + "as_domain": as_domain, + } + + def _get_ip_database_path(db_path: Optional[str]) -> str: db_paths = [ "ipinfo_lite.mmdb", @@ -629,73 +952,37 @@ return db_path -class _IPDatabaseRecord(TypedDict): - country: Optional[str] - asn: Optional[int] - asn_name: Optional[str] - asn_domain: Optional[str] - -
[docs] def get_ip_address_db_record( ip_address: str, *, db_path: Optional[str] = None ) -> _IPDatabaseRecord: - """Look up an IP in the configured MMDB and return country + ASN fields. + """Look up an IP and return country + ASN fields. + + If the IPinfo Lite API is configured via ``configure_ipinfo_api()``, the + API is queried first; any non-fatal failure (rate limit, quota, network) + falls through to the MMDB. An invalid API token raises + ``InvalidIPinfoAPIKey`` and is not caught here. IPinfo Lite carries ``country_code``, ``as_name``, and ``as_domain`` on every record. MaxMind/DBIP country-only databases carry only country, so - ``asn_name`` / ``asn_domain`` come back None for those users. + ``as_name`` / ``as_domain`` come back None for those users. """ + api_record = _ipinfo_api_lookup(ip_address) + if api_record is not None: + return api_record + resolved_path = _get_ip_database_path(db_path) db_reader = maxminddb.open_database(resolved_path) record = db_reader.get(ip_address) - - country: Optional[str] = None - asn: Optional[int] = None - asn_name: Optional[str] = None - asn_domain: Optional[str] = None - if isinstance(record, dict): - # Support both the IPinfo schema (flat top-level ``country_code``) and - # the MaxMind/DBIP schema (nested ``country.iso_code``) so users - # dropping in their own MMDB from any of these providers keeps working. - code = record.get("country_code") - if code is None: - nested = record.get("country") - if isinstance(nested, dict): - code = nested.get("iso_code") - if isinstance(code, str): - country = code - - # Normalize ASN to a plain integer. IPinfo stores it as a string like - # "AS15169"; MaxMind's ASN DB uses ``autonomous_system_number`` as an - # int. Integer form lets consumers do range queries and sort - # numerically; display-time formatting with an "AS" prefix is trivial. - raw_asn = record.get("asn") - if isinstance(raw_asn, int): - asn = raw_asn - elif isinstance(raw_asn, str) and raw_asn: - digits = raw_asn.removeprefix("AS").removeprefix("as") - if digits.isdigit(): - asn = int(digits) - if asn is None: - mm_asn = record.get("autonomous_system_number") - if isinstance(mm_asn, int): - asn = mm_asn - - name = record.get("as_name") or record.get("autonomous_system_organization") - if isinstance(name, str) and name: - asn_name = name - domain = record.get("as_domain") - if isinstance(domain, str) and domain: - asn_domain = domain.lower() - - return { - "country": country, - "asn": asn, - "asn_name": asn_name, - "asn_domain": asn_domain, - }
+ if not isinstance(record, dict): + return { + "country": None, + "asn": None, + "as_name": None, + "as_domain": None, + } + return _normalize_ip_record(record) @@ -919,8 +1206,8 @@ "name": None, "type": None, "asn": None, - "asn_name": None, - "asn_domain": None, + "as_name": None, + "as_domain": None, } if offline: reverse_dns = None @@ -934,8 +1221,8 @@ db_record = get_ip_address_db_record(ip_address, db_path=ip_db_path) info["country"] = db_record["country"] info["asn"] = db_record["asn"] - info["asn_name"] = db_record["asn_name"] - info["asn_domain"] = db_record["asn_domain"] + info["as_name"] = db_record["as_name"] + info["as_domain"] = db_record["as_domain"] info["reverse_dns"] = reverse_dns if reverse_dns is not None: @@ -968,14 +1255,14 @@ url=reverse_dns_map_url, offline=offline, ) - if info["asn_domain"] and info["asn_domain"] in map_value: - service = map_value[info["asn_domain"]] + if info["as_domain"] and info["as_domain"] in map_value: + service = map_value[info["as_domain"]] info["name"] = service["name"] info["type"] = service["type"] - elif info["asn_name"]: + elif info["as_name"]: # ASN-domain not in the map: surface the raw AS name with no # classification. Better than leaving the row unattributed. - info["name"] = info["asn_name"] + info["name"] = info["as_name"] if cache is not None: cache[ip_address] = info diff --git a/_sources/output.md.txt b/_sources/output.md.txt index bc73403..095193b 100644 --- a/_sources/output.md.txt +++ b/_sources/output.md.txt @@ -46,8 +46,8 @@ of the report schema. "name": null, "type": null, "asn": 7018, - "asn_name": "AT&T Services, Inc.", - "asn_domain": "att.com" + "as_name": "AT&T Services, Inc.", + "as_domain": "att.com" }, "count": 2, "alignment": { @@ -93,7 +93,7 @@ of the report schema. ### CSV aggregate report ```text -xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,normalized_timespan,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_asn_name,source_asn_domain,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results +xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,normalized_timespan,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass @@ -130,8 +130,8 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized "name": null, "type": null, "asn": null, - "asn_name": null, - "asn_domain": null + "as_name": null, + "as_domain": null }, "authentication_mechanisms": [], "original_envelope_id": null, @@ -201,7 +201,7 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized ### CSV forensic report ```text -feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_asn_name,source_asn_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only +feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False ``` diff --git a/_sources/usage.md.txt b/_sources/usage.md.txt index 27d682e..cd9cd57 100644 --- a/_sources/usage.md.txt +++ b/_sources/usage.md.txt @@ -134,8 +134,17 @@ The full set of configuration options are: JSON output file - `ip_db_path` - str: An optional custom path to a MMDB file from IPinfo, MaxMind, or DBIP - - `ip_db_url` - str: Overrides the default download URL for the - IP-to-country database (env var: `PARSEDMARC_GENERAL_IP_DB_URL`) + - `ipinfo_url` - str: Overrides the default download URL for the + bundled IPinfo Lite MMDB (env var: + `PARSEDMARC_GENERAL_IPINFO_URL`). The pre-9.10 name `ip_db_url` is + still accepted as a deprecated alias and logs a warning. + - `ipinfo_api_token` - str: Optional [IPinfo Lite REST API] token. When + set, IP lookups hit the API first for the freshest country/ASN data + and fall back to the local MMDB on rate limit, quota exhaustion, or + network errors. An invalid token exits the process with a fatal error. + Ignored when `offline` is set. The Lite tier is free and has no + documented monthly request cap; see the IPinfo Lite docs for current + limits. (env var: `PARSEDMARC_GENERAL_IPINFO_API_TOKEN`) - `offline` - bool: Do not use online queries for geolocation or DNS. Also disables automatic downloading of the IP-to-country database and reverse DNS map. @@ -801,3 +810,4 @@ journalctl -u parsedmarc.service -r [cloudflare's public resolvers]: https://1.1.1.1/ [url encoded]: https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters +[ipinfo lite rest api]: https://ipinfo.io/developers/lite-api diff --git a/_static/documentation_options.js b/_static/documentation_options.js index ec09df0..4ef2c08 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '9.9.0', + VERSION: '9.10.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 96fd838..a7ed035 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 9.9.0 documentation + API reference — parsedmarc 9.10.0 documentation - + @@ -147,7 +147,9 @@
  • DownloadError
  • EmailParserError
  • IPAddressInfo
    • +
  • InvalidIPinfoAPIKey
  • ReverseDNSService
    • +
  • configure_ipinfo_api()
  • convert_outlook_msg()
  • decode_base64()
  • get_base_domain()
    • @@ -1187,11 +1189,38 @@ to save in Splunk

    class parsedmarc.utils.IPAddressInfo[source]
    +
    +
    +exception parsedmarc.utils.InvalidIPinfoAPIKey[source]
    +

    Raised when the IPinfo API rejects the configured token.

    +
    +
    class parsedmarc.utils.ReverseDNSService[source]
    +
    +
    +parsedmarc.utils.configure_ipinfo_api(token: str | None, *, probe: bool = True) → None[source]
    +

    Configure the IPinfo Lite REST API as the primary source for IP lookups.

    +

    When a token is configured, get_ip_address_db_record() hits the API +first for every lookup and falls back to the MMDB on rate-limit, quota, or +network errors. An invalid token raises InvalidIPinfoAPIKey — the CLI +catches that and exits fatally.

    +
    +
    Parameters:
    +
      +

    • token – IPinfo API token. None or empty disables the API.

      • +

    • probe – If True, verify the token by hitting /me (and, if that +is unreachable, by looking up 1.1.1.1). A 401/403 raises +InvalidIPinfoAPIKey; other errors are logged and the token is +still accepted so per-request fallback can take over.

      • +
    +
    +
    +
    +
    parsedmarc.utils.convert_outlook_msg(msg_bytes: bytes) → bytes[source]
    @@ -1288,10 +1317,14 @@ with the given IPv4 or IPv6 address.

    parsedmarc.utils.get_ip_address_db_record(ip_address: str, *, db_path: str | None = None) → _IPDatabaseRecord[source]
    -

    Look up an IP in the configured MMDB and return country + ASN fields.

    +

    Look up an IP and return country + ASN fields.

    +

    If the IPinfo Lite API is configured via configure_ipinfo_api(), the +API is queried first; any non-fatal failure (rate limit, quota, network) +falls through to the MMDB. An invalid API token raises +InvalidIPinfoAPIKey and is not caught here.

    IPinfo Lite carries country_code, as_name, and as_domain on every record. MaxMind/DBIP country-only databases carry only country, so -asn_name / asn_domain come back None for those users.

    +as_name / as_domain come back None for those users.

    diff --git a/contributing.html b/contributing.html index be441d2..18be11e 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 9.9.0 documentation + Contributing to parsedmarc — parsedmarc 9.10.0 documentation - + diff --git a/davmail.html b/davmail.html index 9dbe383..9b00cde 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 9.9.0 documentation + Accessing an inbox using OWA/EWS — parsedmarc 9.10.0 documentation - + diff --git a/dmarc.html b/dmarc.html index 296ecf0..e9be05b 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 9.9.0 documentation + Understanding DMARC — parsedmarc 9.10.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index d1900ae..10144b7 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 9.9.0 documentation + Elasticsearch and Kibana — parsedmarc 9.10.0 documentation - + diff --git a/genindex.html b/genindex.html index 32c671a..3363edc 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 9.9.0 documentation + Index — parsedmarc 9.10.0 documentation - + @@ -139,10 +139,12 @@