Add comprehensive TypedDicts to minimize Any usage in public APIs

Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com>

.vscode/settings.json

@@ -1,4 +1,14 @@
 {
+  "[python]": {
+    "editor.defaultFormatter": "charliermarsh.ruff",
+    "editor.formatOnSave": true,
+
+    // Let Ruff handle lint fixes + import sorting on save
+    "editor.codeActionsOnSave": {
+      "source.fixAll.ruff": "explicit",
+      "source.organizeImports.ruff": "explicit"
+    }
+  },
     "markdownlint.config": {
         "MD024": false
     },

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 9.0.5
+
+## Fixes
+
+- Fix report type detection introduced in `9.0.4`.
+
+## 9.0.4 (Yanked)
+
+### Fixes
+
+- Fix saving reports to OpenSearch ([#637](https://github.com/domainaware/parsedmarc/issues/637))
+- Fix parsing certain DMARC failure/forensic reports
+- Some fixes to type hints (incomplete, but published as-is due to the above bugs)
+
 ## 9.0.3
 
 ### Fixes

README.md

@@ -61,4 +61,4 @@ for RHEL or Debian.
 | 3.11    | ✅         | Actively maintained; supported until June 2028 (Debian 12) |
 | 3.12    | ✅         | Actively maintained; supported until May 2035 (RHEL 10)    |
 | 3.13    | ✅         | Actively maintained; supported until June 2030 (Debian 13) |
-| 3.14    | ❌         | Not currently supported due to [this Python bug](https://github.com/python/cpython/issues/142307)|
+| 3.14    | ❌         | Not currently supported due to Not currently supported due to [this imapclient bug](https://github.com/mjs/imapclient/issues/618)|

build.sh

@@ -9,7 +9,6 @@ fi
 . venv/bin/activate
 pip install .[build]
 ruff format .
-ruff check .
 cd docs
 make clean 
 make html

docs/source/index.md

@@ -61,7 +61,7 @@ for RHEL or Debian.
 | 3.11    | ✅         | Actively maintained; supported until June 2028 (Debian 12) |
 | 3.12    | ✅         | Actively maintained; supported until May 2035 (RHEL 10)    |
 | 3.13    | ✅         | Actively maintained; supported until June 2030 (Debian 13) |
-| 3.14    | ❌         | Not currently supported due to [this Python bug](https://github.com/python/cpython/issues/142307)|
+| 3.14    | ❌         | Not currently supported due to [this imapclient bug](https://github.com/mjs/imapclient/issues/618)|
 
 ```{toctree}
 :caption: 'Contents'

parsedmarc/__init__.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-from typing import Dict, List, Any, Union, Optional, IO, Callable
+from typing import Any, Union, Optional, IO, Callable, TypedDict, NotRequired
 
 import binascii
 import email
@@ -19,7 +19,6 @@ import xml.parsers.expat as expat
 import zipfile
 import zlib
 from base64 import b64decode
-from collections import OrderedDict
 from csv import DictWriter
 from datetime import datetime, timedelta, timezone, tzinfo
 from io import BytesIO, StringIO
@@ -57,6 +56,17 @@ MAGIC_GZIP = b"\x1f\x8b"
 MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20"
 MAGIC_JSON = b"\7b"
 
+EMAIL_SAMPLE_CONTENT_TYPES = (
+    "text/rfc822",
+    "text-rfc-822",
+    "text/rfc822-headers",
+    "text/rfc-822-headers",
+    "message/rfc822",
+    "message/rfc-822",
+    "message/rfc822-headers",
+    "message/rfc-822-headers",
+)
+
 IP_ADDRESS_CACHE = ExpiringDict(max_len=10000, max_age_seconds=14400)
 SEEN_AGGREGATE_REPORT_IDS = ExpiringDict(max_len=100000000, max_age_seconds=3600)
 REVERSE_DNS_MAP = dict()
@@ -82,11 +92,188 @@ class InvalidForensicReport(InvalidDMARCReport):
     """Raised when an invalid DMARC forensic report is encountered"""
 
 
+# TypedDict definitions for structured data
+class DateIntervalBucket(TypedDict):
+    """Represents a time bucket for interval normalization"""
+    begin: datetime
+    end: datetime
+    count: int
+
+
+class IPAddressInfo(TypedDict, total=False):
+    """Information about an IP address"""
+    ip_address: str
+    country: NotRequired[Optional[str]]
+    reverse_dns: NotRequired[Optional[str]]
+    base_domain: NotRequired[Optional[str]]
+    type: NotRequired[Optional[str]]
+    name: NotRequired[Optional[str]]
+
+
+class AlignmentInfo(TypedDict):
+    """DMARC alignment information"""
+    spf: bool
+    dkim: bool
+    dmarc: bool
+
+
+class PolicyOverrideReason(TypedDict, total=False):
+    """DMARC policy override reason"""
+    type: NotRequired[str]
+    comment: NotRequired[Optional[str]]
+
+
+class PolicyEvaluated(TypedDict):
+    """DMARC policy evaluation result"""
+    disposition: str
+    dkim: str
+    spf: str
+    policy_override_reasons: list[PolicyOverrideReason]
+
+
+class DKIMAuthResult(TypedDict):
+    """DKIM authentication result"""
+    domain: str
+    selector: str
+    result: str
+
+
+class SPFAuthResult(TypedDict):
+    """SPF authentication result"""
+    domain: str
+    scope: str
+    result: str
+
+
+class AuthResults(TypedDict):
+    """Authentication results for DKIM and SPF"""
+    dkim: list[DKIMAuthResult]
+    spf: list[SPFAuthResult]
+
+
+class Identifiers(TypedDict):
+    """Message identifiers"""
+    header_from: str
+    envelope_from: Optional[str]
+    envelope_to: Optional[str]
+
+
+class ParsedReportRecord(TypedDict):
+    """A parsed DMARC aggregate report record"""
+    source: IPAddressInfo
+    count: int
+    alignment: AlignmentInfo
+    policy_evaluated: PolicyEvaluated
+    identifiers: Identifiers
+    auth_results: AuthResults
+
+
+class ParsedReportRecordWithMetadata(ParsedReportRecord, total=False):
+    """A parsed DMARC report record with normalization metadata"""
+    normalized_timespan: bool
+    interval_begin: NotRequired[str]
+    interval_end: NotRequired[str]
+
+
+class ReportMetadata(TypedDict, total=False):
+    """DMARC report metadata"""
+    org_name: str
+    org_email: NotRequired[Optional[str]]
+    org_extra_contact_info: NotRequired[Optional[str]]
+    report_id: str
+    begin_date: str
+    end_date: str
+    errors: NotRequired[list[str]]
+
+
+class PolicyPublished(TypedDict, total=False):
+    """DMARC policy as published in DNS"""
+    domain: str
+    adkim: NotRequired[str]
+    aspf: NotRequired[str]
+    p: str
+    sp: NotRequired[str]
+    pct: NotRequired[str]
+    fo: NotRequired[str]
+
+
+class ParsedAggregateReport(TypedDict):
+    """A complete parsed DMARC aggregate report"""
+    xml_schema: str
+    report_metadata: ReportMetadata
+    policy_published: PolicyPublished
+    records: list[ParsedReportRecord]
+
+
+class SMTPTLSFailureDetails(TypedDict):
+    """SMTP TLS failure details"""
+    result_type: str
+    sending_mta_ip: NotRequired[Optional[str]]
+    receiving_mx_hostname: NotRequired[Optional[str]]
+    receiving_mx_helo: NotRequired[Optional[str]]
+    receiving_ip: NotRequired[Optional[str]]
+    failed_session_count: int
+    additional_information: NotRequired[Optional[str]]
+    failure_reason_code: NotRequired[Optional[str]]
+
+
+class SMTPTLSPolicy(TypedDict, total=False):
+    """SMTP TLS policy information"""
+    policy_type: str
+    policy_domain: str
+    policy_strings: NotRequired[list[str]]
+    mx_host_patterns: NotRequired[list[str]]
+    successful_session_count: int
+    failed_session_count: int
+    failure_details: NotRequired[list[SMTPTLSFailureDetails]]
+
+
+class ParsedSMTPTLSReport(TypedDict):
+    """A complete parsed SMTP TLS report"""
+    organization_name: str
+    begin_date: str
+    end_date: str
+    contact_info: str
+    report_id: str
+    policies: list[SMTPTLSPolicy]
+
+
+class ParsedForensicReport(TypedDict, total=False):
+    """A parsed DMARC forensic report"""
+    feedback_type: str
+    user_agent: NotRequired[Optional[str]]
+    version: NotRequired[Optional[str]]
+    original_envelope_id: NotRequired[Optional[str]]
+    original_mail_from: NotRequired[Optional[str]]
+    original_rcpt_to: NotRequired[Optional[str]]
+    arrival_date: str
+    arrival_date_utc: str
+    subject: NotRequired[str]
+    message_id: str
+    authentication_results: str
+    delivery_result: str
+    auth_failure: list[str]
+    reported_domain: str
+    arrival_date_utc: str
+    source: IPAddressInfo
+    authentication_mechanisms: list[str]
+    dkim_domain: NotRequired[Optional[str]]
+    sample_headers_only: bool
+    sample: NotRequired[str]
+    parsed_sample: NotRequired[dict[str, Any]]
+
+
+class ReportTypeWrapper(TypedDict):
+    """Wrapper for report type identification"""
+    report_type: str
+    report: Union[ParsedAggregateReport, ParsedForensicReport, ParsedSMTPTLSReport]
+
+
 def _bucket_interval_by_day(
     begin: datetime,
     end: datetime,
     total_count: int,
-) -> List[Dict[Any]]:
+) -> list[DateIntervalBucket]:
     """
     Split the interval [begin, end) into daily buckets and distribute
     `total_count` proportionally across those buckets.
@@ -148,7 +335,7 @@ def _bucket_interval_by_day(
     if day_cursor > begin:
         day_cursor -= timedelta(days=1)
 
-    day_buckets: List[Dict[str, Any]] = []
+    day_buckets: list[dict[str, Any]] = []
 
     while day_cursor < end:
         day_start = day_cursor
@@ -180,12 +367,12 @@ def _bucket_interval_by_day(
     # Then apply a "largest remainder" rounding strategy to ensure the sum
     # equals exactly total_count.
 
-    exact_values: List[float] = [
+    exact_values: list[float] = [
         (b["seconds"] / interval_seconds) * total_count for b in day_buckets
     ]
 
-    floor_values: List[int] = [int(x) for x in exact_values]
-    fractional_parts: List[float] = [x - int(x) for x in exact_values]
+    floor_values: list[int] = [int(x) for x in exact_values]
+    fractional_parts: list[float] = [x - int(x) for x in exact_values]
 
     # How many counts do we still need to distribute after flooring?
     remainder = total_count - sum(floor_values)
@@ -205,7 +392,7 @@ def _bucket_interval_by_day(
         final_counts[idx] += 1
 
     # --- Step 3: Build the final per-day result list -------------------------
-    results: List[Dict[str, Any]] = []
+    results: list[DateIntervalBucket] = []
     for bucket, count in zip(day_buckets, final_counts):
         if count > 0:
             results.append(
@@ -220,8 +407,8 @@ def _bucket_interval_by_day(
 
 
 def _append_parsed_record(
-    parsed_record: OrderedDict[str, Any],
-    records: OrderedDict[str, Any],
+    parsed_record: dict[str, Any],
+    records: list[dict[str, Any]],
     begin_dt: datetime,
     end_dt: datetime,
     normalize: bool,
@@ -264,22 +451,22 @@ def _append_parsed_record(
 
 
 def _parse_report_record(
-    record: OrderedDict,
+    record: dict[str, Any],
     *,
     ip_db_path: Optional[str] = None,
-    always_use_local_files: bool = False,
+    always_use_local_files: Optional[bool] = False,
     reverse_dns_map_path: Optional[str] = None,
     reverse_dns_map_url: Optional[str] = None,
-    offline: bool = False,
+    offline: Optional[bool] = False,
     nameservers: Optional[list[str]] = None,
     dns_timeout: Optional[float] = 2.0,
-) -> OrderedDict[str, Any]:
+) -> dict[str, Any]:
     """
     Converts a record from a DMARC aggregate report into a more consistent
     format
 
     Args:
-        record (OrderedDict): The record to convert
+        record (dict): The record to convert
         always_use_local_files (bool): Do not download files
         reverse_dns_map_path (str): Path to a reverse DNS map file
         reverse_dns_map_url (str): URL to a reverse DNS map file
@@ -290,10 +477,10 @@ def _parse_report_record(
         dns_timeout (float): Sets the DNS timeout in seconds
 
     Returns:
-        OrderedDict: The converted record
+        dict: The converted record
     """
     record = record.copy()
-    new_record = OrderedDict()
+    new_record: dict[str, Any] = {}
     if record["row"]["source_ip"] is None:
         raise ValueError("Source IP address is empty")
     new_record_source = get_ip_address_info(
@@ -311,14 +498,12 @@ def _parse_report_record(
     new_record["source"] = new_record_source
     new_record["count"] = int(record["row"]["count"])
     policy_evaluated = record["row"]["policy_evaluated"].copy()
-    new_policy_evaluated = OrderedDict(
-        [
-            ("disposition", "none"),
-            ("dkim", "fail"),
-            ("spf", "fail"),
-            ("policy_override_reasons", []),
-        ]
-    )
+    new_policy_evaluated: dict[str, Any] = {
+        "disposition": "none",
+        "dkim": "fail",
+        "spf": "fail",
+        "policy_override_reasons": [],
+    }
     if "disposition" in policy_evaluated:
         new_policy_evaluated["disposition"] = policy_evaluated["disposition"]
         if new_policy_evaluated["disposition"].strip().lower() == "pass":
@@ -355,7 +540,7 @@ def _parse_report_record(
         new_record["identifiers"] = record["identities"].copy()
     else:
         new_record["identifiers"] = record["identifiers"].copy()
-    new_record["auth_results"] = OrderedDict([("dkim", []), ("spf", [])])
+    new_record["auth_results"] = {"dkim": [], "spf": []}
     if type(new_record["identifiers"]["header_from"]) is str:
         lowered_from = new_record["identifiers"]["header_from"].lower()
     else:
@@ -374,7 +559,7 @@ def _parse_report_record(
         auth_results["dkim"] = [auth_results["dkim"]]
     for result in auth_results["dkim"]:
         if "domain" in result and result["domain"] is not None:
-            new_result = OrderedDict([("domain", result["domain"])])
+            new_result: dict[str, str] = {"domain": result["domain"]}
             if "selector" in result and result["selector"] is not None:
                 new_result["selector"] = result["selector"]
             else:
@@ -389,16 +574,16 @@ def _parse_report_record(
         auth_results["spf"] = [auth_results["spf"]]
     for result in auth_results["spf"]:
         if "domain" in result and result["domain"] is not None:
-            new_result = OrderedDict([("domain", result["domain"])])
+            new_spf_result: dict[str, str] = {"domain": result["domain"]}
             if "scope" in result and result["scope"] is not None:
-                new_result["scope"] = result["scope"]
+                new_spf_result["scope"] = result["scope"]
             else:
-                new_result["scope"] = "mfrom"
+                new_spf_result["scope"] = "mfrom"
             if "result" in result and result["result"] is not None:
-                new_result["result"] = result["result"]
+                new_spf_result["result"] = result["result"]
             else:
-                new_result["result"] = "none"
-            new_record["auth_results"]["spf"].append(new_result)
+                new_spf_result["result"] = "none"
+            new_record["auth_results"]["spf"].append(new_spf_result)
 
     if "envelope_from" not in new_record["identifiers"]:
         envelope_from = None
@@ -427,12 +612,12 @@ def _parse_report_record(
     return new_record
 
 
-def _parse_smtp_tls_failure_details(failure_details: dict[str, Any]):
+def _parse_smtp_tls_failure_details(failure_details: dict[str, Any]) -> SMTPTLSFailureDetails:
     try:
-        new_failure_details = OrderedDict(
-            result_type=failure_details["result-type"],
-            failed_session_count=failure_details["failed-session-count"],
-        )
+        new_failure_details: dict[str, Any] = {
+            "result_type": failure_details["result-type"],
+            "failed_session_count": failure_details["failed-session-count"],
+        }
 
         if "sending-mta-ip" in failure_details:
             new_failure_details["sending_mta_ip"] = failure_details["sending-mta-ip"]
@@ -463,7 +648,7 @@ def _parse_smtp_tls_failure_details(failure_details: dict[str, Any]):
         raise InvalidSMTPTLSReport(str(e))
 
 
-def _parse_smtp_tls_report_policy(policy: dict[str, Any]):
+def _parse_smtp_tls_report_policy(policy: dict[str, Any]) -> SMTPTLSPolicy:
     policy_types = ["tlsa", "sts", "no-policy-found"]
     try:
         policy_domain = policy["policy"]["policy-domain"]
@@ -471,7 +656,10 @@ def _parse_smtp_tls_report_policy(policy: dict[str, Any]):
         failure_details = []
         if policy_type not in policy_types:
             raise InvalidSMTPTLSReport(f"Invalid policy type {policy_type}")
-        new_policy = OrderedDict(policy_domain=policy_domain, policy_type=policy_type)
+        new_policy: dict[str, Any] = {
+            "policy_domain": policy_domain,
+            "policy_type": policy_type,
+        }
         if "policy-string" in policy["policy"]:
             if isinstance(policy["policy"]["policy-string"], list):
                 if len(policy["policy"]["policy-string"]) > 0:
@@ -500,7 +688,7 @@ def _parse_smtp_tls_report_policy(policy: dict[str, Any]):
         raise InvalidSMTPTLSReport(str(e))
 
 
-def parse_smtp_tls_report_json(report: dict[str, Any]):
+def parse_smtp_tls_report_json(report: str) -> ParsedSMTPTLSReport:
     """Parses and validates an SMTP TLS report"""
     required_fields = [
         "organization-name",
@@ -512,47 +700,49 @@ def parse_smtp_tls_report_json(report: dict[str, Any]):
 
     try:
         policies = []
-        report = json.loads(report)
+        report_dict = json.loads(report)
         for required_field in required_fields:
-            if required_field not in report:
+            if required_field not in report_dict:
                 raise Exception(f"Missing required field: {required_field}]")
-        if not isinstance(report["policies"], list):
-            policies_type = type(report["policies"])
+        if not isinstance(report_dict["policies"], list):
+            policies_type = type(report_dict["policies"])
             raise InvalidSMTPTLSReport(f"policies must be a list, not {policies_type}")
-        for policy in report["policies"]:
+        for policy in report_dict["policies"]:
             policies.append(_parse_smtp_tls_report_policy(policy))
 
-        new_report = OrderedDict(
-            organization_name=report["organization-name"],
-            begin_date=report["date-range"]["start-datetime"],
-            end_date=report["date-range"]["end-datetime"],
-            contact_info=report["contact-info"],
-            report_id=report["report-id"],
-            policies=policies,
-        )
+        new_report: dict[str, Any] = {
+            "organization_name": report_dict["organization-name"],
+            "begin_date": report_dict["date-range"]["start-datetime"],
+            "end_date": report_dict["date-range"]["end-datetime"],
+            "contact_info": report_dict["contact-info"],
+            "report_id": report_dict["report-id"],
+            "policies": policies,
+        }
 
         return new_report
 
     except KeyError as e:
-        InvalidSMTPTLSReport(f"Missing required field: {e}")
+        raise InvalidSMTPTLSReport(f"Missing required field: {e}")
     except Exception as e:
         raise InvalidSMTPTLSReport(str(e))
 
 
-def parsed_smtp_tls_reports_to_csv_rows(reports: OrderedDict[str, Any]):
-    """Converts one oor more parsed SMTP TLS reports into a list of single
-    layer OrderedDict objects suitable for use in a CSV"""
-    if type(reports) is OrderedDict:
+def parsed_smtp_tls_reports_to_csv_rows(
+    reports: Union[ParsedSMTPTLSReport, list[ParsedSMTPTLSReport]],
+) -> list[dict[str, Any]]:
+    """Converts one or more parsed SMTP TLS reports into a list of single
+    layer dict objects suitable for use in a CSV"""
+    if type(reports) is dict:
         reports = [reports]
 
     rows = []
     for report in reports:
-        common_fields = OrderedDict(
-            organization_name=report["organization_name"],
-            begin_date=report["begin_date"],
-            end_date=report["end_date"],
-            report_id=report["report_id"],
-        )
+        common_fields: dict[str, Any] = {
+            "organization_name": report["organization_name"],
+            "begin_date": report["begin_date"],
+            "end_date": report["end_date"],
+            "report_id": report["report_id"],
+        }
         record = common_fields.copy()
         for policy in report["policies"]:
             if "policy_strings" in policy:
@@ -574,7 +764,7 @@ def parsed_smtp_tls_reports_to_csv_rows(reports: OrderedDict[str, Any]):
     return rows
 
 
-def parsed_smtp_tls_reports_to_csv(reports: OrderedDict[str, Any]) -> str:
+def parsed_smtp_tls_reports_to_csv(reports: ParsedSMTPTLSReport) -> str:
     """
     Converts one or more parsed SMTP TLS reports to flat CSV format, including
     headers
@@ -622,17 +812,17 @@ def parsed_smtp_tls_reports_to_csv(reports: OrderedDict[str, Any]) -> str:
 def parse_aggregate_report_xml(
     xml: str,
     *,
-    ip_db_path: Optional[bool] = None,
+    ip_db_path: Optional[str] = None,
     always_use_local_files: Optional[bool] = False,
-    reverse_dns_map_path: Optional[bool] = None,
-    reverse_dns_map_url: Optional[bool] = None,
+    reverse_dns_map_path: Optional[str] = None,
+    reverse_dns_map_url: Optional[str] = None,
     offline: Optional[bool] = False,
     nameservers: Optional[list[str]] = None,
     timeout: Optional[float] = 2.0,
-    keep_alive: Optional[callable] = None,
-    normalize_timespan_threshold_hours: Optional[float] = 24.0,
-) -> OrderedDict[str, Any]:
-    """Parses a DMARC XML report string and returns a consistent OrderedDict
+    keep_alive: Optional[Callable] = None,
+    normalize_timespan_threshold_hours: float = 24.0,
+) -> ParsedAggregateReport:
+    """Parses a DMARC XML report string and returns a consistent dict
 
     Args:
         xml (str): A string of DMARC aggregate report XML
@@ -648,7 +838,7 @@ def parse_aggregate_report_xml(
         normalize_timespan_threshold_hours (float): Normalize timespans beyond this
 
     Returns:
-        OrderedDict: The parsed aggregate DMARC report
+        dict: The parsed aggregate DMARC report
     """
     errors = []
     # Parse XML and recover from errors
@@ -680,8 +870,8 @@ def parse_aggregate_report_xml(
         schema = "draft"
         if "version" in report:
             schema = report["version"]
-        new_report = OrderedDict([("xml_schema", schema)])
-        new_report_metadata = OrderedDict()
+        new_report: dict[str, Any] = {"xml_schema": schema}
+        new_report_metadata: dict[str, Any] = {}
         if report_metadata["org_name"] is None:
             if report_metadata["email"] is not None:
                 report_metadata["org_name"] = report_metadata["email"].split("@")[-1]
@@ -742,7 +932,7 @@ def parse_aggregate_report_xml(
         policy_published = report["policy_published"]
         if type(policy_published) is list:
             policy_published = policy_published[0]
-        new_policy_published = OrderedDict()
+        new_policy_published: dict[str, Any] = {}
         new_policy_published["domain"] = policy_published["domain"]
         adkim = "r"
         if "adkim" in policy_published:
@@ -847,14 +1037,14 @@ def extract_report(content: Union[bytes, str, IO[Any]]) -> str:
         str: The extracted text
 
     """
-    file_object = None
+    file_object: Union[BytesIO, IO[Any]]
     try:
         if isinstance(content, str):
             try:
                 file_object = BytesIO(b64decode(content))
             except binascii.Error:
                 return content
-        elif type(content) is bytes:
+        elif isinstance(content, bytes):
             file_object = BytesIO(content)
         else:
             file_object = content
@@ -909,12 +1099,12 @@ def parse_aggregate_report_file(
     dns_timeout: Optional[float] = 2.0,
     keep_alive: Optional[Callable] = None,
     normalize_timespan_threshold_hours: Optional[float] = 24.0,
-) -> OrderedDict[str, any]:
+) -> ParsedAggregateReport:
     """Parses a file at the given path, a file-like object. or bytes as an
     aggregate DMARC report
 
     Args:
-        _input (str | bytes | IO): A path to a file, a file like object, or bytes
+        _input (Union[str, bytes, IO]): A path to a file, a file like object, or bytes
         offline (bool): Do not query online for geolocation or DNS
         always_use_local_files (bool): Do not download files
         reverse_dns_map_path (str): Path to a reverse DNS map file
@@ -927,7 +1117,7 @@ def parse_aggregate_report_file(
         normalize_timespan_threshold_hours (float): Normalize timespans beyond this
 
     Returns:
-        OrderedDict: The parsed DMARC aggregate report
+        dict: The parsed DMARC aggregate report
     """
 
     try:
@@ -950,7 +1140,7 @@ def parse_aggregate_report_file(
 
 
 def parsed_aggregate_reports_to_csv_rows(
-    reports: list[OrderedDict[str, Any]],
+    reports: list[ParsedAggregateReport],
 ) -> list[dict[str, Any]]:
     """
     Converts one or more parsed aggregate reports to list of dicts in flat CSV
@@ -967,7 +1157,7 @@ def parsed_aggregate_reports_to_csv_rows(
     def to_str(obj):
         return str(obj).lower()
 
-    if type(reports) is OrderedDict:
+    if type(reports) is dict:
         reports = [reports]
 
     rows = []
@@ -1075,7 +1265,7 @@ def parsed_aggregate_reports_to_csv_rows(
     return rows
 
 
-def parsed_aggregate_reports_to_csv(reports: list[OrderedDict[str, Any]]) -> str:
+def parsed_aggregate_reports_to_csv(reports: list[ParsedAggregateReport]) -> str:
     """
     Converts one or more parsed aggregate reports to flat CSV format, including
     headers
@@ -1148,15 +1338,15 @@ def parse_forensic_report(
     *,
     always_use_local_files: Optional[bool] = False,
     reverse_dns_map_path: Optional[str] = None,
-    reverse_dns_map_url: str = None,
+    reverse_dns_map_url: Optional[str] = None,
     offline: Optional[bool] = False,
     ip_db_path: Optional[str] = None,
     nameservers: Optional[list[str]] = None,
     dns_timeout: Optional[float] = 2.0,
     strip_attachment_payloads: Optional[bool] = False,
-) -> OrderedDict[str, Any]:
+) -> ParsedForensicReport:
     """
-    Converts a DMARC forensic report and sample to a ``OrderedDict``
+    Converts a DMARC forensic report and sample to a ``dict``
 
     Args:
         feedback_report (str): A message's feedback report as a string
@@ -1174,12 +1364,12 @@ def parse_forensic_report(
             forensic report results
 
     Returns:
-        OrderedDict: A parsed report and sample
+        dict: A parsed report and sample
     """
     delivery_results = ["delivered", "spam", "policy", "reject", "other"]
 
     try:
-        parsed_report = OrderedDict()
+        parsed_report: dict[str, Any] = {}
         report_values = feedback_report_regex.findall(feedback_report)
         for report_value in report_values:
             key = report_value[0].lower().replace("-", "_")
@@ -1282,7 +1472,7 @@ def parse_forensic_report(
         raise InvalidForensicReport("Unexpected error: {0}".format(error.__str__()))
 
 
-def parsed_forensic_reports_to_csv_rows(reports: list[OrderedDict[str, Any]]):
+def parsed_forensic_reports_to_csv_rows(reports: list[ParsedForensicReport]):
     """
     Converts one or more parsed forensic reports to a list of dicts in flat CSV
     format
@@ -1293,7 +1483,7 @@ def parsed_forensic_reports_to_csv_rows(reports: list[OrderedDict[str, Any]]):
     Returns:
         list: Parsed forensic report data as a list of dicts in flat CSV format
     """
-    if type(reports) is OrderedDict:
+    if type(reports) is dict:
         reports = [reports]
 
     rows = []
@@ -1318,7 +1508,7 @@ def parsed_forensic_reports_to_csv_rows(reports: list[OrderedDict[str, Any]]):
     return rows
 
 
-def parsed_forensic_reports_to_csv(reports: list[dict[str, Any]]) -> str:
+def parsed_forensic_reports_to_csv(reports: list[ParsedForensicReport]) -> str:
     """
     Converts one or more parsed forensic reports to flat CSV format, including
     headers
@@ -1383,7 +1573,7 @@ def parse_report_email(
     strip_attachment_payloads: Optional[bool] = False,
     keep_alive: Optional[callable] = None,
     normalize_timespan_threshold_hours: Optional[float] = 24.0,
-) -> OrderedDict[str, Any]:
+) -> dict[str, Any]:
     """
     Parses a DMARC report from an email
 
@@ -1402,7 +1592,7 @@ def parse_report_email(
         normalize_timespan_threshold_hours (float): Normalize timespans beyond this
 
     Returns:
-        OrderedDict:
+        dict:
         * ``report_type``: ``aggregate`` or ``forensic``
         * ``report``: The parsed report
     """
@@ -1426,6 +1616,7 @@ def parse_report_email(
     feedback_report = None
     smtp_tls_report = None
     sample = None
+    is_feedback_report: bool = False
     if "From" in msg_headers:
         logger.info("Parsing mail from {0} on {1}".format(msg_headers["From"], date))
     if "Subject" in msg_headers:
@@ -1441,6 +1632,7 @@ def parse_report_email(
         if content_type == "text/html":
             continue
         elif content_type == "message/feedback-report":
+            is_feedback_report = True
             try:
                 if "Feedback-Type" in payload:
                     feedback_report = payload
@@ -1451,23 +1643,23 @@ def parse_report_email(
                 feedback_report = feedback_report.replace("\\n", "\n")
             except (ValueError, TypeError, binascii.Error):
                 feedback_report = payload
-        elif content_type == "text/rfc822-headers":
-            sample = payload
-        elif content_type == "message/rfc822":
+        elif is_feedback_report and content_type in EMAIL_SAMPLE_CONTENT_TYPES:
             sample = payload
         elif content_type == "application/tlsrpt+json":
             if not payload.strip().startswith("{"):
                 payload = str(b64decode(payload))
             smtp_tls_report = parse_smtp_tls_report_json(payload)
-            return OrderedDict(
-                [("report_type", "smtp_tls"), ("report", smtp_tls_report)]
-            )
+            return {
+                "report_type": "smtp_tls",
+                "report": smtp_tls_report,
+            }
         elif content_type == "application/tlsrpt+gzip":
             payload = extract_report(payload)
             smtp_tls_report = parse_smtp_tls_report_json(payload)
-            return OrderedDict(
-                [("report_type", "smtp_tls"), ("report", smtp_tls_report)]
-            )
+            return {
+                "report_type": "smtp_tls",
+                "report": smtp_tls_report,
+            }
         elif content_type == "text/plain":
             if "A message claiming to be from you has failed" in payload:
                 try:
@@ -1498,9 +1690,10 @@ def parse_report_email(
                     payload = payload.decode("utf-8", errors="replace")
                 if payload.strip().startswith("{"):
                     smtp_tls_report = parse_smtp_tls_report_json(payload)
-                    result = OrderedDict(
-                        [("report_type", "smtp_tls"), ("report", smtp_tls_report)]
-                    )
+                    result = {
+                        "report_type": "smtp_tls",
+                        "report": smtp_tls_report,
+                    }
                 elif payload.strip().startswith("<"):
                     aggregate_report = parse_aggregate_report_xml(
                         payload,
@@ -1514,9 +1707,10 @@ def parse_report_email(
                         keep_alive=keep_alive,
                         normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
                     )
-                    result = OrderedDict(
-                        [("report_type", "aggregate"), ("report", aggregate_report)]
-                    )
+                    result = {
+                        "report_type": "aggregate",
+                        "report": aggregate_report,
+                    }
 
                     return result
 
@@ -1560,7 +1754,7 @@ def parse_report_email(
         except Exception as e:
             raise InvalidForensicReport(e.__str__())
 
-        result = OrderedDict([("report_type", "forensic"), ("report", forensic_report)])
+        result = {"report_type": "forensic", "report": forensic_report}
         return result
 
     if result is None:
@@ -1581,12 +1775,12 @@ def parse_report_file(
     offline: Optional[bool] = False,
     keep_alive: Optional[Callable] = None,
     normalize_timespan_threshold_hours: Optional[float] = 24,
-) -> OrderedDict[str, Any]:
+) -> ReportTypeWrapper:
     """Parses a DMARC aggregate or forensic file at the given path, a
     file-like object. or bytes
 
     Args:
-        input_ (str | bytes | IO): A path to a file, a file like object, or bytes
+        input_ (Union[str, bytes, IO]): A path to a file, a file like object, or bytes
         nameservers (list): A list of one or more nameservers to use
             (Cloudflare's public DNS resolvers by default)
         dns_timeout (float): Sets the DNS timeout in seconds
@@ -1600,7 +1794,7 @@ def parse_report_file(
         keep_alive (callable): Keep alive function
 
     Returns:
-        OrderedDict: The parsed DMARC report
+        dict: The parsed DMARC report
     """
     if type(input_) is str:
         logger.debug("Parsing {0}".format(input_))
@@ -1627,11 +1821,11 @@ def parse_report_file(
             keep_alive=keep_alive,
             normalize_timespan_threshold_hours=normalize_timespan_threshold_hours,
         )
-        results = OrderedDict([("report_type", "aggregate"), ("report", report)])
+        results = {"report_type": "aggregate", "report": report}
     except InvalidAggregateReport:
         try:
             report = parse_smtp_tls_report_json(content)
-            results = OrderedDict([("report_type", "smtp_tls"), ("report", report)])
+            results = {"report_type": "smtp_tls", "report": report}
         except InvalidSMTPTLSReport:
             try:
                 sa = strip_attachment_payloads
@@ -1665,7 +1859,7 @@ def get_dmarc_reports_from_mbox(
     reverse_dns_map_url: Optional[str] = None,
     offline: Optional[bool] = False,
     normalize_timespan_threshold_hours: Optional[float] = 24.0,
-) -> OrderedDict[str, OrderedDict[str, Any]]:
+) -> dict[str, dict[str, Any]]:
     """Parses a mailbox in mbox format containing e-mails with attached
     DMARC reports
 
@@ -1684,7 +1878,7 @@ def get_dmarc_reports_from_mbox(
         normalize_timespan_threshold_hours (float): Normalize timespans beyond this
 
     Returns:
-        OrderedDict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``
+        dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``
 
     """
     aggregate_reports = []
@@ -1733,13 +1927,11 @@ def get_dmarc_reports_from_mbox(
                 logger.warning(error.__str__())
     except mailbox.NoSuchMailboxError:
         raise InvalidDMARCReport("Mailbox {0} does not exist".format(input_))
-    return OrderedDict(
-        [
-            ("aggregate_reports", aggregate_reports),
-            ("forensic_reports", forensic_reports),
-            ("smtp_tls_reports", smtp_tls_reports),
-        ]
-    )
+    return {
+        "aggregate_reports": aggregate_reports,
+        "forensic_reports": forensic_reports,
+        "smtp_tls_reports": smtp_tls_reports,
+    }
 
 
 def get_dmarc_reports_from_mailbox(
@@ -1750,19 +1942,19 @@ def get_dmarc_reports_from_mailbox(
     delete: Optional[bool] = False,
     test: Optional[bool] = False,
     ip_db_path: Optional[str] = None,
-    always_use_local_files: Optional[str] = False,
+    always_use_local_files: Optional[bool] = False,
     reverse_dns_map_path: Optional[str] = None,
     reverse_dns_map_url: Optional[str] = None,
     offline: Optional[bool] = False,
     nameservers: Optional[list[str]] = None,
     dns_timeout: Optional[float] = 6.0,
     strip_attachment_payloads: Optional[bool] = False,
-    results: Optional[OrderedDict[str, any]] = None,
+    results: Optional[dict[str, Any]] = None,
     batch_size: Optional[int] = 10,
     since: Optional[datetime] = None,
     create_folders: Optional[bool] = True,
     normalize_timespan_threshold_hours: Optional[float] = 24,
-) -> OrderedDict[str, OrderedDict[str, Any]]:
+) -> dict[str, dict[str, Any]]:
     """
     Fetches and parses DMARC reports from a mailbox
 
@@ -1791,7 +1983,7 @@ def get_dmarc_reports_from_mailbox(
         normalize_timespan_threshold_hours (float): Normalize timespans beyond this
 
     Returns:
-        OrderedDict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``
+        dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports``
     """
     if delete and test:
         raise ValueError("delete and test options are mutually exclusive")
@@ -2025,13 +2217,11 @@ def get_dmarc_reports_from_mailbox(
                     except Exception as e:
                         e = "Error moving message UID {0}: {1}".format(msg_uid, e)
                         logger.error("Mailbox error: {0}".format(e))
-    results = OrderedDict(
-        [
-            ("aggregate_reports", aggregate_reports),
-            ("forensic_reports", forensic_reports),
-            ("smtp_tls_reports", smtp_tls_reports),
-        ]
-    )
+    results: dict[str, Any] = {
+        "aggregate_reports": aggregate_reports,
+        "forensic_reports": forensic_reports,
+        "smtp_tls_reports": smtp_tls_reports,
+    }
 
     if current_time:
         total_messages = len(
@@ -2172,7 +2362,7 @@ def append_csv(filename, csv):
 
 
 def save_output(
-    results: OrderedDict[str, Any],
+    results: dict[str, Any],
     *,
     output_directory: Optional[str] = "output",
     aggregate_json_filename: Optional[str] = "aggregate.json",
@@ -2186,7 +2376,7 @@ def save_output(
     Save report data in the given directory
 
     Args:
-        results (OrderedDict): Parsing results
+        results (dict): Parsing results
         output_directory (str): The path to the directory to save in
         aggregate_json_filename (str): Filename for the aggregate JSON file
         forensic_json_filename (str): Filename for the forensic JSON file
@@ -2258,12 +2448,12 @@ def save_output(
             sample_file.write(sample)
 
 
-def get_report_zip(results: OrderedDict[str, Any]) -> bytes:
+def get_report_zip(results: dict[str, Any]) -> bytes:
     """
     Creates a zip file of parsed report output
 
     Args:
-        results (OrderedDict): The parsed results
+        results (dict): The parsed results
 
     Returns:
         bytes: zip file bytes
@@ -2304,7 +2494,7 @@ def get_report_zip(results: OrderedDict[str, Any]) -> bytes:
 
 
 def email_results(
-    results: OrderedDict,
+    results: dict[str, Any],
     *,
     host: str,
     mail_from: str,
@@ -2324,7 +2514,7 @@ def email_results(
     Emails parsing results as a zip file
 
     Args:
-        results (OrderedDict): Parsing results
+        results (dict): Parsing results
         host (str): Mail server hostname or IP address
         mail_from: The value of the message from header
         mail_to (list): A list of addresses to mail to

parsedmarc/cli.py

@@ -10,7 +10,6 @@ from glob import glob
 import logging
 import math
 import yaml
-from collections import OrderedDict
 import json
 from ssl import CERT_NONE, create_default_context
 from multiprocessing import Pipe, Process
@@ -1481,7 +1480,7 @@ def _main():
                 pbar.update(counter - pbar.n)
 
     for result in results:
-        if type(result[0]) is ParserError:
+        if isinstance(result[0], ParserError) or result[0] is None:
             logger.error("Failed to parse {0} - {1}".format(result[1], result[0]))
         else:
             if result[0]["report_type"] == "aggregate":
@@ -1634,7 +1633,7 @@ def _main():
             logger.exception("Mailbox Error")
             exit(1)
 
-    results = OrderedDict(
+    results = dict(
         [
             ("aggregate_reports", aggregate_reports),
             ("forensic_reports", forensic_reports),

parsedmarc/constants.py

@@ -1,2 +1,3 @@
-__version__ = "9.0.3"
+__version__ = "9.0.5"
+
 USER_AGENT = f"parsedmarc/{__version__}"

parsedmarc/elastic.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 from typing import Optional, Union, Any
 
-from collections import OrderedDict
 
 from elasticsearch_dsl.search import Q
 from elasticsearch_dsl import (
@@ -203,11 +202,11 @@ class _SMTPTLSPolicyDoc(InnerDoc):
 
     def add_failure_details(
         self,
-        result_type: str,
-        ip_address: str,
-        receiving_ip: str,
-        receiving_mx_helo: str,
-        failed_session_count: int,
+        result_type: Optional[str] = None,
+        ip_address: Optional[str] = None,
+        receiving_ip: Optional[str] = None,
+        receiving_mx_helo: Optional[str] = None,
+        failed_session_count: Optional[int] = None,
         sending_mta_ip: Optional[str] = None,
         receiving_mx_hostname: Optional[str] = None,
         additional_information_uri: Optional[str] = None,
@@ -279,7 +278,7 @@ def set_hosts(
     Sets the Elasticsearch hosts to use
 
     Args:
-        hosts (str | list[str]): A single hostname or URL, or list of hostnames or URLs
+        hosts (Union[str, list[str]]): A single hostname or URL, or list of hostnames or URLs
         use_ssl (bool): Use an HTTPS connection to the server
         ssl_cert_path (str): Path to the certificate chain
         username (str): The username to use for authentication
@@ -297,7 +296,7 @@ def set_hosts(
             conn_params["ca_certs"] = ssl_cert_path
         else:
             conn_params["verify_certs"] = False
-    if username:
+    if username and password:
         conn_params["http_auth"] = username + ":" + password
     if api_key:
         conn_params["api_key"] = api_key
@@ -377,7 +376,7 @@ def migrate_indexes(
 
 
 def save_aggregate_report_to_elasticsearch(
-    aggregate_report: OrderedDict[str, Any],
+    aggregate_report: dict[str, Any],
     index_suffix: Optional[str] = None,
     index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
@@ -388,7 +387,7 @@ def save_aggregate_report_to_elasticsearch(
     Saves a parsed DMARC aggregate report to Elasticsearch
 
     Args:
-        aggregate_report (OrderedDict): A parsed forensic report
+        aggregate_report (dict): A parsed forensic report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily indexes
@@ -539,8 +538,8 @@ def save_aggregate_report_to_elasticsearch(
 
 
 def save_forensic_report_to_elasticsearch(
-    forensic_report: OrderedDict[str, Any],
-    index_suffix: Optional[any] = None,
+    forensic_report: dict[str, Any],
+    index_suffix: Optional[Any] = None,
     index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
     number_of_shards: int = 1,
@@ -550,7 +549,7 @@ def save_forensic_report_to_elasticsearch(
     Saves a parsed DMARC forensic report to Elasticsearch
 
     Args:
-        forensic_report (OrderedDict): A parsed forensic report
+        forensic_report (dict): A parsed forensic report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily
@@ -570,7 +569,7 @@ def save_forensic_report_to_elasticsearch(
         sample_date = forensic_report["parsed_sample"]["date"]
         sample_date = human_timestamp_to_datetime(sample_date)
     original_headers = forensic_report["parsed_sample"]["headers"]
-    headers = OrderedDict()
+    headers = dict()
     for original_header in original_headers:
         headers[original_header.lower()] = original_headers[original_header]
 
@@ -706,9 +705,9 @@ def save_forensic_report_to_elasticsearch(
 
 
 def save_smtp_tls_report_to_elasticsearch(
-    report: OrderedDict[str, Any],
-    index_suffix: str = None,
-    index_prefix: str = None,
+    report: dict[str, Any],
+    index_suffix: Optional[str] = None,
+    index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
     number_of_shards: Optional[int] = 1,
     number_of_replicas: Optional[int] = 0,
@@ -717,7 +716,7 @@ def save_smtp_tls_report_to_elasticsearch(
     Saves a parsed SMTP TLS report to Elasticsearch
 
     Args:
-        report (OrderedDict): A parsed SMTP TLS report
+        report (dict): A parsed SMTP TLS report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily indexes

parsedmarc/gelf.py

@@ -8,7 +8,6 @@ import logging
 import logging.handlers
 import json
 import threading
-from collections import OrderedDict
 
 from parsedmarc import (
     parsed_aggregate_reports_to_csv_rows,
@@ -53,7 +52,9 @@ class GelfClient(object):
         )
         self.logger.addHandler(self.handler)
 
-    def save_aggregate_report_to_gelf(self, aggregate_reports: OrderedDict[str, Any]):
+    def save_aggregate_report_to_gelf(
+        self, aggregate_reports: list[dict[str, Any]]
+    ):
         rows = parsed_aggregate_reports_to_csv_rows(aggregate_reports)
         for row in rows:
             log_context_data.parsedmarc = row
@@ -61,12 +62,14 @@ class GelfClient(object):
 
         log_context_data.parsedmarc = None
 
-    def save_forensic_report_to_gelf(self, forensic_reports: OrderedDict[str, Any]):
+    def save_forensic_report_to_gelf(
+        self, forensic_reports: list[dict[str, Any]]
+    ):
         rows = parsed_forensic_reports_to_csv_rows(forensic_reports)
         for row in rows:
             self.logger.info(json.dumps(row))
 
-    def save_smtp_tls_report_to_gelf(self, smtp_tls_reports: OrderedDict[str, Any]):
+    def save_smtp_tls_report_to_gelf(self, smtp_tls_reports: dict[str, Any]):
         rows = parsed_smtp_tls_reports_to_csv_rows(smtp_tls_reports)
         for row in rows:
             self.logger.info(json.dumps(row))

parsedmarc/kafkaclient.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Any, Optional
+from typing import Any, Optional, Union
 from ssl import SSLContext
 
 import json
@@ -10,7 +10,6 @@ from ssl import create_default_context
 
 from kafka import KafkaProducer
 from kafka.errors import NoBrokersAvailable, UnknownTopicOrPartitionError
-from collections import OrderedDict
 from parsedmarc.utils import human_timestamp_to_datetime
 
 from parsedmarc import __version__
@@ -66,7 +65,7 @@ class KafkaClient(object):
             raise KafkaError("No Kafka brokers available")
 
     @staticmethod
-    def strip_metadata(report: OrderedDict[str, Any]):
+    def strip_metadata(report: dict[str, Any]):
         """
         Duplicates org_name, org_email and report_id into JSON root
         and removes report_metadata key to bring it more inline
@@ -80,7 +79,7 @@ class KafkaClient(object):
         return report
 
     @staticmethod
-    def generate_date_range(report: OrderedDict[str, Any]):
+    def generate_date_range(report: dict[str, Any]):
         """
         Creates a date_range timestamp with format YYYY-MM-DD-T-HH:MM:SS
         based on begin and end dates for easier parsing in Kibana.
@@ -98,7 +97,9 @@ class KafkaClient(object):
         return date_range
 
     def save_aggregate_reports_to_kafka(
-        self, aggregate_reports: list[OrderedDict][str, Any], aggregate_topic: str
+        self,
+        aggregate_reports: Union[dict[str, Any], list[dict[str, Any]]],
+        aggregate_topic: str,
     ):
         """
         Saves aggregate DMARC reports to Kafka
@@ -109,9 +110,7 @@ class KafkaClient(object):
             aggregate_topic (str): The name of the Kafka topic
 
         """
-        if isinstance(aggregate_reports, dict) or isinstance(
-            aggregate_reports, OrderedDict
-        ):
+        if isinstance(aggregate_reports, dict):
             aggregate_reports = [aggregate_reports]
 
         if len(aggregate_reports) < 1:
@@ -143,7 +142,9 @@ class KafkaClient(object):
                     raise KafkaError("Kafka error: {0}".format(e.__str__()))
 
     def save_forensic_reports_to_kafka(
-        self, forensic_reports: OrderedDict[str, Any], forensic_topic: str
+        self,
+        forensic_reports: Union[dict[str, Any], list[dict[str, Any]]],
+        forensic_topic: str,
     ):
         """
         Saves forensic DMARC reports to Kafka, sends individual
@@ -175,7 +176,9 @@ class KafkaClient(object):
             raise KafkaError("Kafka error: {0}".format(e.__str__()))
 
     def save_smtp_tls_reports_to_kafka(
-        self, smtp_tls_reports: list[OrderedDict[str, Any]], smtp_tls_topic: str
+        self,
+        smtp_tls_reports: Union[list[dict[str, Any]], dict[str, Any]],
+        smtp_tls_topic: str,
     ):
         """
         Saves SMTP TLS reports to Kafka, sends individual

parsedmarc/loganalytics.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 from typing import Any
-from collections import OrderedDict
 
 from parsedmarc.log import logger
 from azure.core.exceptions import HttpResponseError
@@ -110,7 +109,7 @@ class LogAnalyticsClient(object):
 
     def publish_json(
         self,
-        results: OrderedDict[str, OrderedDict[str, Any]],
+        results,
         logs_client: LogsIngestionClient,
         dcr_stream: str,
     ):
@@ -133,7 +132,7 @@ class LogAnalyticsClient(object):
 
     def publish_results(
         self,
-        results: OrderedDict[str, OrderedDict[str, Any]],
+        results: dict[str, dict[str, Any]],
         save_aggregate: bool,
         save_forensic: bool,
         save_smtp_tls: bool,

parsedmarc/opensearch.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 from typing import Optional, Union, Any
 
-from collections import OrderedDict
 
 from opensearchpy import (
     Q,
@@ -203,11 +202,11 @@ class _SMTPTLSPolicyDoc(InnerDoc):
 
     def add_failure_details(
         self,
-        result_type: str,
-        ip_address: str,
-        receiving_ip: str,
-        receiving_mx_helo: str,
-        failed_session_count: int,
+        result_type: Optional[str] = None,
+        ip_address: Optional[str] = None,
+        receiving_ip: Optional[str] = None,
+        receiving_mx_helo: Optional[str] = None,
+        failed_session_count: Optional[int] = None,
         sending_mta_ip: Optional[str] = None,
         receiving_mx_hostname: Optional[str] = None,
         additional_information_uri: Optional[str] = None,
@@ -297,7 +296,7 @@ def set_hosts(
             conn_params["ca_certs"] = ssl_cert_path
         else:
             conn_params["verify_certs"] = False
-    if username:
+    if username and password:
         conn_params["http_auth"] = username + ":" + password
     if api_key:
         conn_params["api_key"] = api_key
@@ -376,8 +375,8 @@ def migrate_indexes(
         pass
 
 
-def save_aggregate_report_to_elasticsearch(
-    aggregate_report: OrderedDict[str, Any],
+def save_aggregate_report_to_opensearch(
+    aggregate_report: dict[str, Any],
     index_suffix: Optional[str] = None,
     index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
@@ -388,7 +387,7 @@ def save_aggregate_report_to_elasticsearch(
     Saves a parsed DMARC aggregate report to OpenSearch
 
     Args:
-        aggregate_report (OrderedDict): A parsed forensic report
+        aggregate_report (dict): A parsed forensic report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily indexes
@@ -538,9 +537,9 @@ def save_aggregate_report_to_elasticsearch(
             raise OpenSearchError("OpenSearch error: {0}".format(e.__str__()))
 
 
-def save_forensic_report_to_elasticsearch(
-    forensic_report: OrderedDict[str, Any],
-    index_suffix: Optional[any] = None,
+def save_forensic_report_to_opensearch(
+    forensic_report: dict[str, Any],
+    index_suffix: Optional[str] = None,
     index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
     number_of_shards: int = 1,
@@ -550,7 +549,7 @@ def save_forensic_report_to_elasticsearch(
     Saves a parsed DMARC forensic report to OpenSearch
 
     Args:
-        forensic_report (OrderedDict): A parsed forensic report
+        forensic_report (dict): A parsed forensic report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily
@@ -570,7 +569,7 @@ def save_forensic_report_to_elasticsearch(
         sample_date = forensic_report["parsed_sample"]["date"]
         sample_date = human_timestamp_to_datetime(sample_date)
     original_headers = forensic_report["parsed_sample"]["headers"]
-    headers = OrderedDict()
+    headers = dict()
     for original_header in original_headers:
         headers[original_header.lower()] = original_headers[original_header]
 
@@ -705,10 +704,10 @@ def save_forensic_report_to_elasticsearch(
         )
 
 
-def save_smtp_tls_report_to_elasticsearch(
-    report: OrderedDict[str, Any],
-    index_suffix: str = None,
-    index_prefix: str = None,
+def save_smtp_tls_report_to_opensearch(
+    report: dict[str, Any],
+    index_suffix: Optional[str] = None,
+    index_prefix: Optional[str] = None,
     monthly_indexes: Optional[bool] = False,
     number_of_shards: Optional[int] = 1,
     number_of_replicas: Optional[int] = 0,
@@ -717,7 +716,7 @@ def save_smtp_tls_report_to_elasticsearch(
     Saves a parsed SMTP TLS report to OpenSearch
 
     Args:
-        report (OrderedDict): A parsed SMTP TLS report
+        report (dict): A parsed SMTP TLS report
         index_suffix (str): The suffix of the name of the index to save to
         index_prefix (str): The prefix of the name of the index to save to
         monthly_indexes (bool): Use monthly indexes instead of daily indexes

parsedmarc/s3.py

@@ -7,8 +7,6 @@ from typing import Any
 import json
 import boto3
 
-from collections import OrderedDict
-
 from parsedmarc.log import logger
 from parsedmarc.utils import human_timestamp_to_datetime
 
@@ -53,18 +51,18 @@ class S3Client(object):
             aws_access_key_id=access_key_id,
             aws_secret_access_key=secret_access_key,
         )
-        self.bucket = self.s3.Bucket(self.bucket_name)
+        self.bucket: Any = self.s3.Bucket(self.bucket_name)
 
-    def save_aggregate_report_to_s3(self, report: OrderedDict[str, Any]):
+    def save_aggregate_report_to_s3(self, report: dict[str, Any]):
         self.save_report_to_s3(report, "aggregate")
 
-    def save_forensic_report_to_s3(self, report: OrderedDict[str, Any]):
+    def save_forensic_report_to_s3(self, report: dict[str, Any]):
         self.save_report_to_s3(report, "forensic")
 
-    def save_smtp_tls_report_to_s3(self, report: OrderedDict[str, Any]):
+    def save_smtp_tls_report_to_s3(self, report: dict[str, Any]):
         self.save_report_to_s3(report, "smtp_tls")
 
-    def save_report_to_s3(self, report: OrderedDict[str, Any], report_type: str):
+    def save_report_to_s3(self, report: dict[str, Any], report_type: str):
         if report_type == "smtp_tls":
             report_date = report["begin_date"]
             report_id = report["report_id"]

parsedmarc/splunk.py

@@ -2,9 +2,8 @@
 
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Union
 
-from collections import OrderedDict
 
 from urllib.parse import urlparse
 import socket
@@ -35,7 +34,7 @@ class HECClient(object):
         url: str,
         access_token: str,
         index: str,
-        source: bool = "parsedmarc",
+        source: str = "parsedmarc",
         verify=True,
         timeout=60,
     ):
@@ -51,9 +50,9 @@ class HECClient(object):
             timeout (float): Number of seconds to wait for the server to send
                 data before giving up
         """
-        url = urlparse(url)
+        parsed_url = urlparse(url)
         self.url = "{0}://{1}/services/collector/event/1.0".format(
-            url.scheme, url.netloc
+            parsed_url.scheme, parsed_url.netloc
         )
         self.access_token = access_token.lstrip("Splunk ")
         self.index = index
@@ -62,7 +61,9 @@ class HECClient(object):
         self.session = requests.Session()
         self.timeout = timeout
         self.session.verify = verify
-        self._common_data = dict(host=self.host, source=self.source, index=self.index)
+        self._common_data: dict[str, Union[str, int, float, dict]] = dict(
+            host=self.host, source=self.source, index=self.index
+        )
 
         self.session.headers = {
             "User-Agent": USER_AGENT,
@@ -70,7 +71,8 @@ class HECClient(object):
         }
 
     def save_aggregate_reports_to_splunk(
-        self, aggregate_reports: list[OrderedDict[str, Any]]
+        self,
+        aggregate_reports: Union[list[dict[str, Any]], dict[str, Any]],
     ):
         """
         Saves aggregate DMARC reports to Splunk
@@ -91,7 +93,7 @@ class HECClient(object):
         json_str = ""
         for report in aggregate_reports:
             for record in report["records"]:
-                new_report = dict()
+                new_report: dict[str, Union[str, int, float, dict]] = dict()
                 for metadata in report["report_metadata"]:
                     new_report[metadata] = report["report_metadata"][metadata]
                 new_report["interval_begin"] = record["interval_begin"]
@@ -135,7 +137,8 @@ class HECClient(object):
             raise SplunkError(response["text"])
 
     def save_forensic_reports_to_splunk(
-        self, forensic_reports: list[OrderedDict[str, Any]]
+        self,
+        forensic_reports: Union[list[dict[str, Any]], dict[str, Any]],
     ):
         """
         Saves forensic DMARC reports to Splunk
@@ -170,7 +173,9 @@ class HECClient(object):
         if response["code"] != 0:
             raise SplunkError(response["text"])
 
-    def save_smtp_tls_reports_to_splunk(self, reports: OrderedDict[str, Any]):
+    def save_smtp_tls_reports_to_splunk(
+        self, reports: Union[list[dict[str, Any]], dict[str, Any]]
+    ):
         """
         Saves aggregate DMARC reports to Splunk
 

parsedmarc/syslog.py

@@ -8,7 +8,6 @@ import logging.handlers
 
 from typing import Any
 
-from collections import OrderedDict
 
 import json
 
@@ -37,21 +36,21 @@ class SyslogClient(object):
         self.logger.addHandler(log_handler)
 
     def save_aggregate_report_to_syslog(
-        self, aggregate_reports: list[OrderedDict[str, Any]]
+        self, aggregate_reports: list[dict[str, Any]]
     ):
         rows = parsed_aggregate_reports_to_csv_rows(aggregate_reports)
         for row in rows:
             self.logger.info(json.dumps(row))
 
     def save_forensic_report_to_syslog(
-        self, forensic_reports: list[OrderedDict[str, Any]]
+        self, forensic_reports: list[dict[str, Any]]
     ):
         rows = parsed_forensic_reports_to_csv_rows(forensic_reports)
         for row in rows:
             self.logger.info(json.dumps(row))
 
     def save_smtp_tls_report_to_syslog(
-        self, smtp_tls_reports: list[OrderedDict[str, Any]]
+        self, smtp_tls_reports: list[dict[str, Any]]
     ):
         rows = parsed_smtp_tls_reports_to_csv_rows(smtp_tls_reports)
         for row in rows:

parsedmarc/utils.py

@@ -4,14 +4,13 @@
 
 from __future__ import annotations
 
-from typing import Optional, Union
+from typing import Optional, Union, TypedDict, Any
 
 import logging
 import os
 from datetime import datetime
 from datetime import timezone
 from datetime import timedelta
-from collections import OrderedDict
 from expiringdict import ExpiringDict
 import tempfile
 import subprocess
@@ -67,6 +66,14 @@ class DownloadError(RuntimeError):
     """Raised when an error occurs when downloading a file"""
 
 
+class EmailAddress(TypedDict):
+    """Parsed email address information"""
+    display_name: Optional[str]
+    address: str
+    local: Optional[str]
+    domain: Optional[str]
+
+
 def decode_base64(data: str) -> bytes:
     """
     Decodes a base64 string, with padding being optional
@@ -335,14 +342,14 @@ def get_ip_address_country(ip_address: str, *, db_path: Optional[str] = None) ->
 
 
 def get_service_from_reverse_dns_base_domain(
-    base_domain,
+    base_domain: str,
     *,
     always_use_local_file: Optional[bool] = False,
-    local_file_path: Optional[bool] = None,
-    url: Optional[bool] = None,
+    local_file_path: Optional[str] = None,
+    url: Optional[str] = None,
     offline: Optional[bool] = False,
-    reverse_dns_map: Optional[bool] = None,
-) -> str:
+    reverse_dns_map: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
     """
     Returns the service name of a given base domain name from reverse DNS.
 
@@ -411,18 +418,18 @@ def get_service_from_reverse_dns_base_domain(
 
 
 def get_ip_address_info(
-    ip_address,
+    ip_address: str,
     *,
     ip_db_path: Optional[str] = None,
     reverse_dns_map_path: Optional[str] = None,
     always_use_local_files: Optional[bool] = False,
-    reverse_dns_map_url: Optional[bool] = None,
+    reverse_dns_map_url: Optional[str] = None,
     cache: Optional[ExpiringDict] = None,
-    reverse_dns_map: Optional[bool] = None,
+    reverse_dns_map: Optional[dict[str, Any]] = None,
     offline: Optional[bool] = False,
     nameservers: Optional[list[str]] = None,
     timeout: Optional[float] = 2.0,
-) -> OrderedDict[str, str]:
+) -> dict[str, Any]:
     """
     Returns reverse DNS and country information for the given IP address
 
@@ -440,7 +447,7 @@ def get_ip_address_info(
         timeout (float): Sets the DNS timeout in seconds
 
     Returns:
-        OrderedDict: ``ip_address``, ``reverse_dns``, ``country``
+        dict: ``ip_address``, ``reverse_dns``, ``country``
 
     """
     ip_address = ip_address.lower()
@@ -449,7 +456,7 @@ def get_ip_address_info(
         if info:
             logger.debug(f"IP address {ip_address} was found in cache")
             return info
-    info = OrderedDict()
+    info = dict()
     info["ip_address"] = ip_address
     if offline:
         reverse_dns = None
@@ -487,7 +494,7 @@ def get_ip_address_info(
     return info
 
 
-def parse_email_address(original_address: str) -> OrderedDict[str, str]:
+def parse_email_address(original_address: str) -> EmailAddress:
     if original_address[0] == "":
         display_name = None
     else:
@@ -500,14 +507,12 @@ def parse_email_address(original_address: str) -> OrderedDict[str, str]:
         local = address_parts[0].lower()
         domain = address_parts[-1].lower()
 
-    return OrderedDict(
-        [
-            ("display_name", display_name),
-            ("address", address),
-            ("local", local),
-            ("domain", domain),
-        ]
-    )
+    return {
+        "display_name": display_name,
+        "address": address,
+        "local": local,
+        "domain": domain,
+    }
 
 
 def get_filename_safe_string(string: str) -> str:
@@ -553,7 +558,7 @@ def is_mbox(path: str) -> bool:
     return _is_mbox
 
 
-def is_outlook_msg(content) -> bool:
+def is_outlook_msg(content: Union[bytes, Any]) -> bool:
     """
     Checks if the given content is an Outlook msg OLE/MSG file
 
@@ -586,13 +591,14 @@ def convert_outlook_msg(msg_bytes: bytes) -> str:
     os.chdir(tmp_dir)
     with open("sample.msg", "wb") as msg_file:
         msg_file.write(msg_bytes)
+    rfc822_bytes: bytes
     try:
         subprocess.check_call(
             ["msgconvert", "sample.msg"], stdout=null_file, stderr=null_file
         )
         eml_path = "sample.eml"
         with open(eml_path, "rb") as eml_file:
-            rfc822 = eml_file.read()
+            rfc822_bytes = eml_file.read()
     except FileNotFoundError:
         raise EmailParserError(
             "Failed to convert Outlook MSG: msgconvert utility not found"
@@ -601,12 +607,12 @@ def convert_outlook_msg(msg_bytes: bytes) -> str:
         os.chdir(orig_dir)
         shutil.rmtree(tmp_dir)
 
-    return rfc822
+    return rfc822_bytes.decode("utf-8", errors="replace")
 
 
 def parse_email(
     data: Union[bytes, str], *, strip_attachment_payloads: Optional[bool] = False
-):
+) -> dict[str, Any]:
     """
     A simplified email parser
 
@@ -621,7 +627,8 @@ def parse_email(
     if isinstance(data, bytes):
         if is_outlook_msg(data):
             data = convert_outlook_msg(data)
-        data = data.decode("utf-8", errors="replace")
+        else:
+            data = data.decode("utf-8", errors="replace")
     parsed_email = mailparser.parse_from_string(data)
     headers = json.loads(parsed_email.headers_json).copy()
     parsed_email = json.loads(parsed_email.mail_json).copy()

parsedmarc/webhook.py

@@ -4,8 +4,6 @@ from __future__ import annotations
 
 from typing import Any, Optional, Union
 
-from collections import OrderedDict
-
 import requests
 
 from parsedmarc import logger
@@ -40,19 +38,19 @@ class WebhookClient(object):
             "Content-Type": "application/json",
         }
 
-    def save_forensic_report_to_webhook(self, report: OrderedDict[str, Any]):
+    def save_forensic_report_to_webhook(self, report: str):
         try:
             self._send_to_webhook(self.forensic_url, report)
         except Exception as error_:
             logger.error("Webhook Error: {0}".format(error_.__str__()))
 
-    def save_smtp_tls_report_to_webhook(self, report: OrderedDict[str, Any]):
+    def save_smtp_tls_report_to_webhook(self, report: str):
         try:
             self._send_to_webhook(self.smtp_tls_url, report)
         except Exception as error_:
             logger.error("Webhook Error: {0}".format(error_.__str__()))
 
-    def save_aggregate_report_to_webhook(self, report: OrderedDict[str, Any]):
+    def save_aggregate_report_to_webhook(self, report: str):
         try:
             self._send_to_webhook(self.aggregate_url, report)
         except Exception as error_:

pyproject.toml

@@ -2,6 +2,7 @@
 requires = [
     "hatchling>=1.27.0",
 ]
+requires_python = ">=3.9,<3.14"
 build-backend = "hatchling.build"
 
 [project]