diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aaca54..17091d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -278,7 +278,7 @@ Removed improper spaces from `base_reverse_dns_map.csv` (Closes #612) - Fix handling of non-domain organization names (PR #411 fixes issue #410) - Skip processing of aggregate reports with a date range that is too long to be valid (PR #408 fixes issue #282) -- Better error handeling for Elasticsearch queries and file parsing (PR #417) +- Better error handling for Elasticsearch queries and file parsing (PR #417) 8.6.0 ----- @@ -474,7 +474,7 @@ Removed improper spaces from `base_reverse_dns_map.csv` (Closes #612) 6.7.3 ----- -- Make `dkim_aligned` and `spf_aligned` case insensitive (PR #132) +- Make `dkim_aligned` and `spf_aligned` case-insensitive (PR #132) 6.7.2 ----- @@ -511,7 +511,7 @@ Removed improper spaces from `base_reverse_dns_map.csv` (Closes #612) 6.5.5 ----- -- Set set minimum `publicsuffix2` version +- Set minimum `publicsuffix2` version 6.5.4 ----- @@ -641,7 +641,7 @@ in the ``elasticsearch`` configuration file section (closes issue #78) 6.2.0 ----- -- Add support for multi-process parallelized processing via CLI (Thanks zscholl - PR #62) +- Add support for multiprocess parallelized processing via CLI (Thanks zscholl - PR #62) - Save sha256 hashes of attachments in forensic samples to Elasticsearch 6.1.8 @@ -755,7 +755,7 @@ in the ``elasticsearch`` configuration file section (closes issue #78) 5.1.2 ----- -- Workaround unexpected Office365/Exchange IMAP responses +- Workaround unexpected Office 365/Exchange IMAP responses 5.1.1 ----- @@ -982,7 +982,7 @@ system path - Fall back to plain text SMTP if `--outgoing-ssl` is not used and `STARTTLS` is not supported by the server - Always use `\n` as the newline when generating CSVs -- Workaround for random Exchange/Office365 `Server Unavailable` IMAP errors +- Workaround for random Exchange/Office 365 `Server Unavailable` IMAP errors 3.9.7 ----- @@ -1060,7 +1060,7 @@ does not work - fixes dovecot support (#5) 3.7.2 ----- -- Change uses uses of the `DocType` class to `Document`, to properly support `elasticsearch-dsl` `6.2.0` (this also fixes use in pypy) +- Change uses of the `DocType` class to `Document`, to properly support `elasticsearch-dsl` `6.2.0` (this also fixes use in pypy) - Add documentation for installation under pypy 3.7.1 diff --git a/docs/source/conf.py b/docs/source/conf.py index a83821a..24f0337 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,7 +20,7 @@ from parsedmarc import __version__ # -- Project information ----------------------------------------------------- project = "parsedmarc" -copyright = "2018 - 2023, Sean Whalen and contributors" +copyright = "2018 - 2025, Sean Whalen and contributors" author = "Sean Whalen and contributors" # The version info for the project you're documenting, acts as replacement for diff --git a/docs/source/usage.md b/docs/source/usage.md index bd52f50..00d7e28 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -169,7 +169,7 @@ The full set of configuration options are: IDLE response or the number of seconds until the next mail check (Default: `30`) - `since` - str: Search for messages since certain time. (Examples: `5m|3h|2d|1w`) - Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}). + Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}. Defaults to `1d` if incorrect value is provided. - `imap` - `host` - str: The IMAP server hostname or IP address @@ -254,7 +254,7 @@ The full set of configuration options are: ::: - `user` - str: Basic auth username - `password` - str: Basic auth password - - `apiKey` - str: API key + - `api_key` - str: API key - `ssl` - bool: Use an encrypted SSL/TLS connection (Default: `True`) - `timeout` - float: Timeout in seconds (Default: 60) @@ -277,7 +277,7 @@ The full set of configuration options are: ::: - `user` - str: Basic auth username - `password` - str: Basic auth password - - `apiKey` - str: API key + - `api_key` - str: API key - `ssl` - bool: Use an encrypted SSL/TLS connection (Default: `True`) - `timeout` - float: Timeout in seconds (Default: 60) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index b6a2209..a3e6a5c 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1604,7 +1604,7 @@ def get_dmarc_reports_from_mailbox( if isinstance(connection, IMAPConnection): logger.debug( "Only days and weeks values in 'since' option are \ - considered for IMAP conections. Examples: 2d or 1w" + considered for IMAP connections. Examples: 2d or 1w" ) since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).date() current_time = datetime.now(timezone.utc).date() diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index c7f6592..a0ad6b8 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -591,7 +591,7 @@ def _main(): elasticsearch_monthly_indexes=False, elasticsearch_username=None, elasticsearch_password=None, - elasticsearch_apiKey=None, + elasticsearch_api_key=None, opensearch_hosts=None, opensearch_timeout=60, opensearch_number_of_shards=1, @@ -603,7 +603,7 @@ def _main(): opensearch_monthly_indexes=False, opensearch_username=None, opensearch_password=None, - opensearch_apiKey=None, + opensearch_api_key=None, kafka_hosts=None, kafka_username=None, kafka_password=None, @@ -974,8 +974,12 @@ def _main(): opts.elasticsearch_username = elasticsearch_config["user"] if "password" in elasticsearch_config: opts.elasticsearch_password = elasticsearch_config["password"] + # Until 8.20 if "apiKey" in elasticsearch_config: opts.elasticsearch_apiKey = elasticsearch_config["apiKey"] + # Since 8.20 + if "api_key" in elasticsearch_config: + opts.elasticsearch_apiKey = elasticsearch_config["api_key"] if "opensearch" in config: opensearch_config = config["opensearch"] @@ -1010,8 +1014,12 @@ def _main(): opts.opensearch_username = opensearch_config["user"] if "password" in opensearch_config: opts.opensearch_password = opensearch_config["password"] + # Until 8.20 if "apiKey" in opensearch_config: opts.opensearch_apiKey = opensearch_config["apiKey"] + # Since 8.20 + if "api_key" in opensearch_config: + opts.opensearch_apiKey = opensearch_config["api_key"] if "splunk_hec" in config.sections(): hec_config = config["splunk_hec"] @@ -1274,7 +1282,7 @@ def _main(): opts.elasticsearch_ssl_cert_path, opts.elasticsearch_username, opts.elasticsearch_password, - opts.elasticsearch_apiKey, + opts.elasticsearch_api_key, timeout=opts.elasticsearch_timeout, ) elastic.migrate_indexes( @@ -1306,7 +1314,7 @@ def _main(): opts.opensearch_ssl_cert_path, opts.opensearch_username, opts.opensearch_password, - opts.opensearch_apiKey, + opts.opensearch_api_key, timeout=opts.opensearch_timeout, ) opensearch.migrate_indexes( @@ -1512,7 +1520,7 @@ def _main(): if opts.imap_skip_certificate_verification: logger.debug("Skipping IMAP certificate verification") verify = False - if opts.imap_ssl is False: + if not opts.imap_ssl: ssl = False mailbox_connection = IMAPConnection( diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 5c7ca91..9c71f20 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -264,7 +264,7 @@ def set_hosts( ssl_cert_path=None, username=None, password=None, - apiKey=None, + api_key=None, timeout=60.0, ): """ @@ -276,7 +276,7 @@ def set_hosts( ssl_cert_path (str): Path to the certificate chain username (str): The username to use for authentication password (str): The password to use for authentication - apiKey (str): The Base64 encoded API key to use for authentication + api_key (str): The Base64 encoded API key to use for authentication timeout (float): Timeout in seconds """ if not isinstance(hosts, list): @@ -291,8 +291,8 @@ def set_hosts( conn_params["verify_certs"] = False if username: conn_params["http_auth"] = username + ":" + password - if apiKey: - conn_params["api_key"] = apiKey + if api_key: + conn_params["api_key"] = api_key connections.create_connection(**conn_params) diff --git a/parsedmarc/mail/gmail.py b/parsedmarc/mail/gmail.py index e25b20b..f548d3d 100644 --- a/parsedmarc/mail/gmail.py +++ b/parsedmarc/mail/gmail.py @@ -152,3 +152,4 @@ class GmailConnection(MailboxConnection): for label in labels: if label_name == label["id"] or label_name == label["name"]: return label["id"] + return "" diff --git a/parsedmarc/opensearch.py b/parsedmarc/opensearch.py index d947ea7..003400c 100644 --- a/parsedmarc/opensearch.py +++ b/parsedmarc/opensearch.py @@ -264,7 +264,7 @@ def set_hosts( ssl_cert_path=None, username=None, password=None, - apiKey=None, + api_key=None, timeout=60.0, ): """ @@ -276,7 +276,7 @@ def set_hosts( ssl_cert_path (str): Path to the certificate chain username (str): The username to use for authentication password (str): The password to use for authentication - apiKey (str): The Base64 encoded API key to use for authentication + api_key (str): The Base64 encoded API key to use for authentication timeout (float): Timeout in seconds """ if not isinstance(hosts, list): @@ -291,8 +291,8 @@ def set_hosts( conn_params["verify_certs"] = False if username: conn_params["http_auth"] = username + ":" + password - if apiKey: - conn_params["api_key"] = apiKey + if api_key: + conn_params["api_key"] = api_key connections.create_connection(**conn_params) diff --git a/parsedmarc/resources/maps/find_unknown_base_reverse_dns.py b/parsedmarc/resources/maps/find_unknown_base_reverse_dns.py index a23d3ea..09d3f92 100755 --- a/parsedmarc/resources/maps/find_unknown_base_reverse_dns.py +++ b/parsedmarc/resources/maps/find_unknown_base_reverse_dns.py @@ -13,8 +13,6 @@ def _main(): csv_headers = ["source_name", "message_count"] - output_rows = [] - known_unknown_domains = [] psl_overrides = [] known_domains = [] diff --git a/parsedmarc/s3.py b/parsedmarc/s3.py index 1b6c374..a80331d 100644 --- a/parsedmarc/s3.py +++ b/parsedmarc/s3.py @@ -8,7 +8,7 @@ from parsedmarc.utils import human_timestamp_to_datetime class S3Client(object): - """A client for a Amazon S3""" + """A client for interacting with Amazon S3""" def __init__( self, diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 486b37c..24f6dd2 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -277,7 +277,7 @@ def get_ip_address_country(ip_address, db_path=None): ] if db_path is not None: - if os.path.isfile(db_path) is False: + if not os.path.isfile(db_path): db_path = None logger.warning( f"No file exists at {db_path}. Falling back to an " diff --git a/pyproject.toml b/pyproject.toml index f7dd08d..c6cd44a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,11 +86,11 @@ include = [ [tool.hatch.build] exclude = [ -"base_reverse_dns.csv", -"find_bad_utf8.py", -"find_unknown_base_reverse_dns.py", -"unknown_base_reverse_dns.csv", -"sortmaps.py", -"README.md", -"*.bak" + "base_reverse_dns.csv", + "find_bad_utf8.py", + "find_unknown_base_reverse_dns.py", + "unknown_base_reverse_dns.csv", + "sortmaps.py", + "README.md", + "*.bak" ]