mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-03-28 09:22:45 +00:00
* Enhance mailbox connection watch method to support reload functionality - Updated the `watch` method in `GmailConnection`, `MSGraphConnection`, `IMAPConnection`, `MaildirConnection`, and the abstract `MailboxConnection` class to accept an optional `should_reload` parameter. This allows the method to check if a reload is necessary and exit the loop if so. - Modified related tests to accommodate the new method signature. - Changed logger calls from `critical` to `error` for consistency in logging severity. - Added a new settings file for Claude with specific permissions for testing and code checks. * Update parsedmarc/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update parsedmarc/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * [WIP] SIGHUP-based configuration reload for watch mode (#698) * Initial plan * Fix reload state consistency, resource leaks, stale opts; add tests Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/3c2e0bb9-7e2d-4efa-aef6-d2b98478b921 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * [WIP] SIGHUP-based configuration reload for watch mode (#699) * Initial plan * Fix review comments: ConfigurationError wrapping, duplicate parse args, bool parsing, Kafka required topics, should_reload kwarg, SIGHUP test skips Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/0779003c-ccbe-4d76-9748-801dbc238b96 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * SIGHUP-based configuration reload: address review feedback (#700) * Initial plan * Address review feedback: kafka_ssl, duplicate silent, exception chain, log file reload, should_reload timing Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/a8a43c55-23fa-4471-abe6-7ac966f381f9 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * Update parsedmarc/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Best-effort initialization for optional output clients in watch mode (#701) * Initial plan * Wrap optional output client init in try/except for best-effort initialization Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/59241d4e-1b05-4a92-b2d2-e6d13d10a4fd --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * Fix SIGHUP reload tight-loop in watch mode (#702) * Initial plan * Fix _reload_requested tight-loop: reset flag before reload to capture concurrent SIGHUPs Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/879d0bb1-9037-41f7-bc89-f59611956d2e --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * Update parsedmarc/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Fix resource leak when HEC config is invalid in `_init_output_clients()` (#703) * Initial plan * Fix resource leak: validate HEC settings before creating any output clients Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/38c73e09-789d-4d41-b75e-bbc61418859d --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * Ensure SIGHUP never triggers a new email batch across all watch() implementations (#704) * Initial plan * Ensure SIGHUP never starts a new email batch in any watch() implementation Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/45d5be30-8f6b-4200-9bdd-15c655033f17 --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * SIGHUP-based config reload for watch mode: address review feedback (#705) * Initial plan * Address review feedback: Kafka SSL context, SIGHUP handler safety, test formatting Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> Agent-Logs-Url: https://github.com/domainaware/parsedmarc/sessions/8f2fd48f-32a4-4258-9a89-06f7c7ac29bf --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> * Reverted changes by copilot that turned errors into warnings * Enhance usage documentation for config reload: clarify behavior on successful reload and error handling * Update CHANGELOG.md to reflect config reload enhancements * Add pytest command to settings for silent output during testing * Enhance resource management: add close methods for S3Client and HECClient, and improve IMAP connection handling during IDLE. Update CHANGELOG.md for config reload improvements and bug fixes. * Update changelog to not include fixes within the same unreleased version * Refactor changelog entries for clarity and consistency in configuration reload section * Fix changelog entry for msgraph configuration check * Update CHANGELOG..md * make single list items on one line in the changelog instead of doing hard wraps * Remove incorrect IMAP changes * Rename 'should_reload' parameter to 'config_reloading' in mailbox connection methods for clarity * Restore startup configuration checks * Improve error logging for Elasticsearch and OpenSearch exceptions * Bump version to 9.3.0 in constants.py * Refactor GelfClient methods to use specific report types instead of generic dicts * Refactor tests to use assertions consistently and improve type hints --------- Co-authored-by: Sean Whalen <seanthegeek@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
187 lines
7.2 KiB
Python
187 lines
7.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import logging.handlers
|
|
import socket
|
|
import ssl
|
|
import time
|
|
from typing import Any, Optional
|
|
|
|
from parsedmarc import (
|
|
parsed_aggregate_reports_to_csv_rows,
|
|
parsed_forensic_reports_to_csv_rows,
|
|
parsed_smtp_tls_reports_to_csv_rows,
|
|
)
|
|
|
|
|
|
class SyslogClient(object):
|
|
"""A client for Syslog"""
|
|
|
|
def __init__(
|
|
self,
|
|
server_name: str,
|
|
server_port: int,
|
|
protocol: str = "udp",
|
|
cafile_path: Optional[str] = None,
|
|
certfile_path: Optional[str] = None,
|
|
keyfile_path: Optional[str] = None,
|
|
timeout: float = 5.0,
|
|
retry_attempts: int = 3,
|
|
retry_delay: int = 5,
|
|
):
|
|
"""
|
|
Initializes the SyslogClient
|
|
Args:
|
|
server_name (str): The Syslog server
|
|
server_port (int): The Syslog port
|
|
protocol (str): The protocol to use: "udp", "tcp", or "tls" (Default: "udp")
|
|
cafile_path (str): Path to CA certificate file for TLS server verification (Optional)
|
|
certfile_path (str): Path to client certificate file for TLS authentication (Optional)
|
|
keyfile_path (str): Path to client private key file for TLS authentication (Optional)
|
|
timeout (float): Connection timeout in seconds for TCP/TLS (Default: 5.0)
|
|
retry_attempts (int): Number of retry attempts for failed connections (Default: 3)
|
|
retry_delay (int): Delay in seconds between retry attempts (Default: 5)
|
|
"""
|
|
self.server_name = server_name
|
|
self.server_port = server_port
|
|
self.protocol = protocol.lower()
|
|
self.timeout = timeout
|
|
self.retry_attempts = retry_attempts
|
|
self.retry_delay = retry_delay
|
|
|
|
self.logger = logging.getLogger("parsedmarc_syslog")
|
|
self.logger.setLevel(logging.INFO)
|
|
|
|
# Create the appropriate syslog handler based on protocol
|
|
self.log_handler = self._create_syslog_handler(
|
|
server_name,
|
|
server_port,
|
|
self.protocol,
|
|
cafile_path,
|
|
certfile_path,
|
|
keyfile_path,
|
|
timeout,
|
|
retry_attempts,
|
|
retry_delay,
|
|
)
|
|
|
|
self.logger.addHandler(self.log_handler)
|
|
|
|
def _create_syslog_handler(
|
|
self,
|
|
server_name: str,
|
|
server_port: int,
|
|
protocol: str,
|
|
cafile_path: Optional[str],
|
|
certfile_path: Optional[str],
|
|
keyfile_path: Optional[str],
|
|
timeout: float,
|
|
retry_attempts: int,
|
|
retry_delay: int,
|
|
) -> logging.handlers.SysLogHandler:
|
|
"""
|
|
Creates a SysLogHandler with the specified protocol and TLS settings
|
|
"""
|
|
if protocol == "udp":
|
|
# UDP protocol (default, backward compatible)
|
|
return logging.handlers.SysLogHandler(
|
|
address=(server_name, server_port),
|
|
socktype=socket.SOCK_DGRAM,
|
|
)
|
|
elif protocol in ["tcp", "tls"]:
|
|
# TCP or TLS protocol with retry logic
|
|
for attempt in range(1, retry_attempts + 1):
|
|
try:
|
|
if protocol == "tcp":
|
|
# TCP without TLS
|
|
handler = logging.handlers.SysLogHandler(
|
|
address=(server_name, server_port),
|
|
socktype=socket.SOCK_STREAM,
|
|
)
|
|
# Set timeout on the socket
|
|
if hasattr(handler, "socket") and handler.socket:
|
|
handler.socket.settimeout(timeout)
|
|
return handler
|
|
else:
|
|
# TLS protocol
|
|
# Create SSL context with secure defaults
|
|
ssl_context = ssl.create_default_context()
|
|
|
|
# Explicitly set minimum TLS version to 1.2 for security
|
|
ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
|
|
# Configure server certificate verification
|
|
if cafile_path:
|
|
ssl_context.load_verify_locations(cafile=cafile_path)
|
|
|
|
# Configure client certificate authentication
|
|
if certfile_path and keyfile_path:
|
|
ssl_context.load_cert_chain(
|
|
certfile=certfile_path,
|
|
keyfile=keyfile_path,
|
|
)
|
|
elif certfile_path or keyfile_path:
|
|
# Warn if only one of the two required parameters is provided
|
|
self.logger.warning(
|
|
"Both certfile_path and keyfile_path are required for "
|
|
"client certificate authentication. Client authentication "
|
|
"will not be used."
|
|
)
|
|
|
|
# Create TCP handler first
|
|
handler = logging.handlers.SysLogHandler(
|
|
address=(server_name, server_port),
|
|
socktype=socket.SOCK_STREAM,
|
|
)
|
|
|
|
# Wrap socket with TLS
|
|
if hasattr(handler, "socket") and handler.socket:
|
|
handler.socket = ssl_context.wrap_socket(
|
|
handler.socket,
|
|
server_hostname=server_name,
|
|
)
|
|
handler.socket.settimeout(timeout)
|
|
|
|
return handler
|
|
|
|
except Exception as e:
|
|
if attempt < retry_attempts:
|
|
self.logger.warning(
|
|
f"Syslog connection attempt {attempt}/{retry_attempts} failed: {e}. "
|
|
f"Retrying in {retry_delay} seconds..."
|
|
)
|
|
time.sleep(retry_delay)
|
|
else:
|
|
self.logger.error(
|
|
f"Syslog connection failed after {retry_attempts} attempts: {e}"
|
|
)
|
|
raise
|
|
else:
|
|
raise ValueError(
|
|
f"Invalid protocol '{protocol}'. Must be 'udp', 'tcp', or 'tls'."
|
|
)
|
|
|
|
def save_aggregate_report_to_syslog(self, aggregate_reports: list[dict[str, Any]]):
|
|
rows = parsed_aggregate_reports_to_csv_rows(aggregate_reports)
|
|
for row in rows:
|
|
self.logger.info(json.dumps(row))
|
|
|
|
def save_forensic_report_to_syslog(self, forensic_reports: list[dict[str, Any]]):
|
|
rows = parsed_forensic_reports_to_csv_rows(forensic_reports)
|
|
for row in rows:
|
|
self.logger.info(json.dumps(row))
|
|
|
|
def save_smtp_tls_report_to_syslog(self, smtp_tls_reports: list[dict[str, Any]]):
|
|
rows = parsed_smtp_tls_reports_to_csv_rows(smtp_tls_reports)
|
|
for row in rows:
|
|
self.logger.info(json.dumps(row))
|
|
|
|
def close(self):
|
|
"""Remove and close the syslog handler, releasing its socket."""
|
|
self.logger.removeHandler(self.log_handler)
|
|
self.log_handler.close()
|