mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-02-23 01:26:27 +00:00
Compare commits
2 Commits
master
...
copilot/op
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2192d0bfd4 | ||
|
|
f1c6ebef1d |
@@ -1,13 +1,5 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
## 9.1.0
|
|
||||||
|
|
||||||
## Enhancements
|
|
||||||
|
|
||||||
- Add TCP and TLS support for syslog output. (#656)
|
|
||||||
- Skip DNS lookups in GitHub Actions to prevent DNS timeouts during tests timeouts. (#657)
|
|
||||||
- Remove microseconds from DMARC aggregate report time ranges before parsing them.
|
|
||||||
|
|
||||||
## 9.0.10
|
## 9.0.10
|
||||||
|
|
||||||
- Support Python 3.14+
|
- Support Python 3.14+
|
||||||
|
|||||||
@@ -171,8 +171,8 @@ The full set of configuration options are:
|
|||||||
- `check_timeout` - int: Number of seconds to wait for a IMAP
|
- `check_timeout` - int: Number of seconds to wait for a IMAP
|
||||||
IDLE response or the number of seconds until the next
|
IDLE response or the number of seconds until the next
|
||||||
mail check (Default: `30`)
|
mail check (Default: `30`)
|
||||||
- `since` - str: Search for messages since certain time. (Examples: `5m|3h|2d|1w`)
|
- `since` - str: Search for messages since certain time. (Examples: `5m|3h|2d|1w`)
|
||||||
Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}.
|
Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}.
|
||||||
Defaults to `1d` if incorrect value is provided.
|
Defaults to `1d` if incorrect value is provided.
|
||||||
- `imap`
|
- `imap`
|
||||||
- `host` - str: The IMAP server hostname or IP address
|
- `host` - str: The IMAP server hostname or IP address
|
||||||
@@ -240,7 +240,7 @@ The full set of configuration options are:
|
|||||||
group and use that as the group id.
|
group and use that as the group id.
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
New-ApplicationAccessPolicy -AccessRight RestrictAccess
|
New-ApplicationAccessPolicy -AccessRight RestrictAccess
|
||||||
-AppId "<CLIENT_ID>" -PolicyScopeGroupId "<MAILBOX>"
|
-AppId "<CLIENT_ID>" -PolicyScopeGroupId "<MAILBOX>"
|
||||||
-Description "Restrict access to dmarc reports mailbox."
|
-Description "Restrict access to dmarc reports mailbox."
|
||||||
```
|
```
|
||||||
@@ -336,65 +336,13 @@ The full set of configuration options are:
|
|||||||
- `secret_access_key` - str: The secret access key (Optional)
|
- `secret_access_key` - str: The secret access key (Optional)
|
||||||
- `syslog`
|
- `syslog`
|
||||||
- `server` - str: The Syslog server name or IP address
|
- `server` - str: The Syslog server name or IP address
|
||||||
- `port` - int: The port to use (Default: `514`)
|
- `port` - int: The UDP port to use (Default: `514`)
|
||||||
- `protocol` - str: The protocol to use: `udp`, `tcp`, or `tls` (Default: `udp`)
|
|
||||||
- `cafile_path` - str: Path to CA certificate file for TLS server verification (Optional)
|
|
||||||
- `certfile_path` - str: Path to client certificate file for TLS authentication (Optional)
|
|
||||||
- `keyfile_path` - str: Path to client private key file for TLS authentication (Optional)
|
|
||||||
- `timeout` - float: Connection timeout in seconds for TCP/TLS (Default: `5.0`)
|
|
||||||
- `retry_attempts` - int: Number of retry attempts for failed connections (Default: `3`)
|
|
||||||
- `retry_delay` - int: Delay in seconds between retry attempts (Default: `5`)
|
|
||||||
|
|
||||||
**Example UDP configuration (default):**
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[syslog]
|
|
||||||
server = syslog.example.com
|
|
||||||
port = 514
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example TCP configuration:**
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[syslog]
|
|
||||||
server = syslog.example.com
|
|
||||||
port = 6514
|
|
||||||
protocol = tcp
|
|
||||||
timeout = 10.0
|
|
||||||
retry_attempts = 5
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example TLS configuration with server verification:**
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[syslog]
|
|
||||||
server = syslog.example.com
|
|
||||||
port = 6514
|
|
||||||
protocol = tls
|
|
||||||
cafile_path = /path/to/ca-cert.pem
|
|
||||||
timeout = 10.0
|
|
||||||
```
|
|
||||||
|
|
||||||
**Example TLS configuration with mutual authentication:**
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[syslog]
|
|
||||||
server = syslog.example.com
|
|
||||||
port = 6514
|
|
||||||
protocol = tls
|
|
||||||
cafile_path = /path/to/ca-cert.pem
|
|
||||||
certfile_path = /path/to/client-cert.pem
|
|
||||||
keyfile_path = /path/to/client-key.pem
|
|
||||||
timeout = 10.0
|
|
||||||
retry_attempts = 3
|
|
||||||
retry_delay = 5
|
|
||||||
```
|
|
||||||
- `gmail_api`
|
- `gmail_api`
|
||||||
- `credentials_file` - str: Path to file containing the
|
- `credentials_file` - str: Path to file containing the
|
||||||
credentials, None to disable (Default: `None`)
|
credentials, None to disable (Default: `None`)
|
||||||
- `token_file` - str: Path to save the token file
|
- `token_file` - str: Path to save the token file
|
||||||
(Default: `.token`)
|
(Default: `.token`)
|
||||||
|
|
||||||
:::{note}
|
:::{note}
|
||||||
credentials_file and token_file can be got with [quickstart](https://developers.google.com/gmail/api/quickstart/python).Please change the scope to `https://www.googleapis.com/auth/gmail.modify`.
|
credentials_file and token_file can be got with [quickstart](https://developers.google.com/gmail/api/quickstart/python).Please change the scope to `https://www.googleapis.com/auth/gmail.modify`.
|
||||||
:::
|
:::
|
||||||
@@ -494,7 +442,7 @@ Update the limit to 2k per example:
|
|||||||
PUT _cluster/settings
|
PUT _cluster/settings
|
||||||
{
|
{
|
||||||
"persistent" : {
|
"persistent" : {
|
||||||
"cluster.max_shards_per_node" : 2000
|
"cluster.max_shards_per_node" : 2000
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -697,13 +697,6 @@ def _main():
|
|||||||
s3_secret_access_key=None,
|
s3_secret_access_key=None,
|
||||||
syslog_server=None,
|
syslog_server=None,
|
||||||
syslog_port=None,
|
syslog_port=None,
|
||||||
syslog_protocol=None,
|
|
||||||
syslog_cafile_path=None,
|
|
||||||
syslog_certfile_path=None,
|
|
||||||
syslog_keyfile_path=None,
|
|
||||||
syslog_timeout=None,
|
|
||||||
syslog_retry_attempts=None,
|
|
||||||
syslog_retry_delay=None,
|
|
||||||
gmail_api_credentials_file=None,
|
gmail_api_credentials_file=None,
|
||||||
gmail_api_token_file=None,
|
gmail_api_token_file=None,
|
||||||
gmail_api_include_spam_trash=False,
|
gmail_api_include_spam_trash=False,
|
||||||
@@ -1246,28 +1239,6 @@ def _main():
|
|||||||
opts.syslog_port = syslog_config["port"]
|
opts.syslog_port = syslog_config["port"]
|
||||||
else:
|
else:
|
||||||
opts.syslog_port = 514
|
opts.syslog_port = 514
|
||||||
if "protocol" in syslog_config:
|
|
||||||
opts.syslog_protocol = syslog_config["protocol"]
|
|
||||||
else:
|
|
||||||
opts.syslog_protocol = "udp"
|
|
||||||
if "cafile_path" in syslog_config:
|
|
||||||
opts.syslog_cafile_path = syslog_config["cafile_path"]
|
|
||||||
if "certfile_path" in syslog_config:
|
|
||||||
opts.syslog_certfile_path = syslog_config["certfile_path"]
|
|
||||||
if "keyfile_path" in syslog_config:
|
|
||||||
opts.syslog_keyfile_path = syslog_config["keyfile_path"]
|
|
||||||
if "timeout" in syslog_config:
|
|
||||||
opts.syslog_timeout = float(syslog_config["timeout"])
|
|
||||||
else:
|
|
||||||
opts.syslog_timeout = 5.0
|
|
||||||
if "retry_attempts" in syslog_config:
|
|
||||||
opts.syslog_retry_attempts = int(syslog_config["retry_attempts"])
|
|
||||||
else:
|
|
||||||
opts.syslog_retry_attempts = 3
|
|
||||||
if "retry_delay" in syslog_config:
|
|
||||||
opts.syslog_retry_delay = int(syslog_config["retry_delay"])
|
|
||||||
else:
|
|
||||||
opts.syslog_retry_delay = 5
|
|
||||||
|
|
||||||
if "gmail_api" in config.sections():
|
if "gmail_api" in config.sections():
|
||||||
gmail_api_config = config["gmail_api"]
|
gmail_api_config = config["gmail_api"]
|
||||||
@@ -1465,17 +1436,6 @@ def _main():
|
|||||||
syslog_client = syslog.SyslogClient(
|
syslog_client = syslog.SyslogClient(
|
||||||
server_name=opts.syslog_server,
|
server_name=opts.syslog_server,
|
||||||
server_port=int(opts.syslog_port),
|
server_port=int(opts.syslog_port),
|
||||||
protocol=opts.syslog_protocol or "udp",
|
|
||||||
cafile_path=opts.syslog_cafile_path,
|
|
||||||
certfile_path=opts.syslog_certfile_path,
|
|
||||||
keyfile_path=opts.syslog_keyfile_path,
|
|
||||||
timeout=opts.syslog_timeout if opts.syslog_timeout is not None else 5.0,
|
|
||||||
retry_attempts=opts.syslog_retry_attempts
|
|
||||||
if opts.syslog_retry_attempts is not None
|
|
||||||
else 3,
|
|
||||||
retry_delay=opts.syslog_retry_delay
|
|
||||||
if opts.syslog_retry_delay is not None
|
|
||||||
else 5,
|
|
||||||
)
|
)
|
||||||
except Exception as error_:
|
except Exception as error_:
|
||||||
logger.error("Syslog Error: {0}".format(error_.__str__()))
|
logger.error("Syslog Error: {0}".format(error_.__str__()))
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
__version__ = "9.1.0"
|
__version__ = "9.0.10"
|
||||||
|
|
||||||
USER_AGENT = f"parsedmarc/{__version__}"
|
USER_AGENT = f"parsedmarc/{__version__}"
|
||||||
|
|||||||
@@ -6,10 +6,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
import logging.handlers
|
||||||
import socket
|
from typing import Any
|
||||||
import ssl
|
|
||||||
import time
|
|
||||||
from typing import Any, Optional
|
|
||||||
|
|
||||||
from parsedmarc import (
|
from parsedmarc import (
|
||||||
parsed_aggregate_reports_to_csv_rows,
|
parsed_aggregate_reports_to_csv_rows,
|
||||||
@@ -21,150 +18,20 @@ from parsedmarc import (
|
|||||||
class SyslogClient(object):
|
class SyslogClient(object):
|
||||||
"""A client for Syslog"""
|
"""A client for Syslog"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, server_name: str, server_port: int):
|
||||||
self,
|
|
||||||
server_name: str,
|
|
||||||
server_port: int,
|
|
||||||
protocol: str = "udp",
|
|
||||||
cafile_path: Optional[str] = None,
|
|
||||||
certfile_path: Optional[str] = None,
|
|
||||||
keyfile_path: Optional[str] = None,
|
|
||||||
timeout: float = 5.0,
|
|
||||||
retry_attempts: int = 3,
|
|
||||||
retry_delay: int = 5,
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
Initializes the SyslogClient
|
Initializes the SyslogClient
|
||||||
Args:
|
Args:
|
||||||
server_name (str): The Syslog server
|
server_name (str): The Syslog server
|
||||||
server_port (int): The Syslog port
|
server_port (int): The Syslog UDP port
|
||||||
protocol (str): The protocol to use: "udp", "tcp", or "tls" (Default: "udp")
|
|
||||||
cafile_path (str): Path to CA certificate file for TLS server verification (Optional)
|
|
||||||
certfile_path (str): Path to client certificate file for TLS authentication (Optional)
|
|
||||||
keyfile_path (str): Path to client private key file for TLS authentication (Optional)
|
|
||||||
timeout (float): Connection timeout in seconds for TCP/TLS (Default: 5.0)
|
|
||||||
retry_attempts (int): Number of retry attempts for failed connections (Default: 3)
|
|
||||||
retry_delay (int): Delay in seconds between retry attempts (Default: 5)
|
|
||||||
"""
|
"""
|
||||||
self.server_name = server_name
|
self.server_name = server_name
|
||||||
self.server_port = server_port
|
self.server_port = server_port
|
||||||
self.protocol = protocol.lower()
|
|
||||||
self.timeout = timeout
|
|
||||||
self.retry_attempts = retry_attempts
|
|
||||||
self.retry_delay = retry_delay
|
|
||||||
|
|
||||||
self.logger = logging.getLogger("parsedmarc_syslog")
|
self.logger = logging.getLogger("parsedmarc_syslog")
|
||||||
self.logger.setLevel(logging.INFO)
|
self.logger.setLevel(logging.INFO)
|
||||||
|
log_handler = logging.handlers.SysLogHandler(address=(server_name, server_port))
|
||||||
# Create the appropriate syslog handler based on protocol
|
|
||||||
log_handler = self._create_syslog_handler(
|
|
||||||
server_name,
|
|
||||||
server_port,
|
|
||||||
self.protocol,
|
|
||||||
cafile_path,
|
|
||||||
certfile_path,
|
|
||||||
keyfile_path,
|
|
||||||
timeout,
|
|
||||||
retry_attempts,
|
|
||||||
retry_delay,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.logger.addHandler(log_handler)
|
self.logger.addHandler(log_handler)
|
||||||
|
|
||||||
def _create_syslog_handler(
|
|
||||||
self,
|
|
||||||
server_name: str,
|
|
||||||
server_port: int,
|
|
||||||
protocol: str,
|
|
||||||
cafile_path: Optional[str],
|
|
||||||
certfile_path: Optional[str],
|
|
||||||
keyfile_path: Optional[str],
|
|
||||||
timeout: float,
|
|
||||||
retry_attempts: int,
|
|
||||||
retry_delay: int,
|
|
||||||
) -> logging.handlers.SysLogHandler:
|
|
||||||
"""
|
|
||||||
Creates a SysLogHandler with the specified protocol and TLS settings
|
|
||||||
"""
|
|
||||||
if protocol == "udp":
|
|
||||||
# UDP protocol (default, backward compatible)
|
|
||||||
return logging.handlers.SysLogHandler(
|
|
||||||
address=(server_name, server_port),
|
|
||||||
socktype=socket.SOCK_DGRAM,
|
|
||||||
)
|
|
||||||
elif protocol in ["tcp", "tls"]:
|
|
||||||
# TCP or TLS protocol with retry logic
|
|
||||||
for attempt in range(1, retry_attempts + 1):
|
|
||||||
try:
|
|
||||||
if protocol == "tcp":
|
|
||||||
# TCP without TLS
|
|
||||||
handler = logging.handlers.SysLogHandler(
|
|
||||||
address=(server_name, server_port),
|
|
||||||
socktype=socket.SOCK_STREAM,
|
|
||||||
)
|
|
||||||
# Set timeout on the socket
|
|
||||||
if hasattr(handler, "socket") and handler.socket:
|
|
||||||
handler.socket.settimeout(timeout)
|
|
||||||
return handler
|
|
||||||
else:
|
|
||||||
# TLS protocol
|
|
||||||
# Create SSL context with secure defaults
|
|
||||||
ssl_context = ssl.create_default_context()
|
|
||||||
|
|
||||||
# Explicitly set minimum TLS version to 1.2 for security
|
|
||||||
ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
||||||
|
|
||||||
# Configure server certificate verification
|
|
||||||
if cafile_path:
|
|
||||||
ssl_context.load_verify_locations(cafile=cafile_path)
|
|
||||||
|
|
||||||
# Configure client certificate authentication
|
|
||||||
if certfile_path and keyfile_path:
|
|
||||||
ssl_context.load_cert_chain(
|
|
||||||
certfile=certfile_path,
|
|
||||||
keyfile=keyfile_path,
|
|
||||||
)
|
|
||||||
elif certfile_path or keyfile_path:
|
|
||||||
# Warn if only one of the two required parameters is provided
|
|
||||||
self.logger.warning(
|
|
||||||
"Both certfile_path and keyfile_path are required for "
|
|
||||||
"client certificate authentication. Client authentication "
|
|
||||||
"will not be used."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Create TCP handler first
|
|
||||||
handler = logging.handlers.SysLogHandler(
|
|
||||||
address=(server_name, server_port),
|
|
||||||
socktype=socket.SOCK_STREAM,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Wrap socket with TLS
|
|
||||||
if hasattr(handler, "socket") and handler.socket:
|
|
||||||
handler.socket = ssl_context.wrap_socket(
|
|
||||||
handler.socket,
|
|
||||||
server_hostname=server_name,
|
|
||||||
)
|
|
||||||
handler.socket.settimeout(timeout)
|
|
||||||
|
|
||||||
return handler
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
if attempt < retry_attempts:
|
|
||||||
self.logger.warning(
|
|
||||||
f"Syslog connection attempt {attempt}/{retry_attempts} failed: {e}. "
|
|
||||||
f"Retrying in {retry_delay} seconds..."
|
|
||||||
)
|
|
||||||
time.sleep(retry_delay)
|
|
||||||
else:
|
|
||||||
self.logger.error(
|
|
||||||
f"Syslog connection failed after {retry_attempts} attempts: {e}"
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid protocol '{protocol}'. Must be 'udp', 'tcp', or 'tls'."
|
|
||||||
)
|
|
||||||
|
|
||||||
def save_aggregate_report_to_syslog(self, aggregate_reports: list[dict[str, Any]]):
|
def save_aggregate_report_to_syslog(self, aggregate_reports: list[dict[str, Any]]):
|
||||||
rows = parsed_aggregate_reports_to_csv_rows(aggregate_reports)
|
rows = parsed_aggregate_reports_to_csv_rows(aggregate_reports)
|
||||||
for row in rows:
|
for row in rows:
|
||||||
|
|||||||
Reference in New Issue
Block a user