mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-11-21 14:42:39 +00:00
Add Graph credentials cache and fix issue with batch size (#334)
* implement token cache for Graph API * fix case when username isn't defined (device code) * add batch size when listing messages from Graph API * fix string concat
This commit is contained in:
1 parent
315b99f95a
commit
b15425f50e
9 files changed
+65
-24
No files matched your search
+5
-4
@@ -228,6 +228,7 @@ The full set of configuration options are:
|
||||
- ``client_secret`` - str: The app registration's secret
|
||||
- ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword.
|
||||
- ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox
|
||||
- ``token_file`` - str: Path to save the token file (Default: .token)
|
||||
|
||||
.. note::
|
||||
You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app.
|
||||
@@ -290,10 +291,10 @@ The full set of configuration options are:
|
||||
- ``server`` - str: The Syslog server name or IP address
|
||||
- ``port`` - int: The UDP port to use (Default: 514)
|
||||
- ``gmail_api``
|
||||
- ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
|
||||
- ``gmail_api_token_file`` - str: Path to save the token file (Default: .token)
|
||||
- ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
|
||||
- ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
|
||||
- ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
|
||||
- ``token_file`` - str: Path to save the token file (Default: .token)
|
||||
- ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
|
||||
- ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
|
||||
|
||||
.. warning::
|
||||
|
||||
|
||||
+5
-4
@@ -223,6 +223,7 @@ The full set of configuration options are:
|
||||
- ``client_secret`` - str: The app registration's secret
|
||||
- ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword.
|
||||
- ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox
|
||||
- ``token_file`` - str: Path to save the token file (Default: .token)
|
||||
|
||||
.. note::
|
||||
You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app.
|
||||
@@ -289,10 +290,10 @@ The full set of configuration options are:
|
||||
- ``server`` - str: The Syslog server name or IP address
|
||||
- ``port`` - int: The UDP port to use (Default: 514)
|
||||
- ``gmail_api``
|
||||
- ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
|
||||
- ``gmail_api_token_file`` - str: Path to save the token file (Default: .token)
|
||||
- ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
|
||||
- ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
|
||||
- ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
|
||||
- ``token_file`` - str: Path to save the token file (Default: .token)
|
||||
- ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
|
||||
- ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
|
||||
|
||||
.. warning::
|
||||
|
||||
|
||||
@@ -1087,7 +1087,7 @@ def get_dmarc_reports_from_mailbox(connection: MailboxConnection,
|
||||
connection.create_folder(forensic_reports_folder)
|
||||
connection.create_folder(invalid_reports_folder)
|
||||
|
||||
messages = connection.fetch_messages(reports_folder)
|
||||
messages = connection.fetch_messages(reports_folder, batch_size=batch_size)
|
||||
total_messages = len(messages)
|
||||
logger.debug("Found {0} messages in {1}".format(len(messages),
|
||||
reports_folder))
|
||||
|
||||
+4
-1
@@ -480,6 +480,8 @@ def _main():
|
||||
|
||||
if "msgraph" in config.sections():
|
||||
graph_config = config["msgraph"]
|
||||
opts.graph_token_file = graph_config.get("token_file", ".token")
|
||||
|
||||
if "auth_method" not in graph_config:
|
||||
logger.info("auth_method setting missing from the "
|
||||
"msgraph config section "
|
||||
@@ -894,7 +896,8 @@ def _main():
|
||||
client_id=opts.graph_client_id,
|
||||
client_secret=opts.graph_client_secret,
|
||||
username=opts.graph_user,
|
||||
password=opts.graph_password
|
||||
password=opts.graph_password,
|
||||
token_file=opts.graph_token_file
|
||||
)
|
||||
|
||||
except Exception as error:
|
||||
|
||||
@@ -65,7 +65,7 @@ class GmailConnection(MailboxConnection):
|
||||
else:
|
||||
raise e
|
||||
|
||||
def fetch_messages(self, reports_folder: str) -> List[str]:
|
||||
def fetch_messages(self, reports_folder: str, **kwargs) -> List[str]:
|
||||
reports_label_id = self._find_label_id_for_label(reports_folder)
|
||||
results = self.service.users().messages()\
|
||||
.list(userId='me',
|
||||
|
||||
+44
-10
@@ -1,11 +1,13 @@
|
||||
import logging
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
from typing import List, Optional
|
||||
|
||||
from azure.identity import UsernamePasswordCredential, \
|
||||
DeviceCodeCredential, ClientSecretCredential
|
||||
DeviceCodeCredential, ClientSecretCredential, \
|
||||
TokenCachePersistenceOptions, AuthenticationRecord
|
||||
from msgraph.core import GraphClient
|
||||
|
||||
from parsedmarc.mail.mailbox_connection import MailboxConnection
|
||||
@@ -20,13 +22,39 @@ class AuthMethod(Enum):
|
||||
logger = logging.getLogger('parsedmarc')
|
||||
|
||||
|
||||
def _generate_credential(auth_method: str, **kwargs):
|
||||
def _get_cache_args(token_path: Path):
|
||||
cache_args = {
|
||||
'cache_persistence_options':
|
||||
TokenCachePersistenceOptions(name='parsedmarc')
|
||||
}
|
||||
auth_record = _load_token(token_path)
|
||||
if auth_record:
|
||||
cache_args['authentication_record'] = \
|
||||
AuthenticationRecord.deserialize(auth_record)
|
||||
return cache_args
|
||||
|
||||
|
||||
def _load_token(token_path: Path) -> Optional[str]:
|
||||
if not token_path.exists():
|
||||
return None
|
||||
with token_path.open() as token_file:
|
||||
return token_file.read()
|
||||
|
||||
|
||||
def _cache_auth_record(record: AuthenticationRecord, token_path: Path):
|
||||
token = record.serialize()
|
||||
with token_path.open('w') as token_file:
|
||||
token_file.write(token)
|
||||
|
||||
|
||||
def _generate_credential(auth_method: str, token_path: Path, **kwargs):
|
||||
if auth_method == AuthMethod.DeviceCode.name:
|
||||
credential = DeviceCodeCredential(
|
||||
client_id=kwargs['client_id'],
|
||||
client_secret=kwargs['client_secret'],
|
||||
disable_automatic_authentication=True,
|
||||
tenant_id=kwargs['tenant_id']
|
||||
tenant_id=kwargs['tenant_id'],
|
||||
**_get_cache_args(token_path)
|
||||
)
|
||||
elif auth_method == AuthMethod.UsernamePassword.name:
|
||||
credential = UsernamePasswordCredential(
|
||||
@@ -34,7 +62,8 @@ def _generate_credential(auth_method: str, **kwargs):
|
||||
client_credential=kwargs['client_secret'],
|
||||
disable_automatic_authentication=True,
|
||||
username=kwargs['username'],
|
||||
password=kwargs['password']
|
||||
password=kwargs['password'],
|
||||
**_get_cache_args(token_path)
|
||||
)
|
||||
elif auth_method == AuthMethod.ClientSecret.name:
|
||||
credential = ClientSecretCredential(
|
||||
@@ -55,19 +84,23 @@ class MSGraphConnection(MailboxConnection):
|
||||
client_secret: str,
|
||||
username: str,
|
||||
password: str,
|
||||
tenant_id: str):
|
||||
tenant_id: str,
|
||||
token_file: str):
|
||||
token_path = Path(token_file)
|
||||
credential = _generate_credential(auth_method,
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
username=username,
|
||||
password=password,
|
||||
tenant_id=tenant_id)
|
||||
tenant_id=tenant_id,
|
||||
token_path=token_path)
|
||||
scopes = ['Mail.ReadWrite']
|
||||
# Detect if mailbox is shared
|
||||
if username and mailbox and username != mailbox:
|
||||
if mailbox and username != mailbox:
|
||||
scopes = ['Mail.ReadWrite.Shared']
|
||||
if not isinstance(credential, ClientSecretCredential):
|
||||
credential.authenticate(scopes=scopes)
|
||||
auth_record = credential.authenticate(scopes=scopes)
|
||||
_cache_auth_record(auth_record, token_path)
|
||||
self._client = GraphClient(credential=credential)
|
||||
self.mailbox_name = mailbox
|
||||
|
||||
@@ -96,11 +129,12 @@ class MSGraphConnection(MailboxConnection):
|
||||
logger.warning(f'Unknown response '
|
||||
f'{resp.status_code} {resp.json()}')
|
||||
|
||||
def fetch_messages(self, folder_name: str) -> List[str]:
|
||||
def fetch_messages(self, folder_name: str, **kwargs) -> List[str]:
|
||||
""" Returns a list of message UIDs in the specified folder """
|
||||
folder_id = self._find_folder_id_from_folder_path(folder_name)
|
||||
batch_size = kwargs.get('batch_size', 10)
|
||||
url = f'/users/{self.mailbox_name}/mailFolders/' \
|
||||
f'{folder_id}/messages?$select=id'
|
||||
f'{folder_id}/messages?$select=id&$top={batch_size}'
|
||||
result = self._client.get(url)
|
||||
emails = result.json()['value']
|
||||
return [email['id'] for email in emails]
|
||||
|
||||
@@ -32,7 +32,7 @@ class IMAPConnection(MailboxConnection):
|
||||
def create_folder(self, folder_name: str):
|
||||
self._client.create_folder(folder_name)
|
||||
|
||||
def fetch_messages(self, reports_folder: str):
|
||||
def fetch_messages(self, reports_folder: str, **kwargs):
|
||||
self._client.select_folder(reports_folder)
|
||||
return self._client.search()
|
||||
|
||||
|
||||
@@ -9,7 +9,9 @@ class MailboxConnection(ABC):
|
||||
def create_folder(self, folder_name: str):
|
||||
raise NotImplementedError
|
||||
|
||||
def fetch_messages(self, reports_folder: str) -> List[str]:
|
||||
def fetch_messages(self,
|
||||
reports_folder: str,
|
||||
**kwargs) -> List[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def fetch_message(self, message_id) -> str:
|
||||
|
||||
Reference in new issue
Block a user