Add Graph credentials cache and fix issue with batch size (#334)

* implement token cache for Graph API

* fix case when username isn't defined (device code)

* add batch size when listing messages from Graph API

* fix string concat
This commit is contained in:
Nathan Thorpe authored and GitHub committed 2022-07-21 13:43:21 -04:00
1 parent 315b99f95a
commit b15425f50e
9 files changed
+65 -24

No files matched your search

+5 -4
View File
@@ -228,6 +228,7 @@ The full set of configuration options are:
- ``client_secret`` - str: The app registration's secret
- ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword.
- ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox
- ``token_file`` - str: Path to save the token file (Default: .token)
.. note::
You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app.
@@ -290,10 +291,10 @@ The full set of configuration options are:
- ``server`` - str: The Syslog server name or IP address
- ``port`` - int: The UDP port to use (Default: 514)
- ``gmail_api``
- ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
- ``gmail_api_token_file`` - str: Path to save the token file (Default: .token)
- ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
- ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
- ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
- ``token_file`` - str: Path to save the token file (Default: .token)
- ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
- ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
.. warning::
+5 -4
View File
@@ -223,6 +223,7 @@ The full set of configuration options are:
- ``client_secret`` - str: The app registration's secret
- ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword.
- ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox
- ``token_file`` - str: Path to save the token file (Default: .token)
.. note::
You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app.
@@ -289,10 +290,10 @@ The full set of configuration options are:
- ``server`` - str: The Syslog server name or IP address
- ``port`` - int: The UDP port to use (Default: 514)
- ``gmail_api``
- ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
- ``gmail_api_token_file`` - str: Path to save the token file (Default: .token)
- ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
- ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
- ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None)
- ``token_file`` - str: Path to save the token file (Default: .token)
- ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False)
- ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify)
.. warning::
+1 -1
View File
@@ -1087,7 +1087,7 @@ def get_dmarc_reports_from_mailbox(connection: MailboxConnection,
connection.create_folder(forensic_reports_folder)
connection.create_folder(invalid_reports_folder)
messages = connection.fetch_messages(reports_folder)
messages = connection.fetch_messages(reports_folder, batch_size=batch_size)
total_messages = len(messages)
logger.debug("Found {0} messages in {1}".format(len(messages),
reports_folder))
+4 -1
View File
@@ -480,6 +480,8 @@ def _main():
if "msgraph" in config.sections():
graph_config = config["msgraph"]
opts.graph_token_file = graph_config.get("token_file", ".token")
if "auth_method" not in graph_config:
logger.info("auth_method setting missing from the "
"msgraph config section "
@@ -894,7 +896,8 @@ def _main():
client_id=opts.graph_client_id,
client_secret=opts.graph_client_secret,
username=opts.graph_user,
password=opts.graph_password
password=opts.graph_password,
token_file=opts.graph_token_file
)
except Exception as error:
+1 -1
View File
@@ -65,7 +65,7 @@ class GmailConnection(MailboxConnection):
else:
raise e
def fetch_messages(self, reports_folder: str) -> List[str]:
def fetch_messages(self, reports_folder: str, **kwargs) -> List[str]:
reports_label_id = self._find_label_id_for_label(reports_folder)
results = self.service.users().messages()\
.list(userId='me',
+44 -10
View File
@@ -1,11 +1,13 @@
import logging
from enum import Enum
from functools import lru_cache
from pathlib import Path
from time import sleep
from typing import List, Optional
from azure.identity import UsernamePasswordCredential, \
DeviceCodeCredential, ClientSecretCredential
DeviceCodeCredential, ClientSecretCredential, \
TokenCachePersistenceOptions, AuthenticationRecord
from msgraph.core import GraphClient
from parsedmarc.mail.mailbox_connection import MailboxConnection
@@ -20,13 +22,39 @@ class AuthMethod(Enum):
logger = logging.getLogger('parsedmarc')
def _generate_credential(auth_method: str, **kwargs):
def _get_cache_args(token_path: Path):
cache_args = {
'cache_persistence_options':
TokenCachePersistenceOptions(name='parsedmarc')
}
auth_record = _load_token(token_path)
if auth_record:
cache_args['authentication_record'] = \
AuthenticationRecord.deserialize(auth_record)
return cache_args
def _load_token(token_path: Path) -> Optional[str]:
if not token_path.exists():
return None
with token_path.open() as token_file:
return token_file.read()
def _cache_auth_record(record: AuthenticationRecord, token_path: Path):
token = record.serialize()
with token_path.open('w') as token_file:
token_file.write(token)
def _generate_credential(auth_method: str, token_path: Path, **kwargs):
if auth_method == AuthMethod.DeviceCode.name:
credential = DeviceCodeCredential(
client_id=kwargs['client_id'],
client_secret=kwargs['client_secret'],
disable_automatic_authentication=True,
tenant_id=kwargs['tenant_id']
tenant_id=kwargs['tenant_id'],
**_get_cache_args(token_path)
)
elif auth_method == AuthMethod.UsernamePassword.name:
credential = UsernamePasswordCredential(
@@ -34,7 +62,8 @@ def _generate_credential(auth_method: str, **kwargs):
client_credential=kwargs['client_secret'],
disable_automatic_authentication=True,
username=kwargs['username'],
password=kwargs['password']
password=kwargs['password'],
**_get_cache_args(token_path)
)
elif auth_method == AuthMethod.ClientSecret.name:
credential = ClientSecretCredential(
@@ -55,19 +84,23 @@ class MSGraphConnection(MailboxConnection):
client_secret: str,
username: str,
password: str,
tenant_id: str):
tenant_id: str,
token_file: str):
token_path = Path(token_file)
credential = _generate_credential(auth_method,
client_id=client_id,
client_secret=client_secret,
username=username,
password=password,
tenant_id=tenant_id)
tenant_id=tenant_id,
token_path=token_path)
scopes = ['Mail.ReadWrite']
# Detect if mailbox is shared
if username and mailbox and username != mailbox:
if mailbox and username != mailbox:
scopes = ['Mail.ReadWrite.Shared']
if not isinstance(credential, ClientSecretCredential):
credential.authenticate(scopes=scopes)
auth_record = credential.authenticate(scopes=scopes)
_cache_auth_record(auth_record, token_path)
self._client = GraphClient(credential=credential)
self.mailbox_name = mailbox
@@ -96,11 +129,12 @@ class MSGraphConnection(MailboxConnection):
logger.warning(f'Unknown response '
f'{resp.status_code} {resp.json()}')
def fetch_messages(self, folder_name: str) -> List[str]:
def fetch_messages(self, folder_name: str, **kwargs) -> List[str]:
""" Returns a list of message UIDs in the specified folder """
folder_id = self._find_folder_id_from_folder_path(folder_name)
batch_size = kwargs.get('batch_size', 10)
url = f'/users/{self.mailbox_name}/mailFolders/' \
f'{folder_id}/messages?$select=id'
f'{folder_id}/messages?$select=id&$top={batch_size}'
result = self._client.get(url)
emails = result.json()['value']
return [email['id'] for email in emails]
+1 -1
View File
@@ -32,7 +32,7 @@ class IMAPConnection(MailboxConnection):
def create_folder(self, folder_name: str):
self._client.create_folder(folder_name)
def fetch_messages(self, reports_folder: str):
def fetch_messages(self, reports_folder: str, **kwargs):
self._client.select_folder(reports_folder)
return self._client.search()
+3 -1
View File
@@ -9,7 +9,9 @@ class MailboxConnection(ABC):
def create_folder(self, folder_name: str):
raise NotImplementedError
def fetch_messages(self, reports_folder: str) -> List[str]:
def fetch_messages(self,
reports_folder: str,
**kwargs) -> List[str]:
raise NotImplementedError
def fetch_message(self, message_id) -> str:
+1 -1
View File
@@ -105,7 +105,7 @@ setup(
'lxml>=4.4.0',
'boto3>=1.16.63',
'msgraph-core>=0.2.2',
'azure-identity>=1.8.0'
'azure-identity>=1.8.0',
'google-api-core>=2.4.0',
'google-api-python-client>=2.35.0',
'google-auth>=2.3.3',