From b15425f50e50637baa2b1181f6ad9c4399538e0d Mon Sep 17 00:00:00 2001 From: Nathan Thorpe Date: Thu, 21 Jul 2022 10:43:21 -0700 Subject: [PATCH] Add Graph credentials cache and fix issue with batch size (#334) * implement token cache for Graph API * fix case when username isn't defined (device code) * add batch size when listing messages from Graph API * fix string concat --- README.rst | 9 +++-- docs/index.rst | 9 +++-- parsedmarc/__init__.py | 2 +- parsedmarc/cli.py | 5 ++- parsedmarc/mail/gmail.py | 2 +- parsedmarc/mail/graph.py | 54 ++++++++++++++++++++++----- parsedmarc/mail/imap.py | 2 +- parsedmarc/mail/mailbox_connection.py | 4 +- setup.py | 2 +- 9 files changed, 65 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index c884ead1..e437e8b9 100644 --- a/README.rst +++ b/README.rst @@ -228,6 +228,7 @@ The full set of configuration options are: - ``client_secret`` - str: The app registration's secret - ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword. - ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox + - ``token_file`` - str: Path to save the token file (Default: .token) .. note:: You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app. @@ -290,10 +291,10 @@ The full set of configuration options are: - ``server`` - str: The Syslog server name or IP address - ``port`` - int: The UDP port to use (Default: 514) - ``gmail_api`` - - ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None) - - ``gmail_api_token_file`` - str: Path to save the token file (Default: .token) - - ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False) - - ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify) + - ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None) + - ``token_file`` - str: Path to save the token file (Default: .token) + - ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False) + - ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify) .. warning:: diff --git a/docs/index.rst b/docs/index.rst index f7b49e38..49d1a525 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -223,6 +223,7 @@ The full set of configuration options are: - ``client_secret`` - str: The app registration's secret - ``tenant_id`` - str: The Azure AD tenant ID. This is required for all auth methods except UsernamePassword. - ``mailbox`` - str: The mailbox name. This defaults to the current user if using the UsernamePassword auth method, but could be a shared mailbox if the user has access to the mailbox + - ``token_file`` - str: Path to save the token file (Default: .token) .. note:: You must create an app registration in Azure AD and have an admin grant the Microsoft Graph ``Mail.ReadWrite`` (delegated) permission to the app. @@ -289,10 +290,10 @@ The full set of configuration options are: - ``server`` - str: The Syslog server name or IP address - ``port`` - int: The UDP port to use (Default: 514) - ``gmail_api`` - - ``gmail_api_credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None) - - ``gmail_api_token_file`` - str: Path to save the token file (Default: .token) - - ``gmail_api_include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False) - - ``gmail_api_scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify) + - ``credentials_file`` - str: Path to file containing the credentials, None to disable (Default: None) + - ``token_file`` - str: Path to save the token file (Default: .token) + - ``include_spam_trash`` - bool: Include messages in Spam and Trash when searching reports (Default: False) + - ``scopes`` - str: Comma separated list of scopes to use when acquiring credentials (Default: https://www.googleapis.com/auth/gmail.modify) .. warning:: diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 76a3418a..5e634127 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1087,7 +1087,7 @@ def get_dmarc_reports_from_mailbox(connection: MailboxConnection, connection.create_folder(forensic_reports_folder) connection.create_folder(invalid_reports_folder) - messages = connection.fetch_messages(reports_folder) + messages = connection.fetch_messages(reports_folder, batch_size=batch_size) total_messages = len(messages) logger.debug("Found {0} messages in {1}".format(len(messages), reports_folder)) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 957ca018..8bd31c0a 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -480,6 +480,8 @@ def _main(): if "msgraph" in config.sections(): graph_config = config["msgraph"] + opts.graph_token_file = graph_config.get("token_file", ".token") + if "auth_method" not in graph_config: logger.info("auth_method setting missing from the " "msgraph config section " @@ -894,7 +896,8 @@ def _main(): client_id=opts.graph_client_id, client_secret=opts.graph_client_secret, username=opts.graph_user, - password=opts.graph_password + password=opts.graph_password, + token_file=opts.graph_token_file ) except Exception as error: diff --git a/parsedmarc/mail/gmail.py b/parsedmarc/mail/gmail.py index 90993531..1b85430e 100644 --- a/parsedmarc/mail/gmail.py +++ b/parsedmarc/mail/gmail.py @@ -65,7 +65,7 @@ class GmailConnection(MailboxConnection): else: raise e - def fetch_messages(self, reports_folder: str) -> List[str]: + def fetch_messages(self, reports_folder: str, **kwargs) -> List[str]: reports_label_id = self._find_label_id_for_label(reports_folder) results = self.service.users().messages()\ .list(userId='me', diff --git a/parsedmarc/mail/graph.py b/parsedmarc/mail/graph.py index 2b2dfa1b..09bbbfa8 100644 --- a/parsedmarc/mail/graph.py +++ b/parsedmarc/mail/graph.py @@ -1,11 +1,13 @@ import logging from enum import Enum from functools import lru_cache +from pathlib import Path from time import sleep from typing import List, Optional from azure.identity import UsernamePasswordCredential, \ - DeviceCodeCredential, ClientSecretCredential + DeviceCodeCredential, ClientSecretCredential, \ + TokenCachePersistenceOptions, AuthenticationRecord from msgraph.core import GraphClient from parsedmarc.mail.mailbox_connection import MailboxConnection @@ -20,13 +22,39 @@ class AuthMethod(Enum): logger = logging.getLogger('parsedmarc') -def _generate_credential(auth_method: str, **kwargs): +def _get_cache_args(token_path: Path): + cache_args = { + 'cache_persistence_options': + TokenCachePersistenceOptions(name='parsedmarc') + } + auth_record = _load_token(token_path) + if auth_record: + cache_args['authentication_record'] = \ + AuthenticationRecord.deserialize(auth_record) + return cache_args + + +def _load_token(token_path: Path) -> Optional[str]: + if not token_path.exists(): + return None + with token_path.open() as token_file: + return token_file.read() + + +def _cache_auth_record(record: AuthenticationRecord, token_path: Path): + token = record.serialize() + with token_path.open('w') as token_file: + token_file.write(token) + + +def _generate_credential(auth_method: str, token_path: Path, **kwargs): if auth_method == AuthMethod.DeviceCode.name: credential = DeviceCodeCredential( client_id=kwargs['client_id'], client_secret=kwargs['client_secret'], disable_automatic_authentication=True, - tenant_id=kwargs['tenant_id'] + tenant_id=kwargs['tenant_id'], + **_get_cache_args(token_path) ) elif auth_method == AuthMethod.UsernamePassword.name: credential = UsernamePasswordCredential( @@ -34,7 +62,8 @@ def _generate_credential(auth_method: str, **kwargs): client_credential=kwargs['client_secret'], disable_automatic_authentication=True, username=kwargs['username'], - password=kwargs['password'] + password=kwargs['password'], + **_get_cache_args(token_path) ) elif auth_method == AuthMethod.ClientSecret.name: credential = ClientSecretCredential( @@ -55,19 +84,23 @@ class MSGraphConnection(MailboxConnection): client_secret: str, username: str, password: str, - tenant_id: str): + tenant_id: str, + token_file: str): + token_path = Path(token_file) credential = _generate_credential(auth_method, client_id=client_id, client_secret=client_secret, username=username, password=password, - tenant_id=tenant_id) + tenant_id=tenant_id, + token_path=token_path) scopes = ['Mail.ReadWrite'] # Detect if mailbox is shared - if username and mailbox and username != mailbox: + if mailbox and username != mailbox: scopes = ['Mail.ReadWrite.Shared'] if not isinstance(credential, ClientSecretCredential): - credential.authenticate(scopes=scopes) + auth_record = credential.authenticate(scopes=scopes) + _cache_auth_record(auth_record, token_path) self._client = GraphClient(credential=credential) self.mailbox_name = mailbox @@ -96,11 +129,12 @@ class MSGraphConnection(MailboxConnection): logger.warning(f'Unknown response ' f'{resp.status_code} {resp.json()}') - def fetch_messages(self, folder_name: str) -> List[str]: + def fetch_messages(self, folder_name: str, **kwargs) -> List[str]: """ Returns a list of message UIDs in the specified folder """ folder_id = self._find_folder_id_from_folder_path(folder_name) + batch_size = kwargs.get('batch_size', 10) url = f'/users/{self.mailbox_name}/mailFolders/' \ - f'{folder_id}/messages?$select=id' + f'{folder_id}/messages?$select=id&$top={batch_size}' result = self._client.get(url) emails = result.json()['value'] return [email['id'] for email in emails] diff --git a/parsedmarc/mail/imap.py b/parsedmarc/mail/imap.py index d91707a8..66ea2e67 100644 --- a/parsedmarc/mail/imap.py +++ b/parsedmarc/mail/imap.py @@ -32,7 +32,7 @@ class IMAPConnection(MailboxConnection): def create_folder(self, folder_name: str): self._client.create_folder(folder_name) - def fetch_messages(self, reports_folder: str): + def fetch_messages(self, reports_folder: str, **kwargs): self._client.select_folder(reports_folder) return self._client.search() diff --git a/parsedmarc/mail/mailbox_connection.py b/parsedmarc/mail/mailbox_connection.py index ecaa7f4e..ecaa2cbb 100644 --- a/parsedmarc/mail/mailbox_connection.py +++ b/parsedmarc/mail/mailbox_connection.py @@ -9,7 +9,9 @@ class MailboxConnection(ABC): def create_folder(self, folder_name: str): raise NotImplementedError - def fetch_messages(self, reports_folder: str) -> List[str]: + def fetch_messages(self, + reports_folder: str, + **kwargs) -> List[str]: raise NotImplementedError def fetch_message(self, message_id) -> str: diff --git a/setup.py b/setup.py index 81be1d4d..9a71f6f4 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ setup( 'lxml>=4.4.0', 'boto3>=1.16.63', 'msgraph-core>=0.2.2', - 'azure-identity>=1.8.0' + 'azure-identity>=1.8.0', 'google-api-core>=2.4.0', 'google-api-python-client>=2.35.0', 'google-auth>=2.3.3',