Fix issue with MS Graph batch size, scopes, and exception logging improvements (#352)

* log the stack trace when handling an exception

* fix issue when batch size is not supplied, raise error on fetch_messages

* fix unused var

* initialize graph client with scopes,

* add check for status code

parsedmarc/cli.py

@@ -795,8 +795,8 @@ def _main():
                                   timeout=opts.elasticsearch_timeout)
                 elastic.migrate_indexes(aggregate_indexes=[es_aggregate_index],
                                         forensic_indexes=[es_forensic_index])
-        except elastic.ElasticsearchError as error:
-            logger.error("Elasticsearch Error: {0}".format(error.__str__()))
+        except elastic.ElasticsearchError:
+            logger.exception("Elasticsearch Error")
             exit(1)
 
     if opts.hec:
@@ -902,8 +902,8 @@ def _main():
                 password=opts.imap_password,
             )
 
-        except Exception as error:
-            logger.error("IMAP Error: {0}".format(error.__str__()))
+        except Exception:
+            logger.exception("IMAP Error")
             exit(1)
 
     if opts.graph_client_id:
@@ -920,8 +920,8 @@ def _main():
                 token_file=opts.graph_token_file
             )
 
-        except Exception as error:
-            logger.error("MS Graph Error: {0}".format(error.__str__()))
+        except Exception:
+            logger.exception("MS Graph Error")
             exit(1)
 
     if opts.gmail_api_credentials_file:
@@ -943,8 +943,8 @@ def _main():
                 oauth2_port=opts.gmail_api_oauth2_port
             )
 
-        except Exception as error:
-            logger.error("Gmail API Error: {0}".format(error.__str__()))
+        except Exception:
+            logger.exception("Gmail API Error")
             exit(1)
 
     if mailbox_connection:
@@ -965,8 +965,8 @@ def _main():
             aggregate_reports += reports["aggregate_reports"]
             forensic_reports += reports["forensic_reports"]
 
-        except Exception as error:
-            logger.error("Mailbox Error: {0}".format(error.__str__()))
+        except Exception:
+            logger.exception("Mailbox Error")
             exit(1)
 
     results = OrderedDict([("aggregate_reports", aggregate_reports),
@@ -984,8 +984,8 @@ def _main():
                           username=opts.smtp_user,
                           password=opts.smtp_password,
                           subject=opts.smtp_subject)
-        except Exception as error:
-            logger.error("{0}".format(error.__str__()))
+        except Exception:
+            logger.exception("Failed to email results")
             exit(1)
 
     if mailbox_connection and opts.mailbox_watch:

parsedmarc/mail/graph.py

@@ -91,14 +91,19 @@ class MSGraphConnection(MailboxConnection):
                                           password=password,
                                           tenant_id=tenant_id,
                                           token_path=token_path)
-        scopes = ['Mail.ReadWrite']
-        # Detect if mailbox is shared
-        if mailbox and username != mailbox:
-            scopes = ['Mail.ReadWrite.Shared']
+        client_params = {
+            'credential': credential
+        }
         if not isinstance(credential, ClientSecretCredential):
+            scopes = ['Mail.ReadWrite']
+            # Detect if mailbox is shared
+            if mailbox and username != mailbox:
+                scopes = ['Mail.ReadWrite.Shared']
             auth_record = credential.authenticate(scopes=scopes)
             _cache_auth_record(auth_record, token_path)
-        self._client = GraphClient(credential=credential)
+            client_params['scopes'] = scopes
+
+        self._client = GraphClient(**client_params)
         self.mailbox_name = mailbox
 
     def create_folder(self, folder_name: str):
@@ -129,10 +134,17 @@ class MSGraphConnection(MailboxConnection):
     def fetch_messages(self, folder_name: str, **kwargs) -> List[str]:
         """ Returns a list of message UIDs in the specified folder """
         folder_id = self._find_folder_id_from_folder_path(folder_name)
-        batch_size = kwargs.get('batch_size', 10)
         url = f'/users/{self.mailbox_name}/mailFolders/' \
-              f'{folder_id}/messages?$select=id&$top={batch_size}'
-        result = self._client.get(url)
+              f'{folder_id}/messages'
+        params = {
+            '$select': 'id'
+        }
+        batch_size = kwargs.get('batch_size')
+        if batch_size and batch_size > 0:
+            params['$top'] = batch_size
+        result = self._client.get(url, params=params)
+        if result.status_code != 200:
+            raise RuntimeError(f'Failed to fetch messages {result.text}')
         emails = result.json()['value']
         return [email['id'] for email in emails]
 
@@ -147,6 +159,9 @@ class MSGraphConnection(MailboxConnection):
     def fetch_message(self, message_id: str):
         url = f'/users/{self.mailbox_name}/messages/{message_id}/$value'
         result = self._client.get(url)
+        if result.status_code != 200:
+            raise RuntimeWarning(f"Failed to fetch message"
+                                 f"{result.status_code}: {result.json()}")
         self.mark_message_read(message_id)
         return result.text
 
@@ -200,6 +215,9 @@ class MSGraphConnection(MailboxConnection):
             sub_url = f'/{parent_folder_id}/childFolders'
         url = f'/users/{self.mailbox_name}/mailFolders{sub_url}'
         folders_resp = self._client.get(url)
+        if folders_resp.status_code != 200:
+            raise RuntimeWarning(f"Failed to list folders."
+                                 f"{folders_resp.json()}")
         folders = folders_resp.json()['value']
         matched_folders = [folder for folder in folders
                            if folder['displayName'] == folder_name]