diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 0c1e7c048..0b23c413c 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -73,6 +73,8 @@ APPLE_MAIL_TAG_COLORS = { "grey": ["$MailFlagBit1", "$MailFlagBit2"], } +MAIL_FETCH_BATCH_SIZE = 500 + class MailError(Exception): pass @@ -709,17 +711,26 @@ class MailAccountHandler(LoggingMixin): ) return 0 - try: - messages = M.fetch( - criteria=AND(uid=list(new_uids)), - mark_seen=False, - charset=rule.account.character_set, - bulk=True, - ) - except Exception as err: - raise MailError( - f"Rule {rule}: Error while fetching folder {rule.folder}", - ) from err + sorted_new_uids = sorted(new_uids, key=int) + message_batches = [] + # ikvk/imap_tools#268 requests a direct UID-list fetch that would let us drop this manual batching loop + for batch_start in range(0, len(sorted_new_uids), MAIL_FETCH_BATCH_SIZE): + batch = sorted_new_uids[batch_start : batch_start + MAIL_FETCH_BATCH_SIZE] + try: + message_batches.append( + M.fetch( + criteria=AND(uid=batch), + mark_seen=False, + charset=rule.account.character_set, + bulk=True, + ), + ) + except Exception as err: + raise MailError( + f"Rule {rule}: Error while fetching folder {rule.folder}", + ) from err + + messages = itertools.chain(*message_batches) mails_processed = 0 total_processed_files = 0 diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 97f1aa824..415f0e459 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -435,6 +435,54 @@ class TestMail( super().setUp() + @mock.patch("paperless_mail.mail.MAIL_FETCH_BATCH_SIZE", 5) + def test_handle_mail_account_batches_body_fetch_for_large_backlog(self) -> None: + """ + GIVEN: + - More new/unprocessed mail than MAIL_FETCH_BATCH_SIZE + WHEN: + - The mail account is processed + THEN: + - The body fetch is issued in multiple batches + - Every message is still processed (none dropped at a batch boundary) + """ + account = MailAccount.objects.create( + name="test", + imap_server="", + username="admin", + password="secret", + ) + rule = MailRule.objects.create( + name="testrule", + account=account, + action=MailRule.MailAction.MARK_READ, + consumption_scope=MailRule.ConsumptionScope.ATTACHMENTS_ONLY, + ) + + message_count = 12 # more than the patched batch size of 5 + self.mailMocker.bogus_mailbox.messages = [ + self.mailMocker.messageBuilder.create_message( + subject=f"No attachment {i}", + attachments=[], + ) + for i in range(message_count) + ] + self.mailMocker.bogus_mailbox.updateClient() + + with mock.patch.object( + self.mailMocker.bogus_mailbox, + "fetch", + wraps=self.mailMocker.bogus_mailbox.fetch, + ) as fetch_spy: + self.mail_account_handler.handle_mail_account(account) + + # ceil(12 / 5) == 3 batches + self.assertEqual(fetch_spy.call_count, 3) + self.assertEqual( + ProcessedMail.objects.filter(rule=rule).count(), + message_count, + ) + def test_get_correspondent(self) -> None: message = namedtuple("MailMessage", []) message.from_ = "someone@somewhere.com"