perf: batch body fetches when many new UIDs are pending

src/paperless_mail/mail.py

@@ -73,6 +73,8 @@ APPLE_MAIL_TAG_COLORS = {
     "grey": ["$MailFlagBit1", "$MailFlagBit2"],
 }
 
+MAIL_FETCH_BATCH_SIZE = 500
+
 
 class MailError(Exception):
     pass
@@ -709,17 +711,26 @@ class MailAccountHandler(LoggingMixin):
             )
             return 0
 
-        try:
-            messages = M.fetch(
-                criteria=AND(uid=list(new_uids)),
-                mark_seen=False,
-                charset=rule.account.character_set,
-                bulk=True,
-            )
-        except Exception as err:
-            raise MailError(
-                f"Rule {rule}: Error while fetching folder {rule.folder}",
-            ) from err
+        sorted_new_uids = sorted(new_uids, key=int)
+        message_batches = []
+        # ikvk/imap_tools#268 requests a direct UID-list fetch that would let us drop this manual batching loop
+        for batch_start in range(0, len(sorted_new_uids), MAIL_FETCH_BATCH_SIZE):
+            batch = sorted_new_uids[batch_start : batch_start + MAIL_FETCH_BATCH_SIZE]
+            try:
+                message_batches.append(
+                    M.fetch(
+                        criteria=AND(uid=batch),
+                        mark_seen=False,
+                        charset=rule.account.character_set,
+                        bulk=True,
+                    ),
+                )
+            except Exception as err:
+                raise MailError(
+                    f"Rule {rule}: Error while fetching folder {rule.folder}",
+                ) from err
+
+        messages = itertools.chain(*message_batches)
 
         mails_processed = 0
         total_processed_files = 0

src/paperless_mail/tests/test_mail.py

@@ -435,6 +435,54 @@ class TestMail(
 
         super().setUp()
 
+    @mock.patch("paperless_mail.mail.MAIL_FETCH_BATCH_SIZE", 5)
+    def test_handle_mail_account_batches_body_fetch_for_large_backlog(self) -> None:
+        """
+        GIVEN:
+            - More new/unprocessed mail than MAIL_FETCH_BATCH_SIZE
+        WHEN:
+            - The mail account is processed
+        THEN:
+            - The body fetch is issued in multiple batches
+            - Every message is still processed (none dropped at a batch boundary)
+        """
+        account = MailAccount.objects.create(
+            name="test",
+            imap_server="",
+            username="admin",
+            password="secret",
+        )
+        rule = MailRule.objects.create(
+            name="testrule",
+            account=account,
+            action=MailRule.MailAction.MARK_READ,
+            consumption_scope=MailRule.ConsumptionScope.ATTACHMENTS_ONLY,
+        )
+
+        message_count = 12  # more than the patched batch size of 5
+        self.mailMocker.bogus_mailbox.messages = [
+            self.mailMocker.messageBuilder.create_message(
+                subject=f"No attachment {i}",
+                attachments=[],
+            )
+            for i in range(message_count)
+        ]
+        self.mailMocker.bogus_mailbox.updateClient()
+
+        with mock.patch.object(
+            self.mailMocker.bogus_mailbox,
+            "fetch",
+            wraps=self.mailMocker.bogus_mailbox.fetch,
+        ) as fetch_spy:
+            self.mail_account_handler.handle_mail_account(account)
+
+        # ceil(12 / 5) == 3 batches
+        self.assertEqual(fetch_spy.call_count, 3)
+        self.assertEqual(
+            ProcessedMail.objects.filter(rule=rule).count(),
+            message_count,
+        )
+
     def test_get_correspondent(self) -> None:
         message = namedtuple("MailMessage", [])
         message.from_ = "someone@somewhere.com"