[Web][Dovecot] imapsync: update to 2.314, refresh OAuth tokens of running jobs, add source presets

data/web/inc/lib/mailcow/Imapsync/Controller/ImapsyncApiController.php

@@ -7,6 +7,7 @@ namespace Mailcow\Imapsync\Controller;
 use Mailcow\Core\AccessDenied;
 use Mailcow\Core\ApiResponse;
 use Mailcow\Core\UserError;
+use Mailcow\Imapsync\Model\Source;
 use Mailcow\Imapsync\Service\JobService;
 use Mailcow\Imapsync\Service\SourceService;
 use PDOException;
@@ -37,8 +38,11 @@ class ImapsyncApiController
     /** POST add/syncjob_source */
     public function addSource(array $attr): bool
     {
-        return $this->handle('add', 'source', $attr,
-            fn(array $log): bool => $this->response->success(['imapsync_source_added', $this->sources->create($attr)->name], $log));
+        return $this->handle('add', 'source', $attr, function (array $log) use ($attr): bool {
+            $source = $this->sources->create($attr);
+            $this->tokenFailure($source, $log);
+            return $this->response->success(['imapsync_source_added', $source->name], $log);
+        });
     }
 
     /** POST edit/syncjob: one message per id; `id` in $attr wins over $items (array_merge like before) */
@@ -70,8 +74,11 @@ class ImapsyncApiController
         $data = array_merge(['id' => $items], $attr);
         return $this->handle('edit', 'source', $data, function (array $log) use ($data): bool {
             $this->sources->requireEditor();
-            return $this->batch(array_filter((array)($data['id'] ?? []), 'is_numeric'), $log,
-                fn(int $id): array => ['imapsync_source_modified', $this->sources->update($id, $data)->name]);
+            return $this->batch(array_filter((array)($data['id'] ?? []), 'is_numeric'), $log, function (int $id) use ($data, $log): array {
+                $source = $this->sources->update($id, $data);
+                $this->tokenFailure($source, $log);
+                return ['imapsync_source_modified', $source->name];
+            });
         });
     }
 
@@ -201,6 +208,14 @@ class ImapsyncApiController
         $this->response->flush();
     }
 
+    /** The token fetched when saving a client_credentials source failed: the source is saved, report the error */
+    private function tokenFailure(Source $source, array $log): void
+    {
+        if ($source->usesClientCredentials() && $source->active === 1 && $source->lastRefreshError !== null) {
+            $this->response->danger(['imapsync_source_token_refresh_failed', $source->name . ' (' . $source->lastRefreshError . ')'], $log);
+        }
+    }
+
     /**
      * Runs one write call: $work gets the log entry and returns the result for process_*_return().
      * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.

data/web/inc/lib/mailcow/Imapsync/Model/Source.php

@@ -37,7 +37,7 @@ class Source
     public ?string $clientId = null;
     public ?string $clientSecret = null;
     public ?string $oauthScope = null;
-    /** Extra token request parameters, JSON object text */
+    /** Extra OAuth parameters (sign-in or token request, see OAuthService), JSON object text */
     public ?string $extraParams = null;
     /** Shared token of a client_credentials source */
     public ?string $accessToken = null;

data/web/inc/lib/mailcow/Imapsync/Service/OAuthService.php

@@ -22,10 +22,12 @@ use PDOException;
  */
 class OAuthService
 {
-    /** Tokens expiring within this many seconds are refreshed */
-    private const REFRESH_AHEAD = 300;
+    /** Tokens expiring within this many seconds are refreshed (the runner starts jobs with >= 15 min left) */
+    private const REFRESH_AHEAD = 1200;
     /** A token whose last refresh failed is retried after this many seconds */
-    private const RETRY_AFTER = 900;
+    private const RETRY_AFTER = 180;
+    /** Extra params that belong to the browser sign-in; all others go to the token request */
+    private const AUTHORIZE_PARAMS = ['prompt', 'access_type', 'login_hint', 'domain_hint', 'hd'];
     /** Per-user tokens that no job uses are kept this long (time to save the job after signing in) */
     private const UNUSED_TOKEN_GRACE = 86400;
 
@@ -106,9 +108,11 @@ class OAuthService
         if (!$this->endpointAllowed($endpoint, $source->createdBy)) {
             return null;
         }
-        return $endpoint . (str_contains($endpoint, '?') ? '&' : '?') . http_build_query(['client_id' => $source->clientId,
+        // select_account: never sign in silently with the account the browser already uses (often the admin's)
+        $params = array_merge(['prompt' => 'select_account'], $this->extraParamsFor($source, true), ['client_id' => $source->clientId,
             'response_type' => 'code', 'redirect_uri' => $this->redirectUri(), 'response_mode' => 'query',
             'scope' => $source->oauthScope, 'state' => $state]);
+        return $endpoint . (str_contains($endpoint, '?') ? '&' : '?') . http_build_query($params);
     }
 
     /**
@@ -248,9 +252,17 @@ class OAuthService
         return $value ?? throw new UserError('imapsync_source_oauth_invalid');
     }
 
+    /** Stored extra params of the browser sign-in ($authorize) or of the token request */
+    private function extraParamsFor(Source $source, bool $authorize): array
+    {
+        $extra = json_decode((string)$source->extraParams, true);
+        return array_filter(is_array($extra) ? $extra : [],
+            fn($key): bool => in_array($key, self::AUTHORIZE_PARAMS, true) === $authorize, ARRAY_FILTER_USE_KEY);
+    }
+
     /**
-     * Extra token request parameters: a JSON object with scalar values, as string or as array
-     * (after 011-api.js auto-parsing). Returns the JSON to store, null for none.
+     * Extra parameters: a JSON object with scalar values, as string or as array (after 011-api.js
+     * auto-parsing). Returns the JSON to store, null for none.
      */
     private function extraParams(mixed $extra): ?string
     {
@@ -277,8 +289,7 @@ class OAuthService
      */
     private function requestToken(Source $source, array $grant): array
     {
-        $extra = json_decode((string)$source->extraParams, true);
-        $post = array_merge(is_array($extra) ? $extra : [],
+        $post = array_merge($this->extraParamsFor($source, false),
             ['client_id' => $source->clientId, 'client_secret' => $source->clientSecret, 'scope' => $source->oauthScope], $grant);
         $token = $this->client->request((string)$source->tokenEndpoint, $this->isStrict($source->createdBy), $post);
         if (empty($token['access_token']) || !is_string($token['access_token'])) {

data/web/inc/lib/mailcow/Imapsync/Service/SourceService.php

@@ -97,6 +97,7 @@ class SourceService
             throw new UserError('object_exists', [htmlspecialchars($source->name)]);
         }
         $this->repo->insert($source);
+        $this->fetchToken($source);
         return $source;
     }
 
@@ -121,6 +122,7 @@ class SourceService
             $this->applyScope($source, $input['scope'] ?? $current->scope, $input['domains'] ?? [], $input['users'] ?? []);
         }
         $this->repo->update($source, $writeTargets, self::connection($current) !== self::connection($source));
+        $this->fetchToken($source);
         return $source;
     }
 
@@ -338,6 +340,14 @@ class SourceService
         return array_values(array_unique($clean));
     }
 
+    /** A saved, active client_credentials source gets its token now instead of at the next cron run (error in lastRefreshError) */
+    private function fetchToken(Source $source): void
+    {
+        if ($source->usesClientCredentials() && $source->active === 1) {
+            $source->lastRefreshError = $this->oauth->refreshSourceToken($source);
+        }
+    }
+
     /** Tokens never follow a source to another OAuth client or IMAP server, so these values reset them */
     private static function connection(Source $source): array
     {

data/web/inc/vars.inc.php

@@ -294,7 +294,7 @@ $RSPAMD_MAPS = array(
 // Anything not listed here is rejected when a job is saved (the runner passes stored options as they are).
 // Override in vars.local.inc.php per option, e.g. $IMAPSYNC_OPTIONS['someoption'] = true; or unset($IMAPSYNC_OPTIONS['debug']);
 // The old format $IMAPSYNC_OPTIONS['whitelist'] (list of names) is no longer supported and ignored (logged).
-// NEVER add (RCE): 'delete2foldersonly', 'delete2foldersbutnot', 'regexflag', 'regexmess', 'pipemess', 'regextrans2', 'maxlinelengthcmd'
+// NEVER add (RCE): 'delete2foldersonly', 'delete2foldersbutnot', 'regexflag', 'regexmess', 'pipemess', 'regextrans2', 'maxlinelengthcmd', 'oauthrefreshcmd1', 'oauthrefreshcmd2'
 // NEVER add options taking local file paths (e.g. 'logfile', 'logdir', 'pidfile', 'passfile1', 'sslargs1' with SSL_ca_file)
 $IMAPSYNC_OPTIONS_DEFAULT = array(
   'abort'                  => false,

data/web/js/site/imapsync.js

@@ -588,6 +588,74 @@ var mailcowImapsync = (function($) {
     }
   }
 
+  // ===== Provider presets (add source modal) =====
+
+  // Server and OAuth settings of well-known providers; fields a preset leaves out get the defaults of applyPreset
+  var m365 = 'https://login.microsoftonline.com/{tenant}/oauth2/v2.0/';
+  var presets = {
+    m365_app: {host: 'outlook.office365.com', auth_type: 'XOAUTH2', oauth_token_endpoint: m365 + 'token',
+      oauth_scope: 'https://outlook.office365.com/.default'},
+    m365_user: {host: 'outlook.office365.com', auth_type: 'XOAUTH2', oauth_flow: 'authorization_code',
+      oauth_authorize_endpoint: m365 + 'authorize', oauth_token_endpoint: m365 + 'token',
+      oauth_scope: 'https://outlook.office.com/IMAP.AccessAsUser.All offline_access openid email profile'},
+    // Google issues a refresh token only for offline access, and again only after consent
+    google_user: {host: 'imap.gmail.com', auth_type: 'XOAUTH2', oauth_flow: 'authorization_code',
+      oauth_authorize_endpoint: 'https://accounts.google.com/o/oauth2/v2/auth', oauth_token_endpoint: 'https://oauth2.googleapis.com/token',
+      oauth_scope: 'https://mail.google.com/ openid email', oauth_extra_params: '{"access_type":"offline","prompt":"consent select_account"}'},
+    gmail: {host: 'imap.gmail.com'},
+    gmx: {host: 'imap.gmx.net'},
+    webde: {host: 'imap.web.de'},
+    tonline: {host: 'secureimap.t-online.de'},
+    icloud: {host: 'imap.mail.me.com'},
+    yahoo: {host: 'imap.mail.yahoo.com'},
+    ionos: {host: 'imap.ionos.de'},
+    strato: {host: 'imap.strato.de'}
+  };
+
+  // The name follows the preset until it is changed by hand; client ID and secret are kept
+  function applyPreset($link) {
+    var $form = $link.closest('form');
+    var preset = $.extend({port: 993, enc: 'SSL', auth_type: 'PLAIN', oauth_flow: 'client_credentials', oauth_authorize_endpoint: '',
+      oauth_token_endpoint: '', oauth_userinfo_endpoint: '', oauth_scope: '', oauth_extra_params: ''}, presets[$link.data('preset')]);
+    var $name = $form.find('input[name="name"]');
+    var names = $form.find('.imapsync-source-preset').map(function() { return $.trim($(this).text()); }).get();
+    if (!$name.val() || names.indexOf($name.val()) !== -1) $name.val($.trim($link.text()));
+    // bootstrap-select 1.14 duplicates the button label on refresh after .val(), use its own setter
+    $.each(preset, function(field, value) {
+      var $f = $form.find('[name="' + field + '"]');
+      if ($f.data('selectpicker')) {
+        $f.selectpicker('val', value);
+      } else {
+        $f.val(value);
+      }
+    });
+    var needsTenant = preset.oauth_token_endpoint.indexOf('{tenant}') !== -1;
+    $form.find('.imapsync-source-tenant-row').toggle(needsTenant);
+    applyTenant($form);
+    sourceFormToggle($form);
+    if (needsTenant) $form.find('.imapsync-source-tenant').trigger('focus');
+  }
+
+  // M365: the tenant (ID or domain) is part of the endpoint URLs
+  function applyTenant($form) {
+    var tenant = encodeURIComponent($.trim($form.find('.imapsync-source-tenant').val() || '')) || '{tenant}';
+    $form.find('input[name="oauth_authorize_endpoint"], input[name="oauth_token_endpoint"]').val(function(_, url) {
+      return url.replace(/^(https:\/\/login\.microsoftonline\.com\/)[^\/]*/, function(_, base) { return base + tenant; });
+    });
+  }
+
+  // A visible endpoint must not keep the {tenant} placeholder; checked before the form is posted
+  function tenantValid($form) {
+    var missing = $form.find('input[name="oauth_authorize_endpoint"]:visible, input[name="oauth_token_endpoint"]:visible')
+      .filter(function() { return this.value.indexOf('{tenant}') !== -1; }).length > 0;
+    $form.find('.imapsync-source-tenant').toggleClass('inputMissingAttr', missing);
+    if (missing) {
+      $form.find('.imapsync-source-tenant-row').show();
+      mailcow_alert_box(lang_danger.imapsync_source_tenant_missing, 'danger');
+    }
+    return !missing;
+  }
+
   // ===== Wiring =====
 
   function bind() {
@@ -659,12 +727,20 @@ var mailcowImapsync = (function($) {
       sourceScopeToggle($(this).closest('form'));
     });
     $('[data-action="add_item"][data-id="add_imapsync_source"], [data-action="edit_selected"][data-id="edit_imapsync_source"]').on('click', function(e) {
-      if (!extraParamsValid($(this).closest('form'))) {
+      var $form = $(this).closest('form');
+      if (!extraParamsValid($form) || !tenantValid($form)) {
         e.preventDefault();
         e.stopImmediatePropagation();
         return false;
       }
     });
+    $(document).on('click', '.imapsync-source-preset', function(e) {
+      e.preventDefault();
+      applyPreset($(this));
+    });
+    $(document).on('input', '.imapsync-source-tenant', function() {
+      applyTenant($(this).closest('form'));
+    });
     $(document).on('click', '.imapsync-copy-btn', function(e) {
       e.preventDefault();
       copyField($(this).data('copy-target'));

data/web/lang/lang.de-de.json

@@ -572,6 +572,7 @@
         "imapsync_source_oauth_required": "OAuth2 erfordert Token-Endpunkt, Client-ID, Client-Secret und Scope",
         "imapsync_source_oauth_endpoint_invalid": "Ungültiger OAuth2-Endpunkt: Erforderlich ist eine https://-URL mit gültigem Hostnamen (Quellen, die nicht von einem Administrator angelegt wurden, brauchen einen öffentlich erreichbaren Host)",
         "imapsync_source_oauth_extra_invalid": "OAuth2-Extra-Parameter müssen ein JSON-Objekt mit einfachen Werten sein, z. B. {\"resource\": \"https://example.com\"}",
+        "imapsync_source_tenant_missing": "Bitte den Microsoft-365-Tenant angeben",
         "imapsync_source_oauth_invalid": "Ungültige OAuth2-Konfiguration",
         "imapsync_max_parallel_invalid": "Die Anzahl paralleler Prozesse muss zwischen 1 und 32 liegen",
         "imapsync_source_description_invalid": "Die Beschreibung darf höchstens 255 Zeichen lang sein",
@@ -1356,7 +1357,11 @@
         "source_oauth_client_secret": "Client-Secret",
         "source_oauth_scope": "Scope",
         "source_oauth_extra_params": "Zusätzliche Parameter (JSON)",
-        "source_oauth_extra_params_hint": "Optionales JSON-Objekt mit zusätzlichen Parametern für die Token-Anfrage, z. B. {\"resource\": \"https://provider.example.com\"}",
+        "source_oauth_extra_params_hint": "Optionales JSON-Objekt. prompt, access_type, login_hint, domain_hint und hd gehen an die Anmeldung beim Provider, alle anderen an die Token-Anfrage, z. B. {\"resource\": \"https://provider.example.com\"}",
+        "source_presets": "Vorlagen",
+        "source_presets_hint": "Füllt die Serverdaten des Anbieters aus. Viele Anbieter (z. B. Gmail, iCloud, Yahoo) verlangen für IMAP ein App-Passwort oder die Freischaltung von IMAP in den Postfach-Einstellungen.",
+        "source_tenant": "Microsoft-365-Tenant",
+        "source_tenant_hint": "Verzeichnis-ID (Mandant) oder eine Domain des Tenants, z. B. contoso.onmicrosoft.com. Wird in die Endpunkte eingesetzt.",
         "source_scope_cc_hint": "Quellen mit Client-Credentials-Flow sind immer nur für ihren Ersteller sichtbar.",
         "source_readonly_hint": "Diese Sync-Quelle ist freigegeben und kann nur angesehen werden.",
         "copy": "Kopieren",

data/web/lang/lang.en-gb.json

@@ -572,6 +572,7 @@
         "imapsync_source_oauth_required": "OAuth2 requires token endpoint, client ID, client secret and scope",
         "imapsync_source_oauth_endpoint_invalid": "Invalid OAuth2 endpoint: an https:// URL with a valid host name is required (sources not created by an administrator need a publicly reachable host)",
         "imapsync_source_oauth_extra_invalid": "OAuth2 extra parameters must be a JSON object with simple values, e.g. {\"resource\": \"https://example.com\"}",
+        "imapsync_source_tenant_missing": "Please enter the Microsoft 365 tenant",
         "imapsync_source_oauth_invalid": "Invalid OAuth2 configuration",
         "imapsync_max_parallel_invalid": "Parallel process count must be between 1 and 32",
         "imapsync_source_description_invalid": "The description must not exceed 255 characters",
@@ -1363,7 +1364,11 @@
         "source_oauth_client_secret": "Client secret",
         "source_oauth_scope": "Scope",
         "source_oauth_extra_params": "Extra parameters (JSON)",
-        "source_oauth_extra_params_hint": "Optional JSON object with additional parameters for the token request, e.g. {\"resource\": \"https://provider.example.com\"}",
+        "source_oauth_extra_params_hint": "Optional JSON object. prompt, access_type, login_hint, domain_hint and hd are sent with the provider sign-in, all others with the token request, e.g. {\"resource\": \"https://provider.example.com\"}",
+        "source_presets": "Presets",
+        "source_presets_hint": "Fills in the provider's server settings. Many providers (e.g. Gmail, iCloud, Yahoo) require an app password or IMAP access to be enabled in the mailbox settings.",
+        "source_tenant": "Microsoft 365 tenant",
+        "source_tenant_hint": "Directory (tenant) ID or a domain of the tenant, e.g. contoso.onmicrosoft.com. It is inserted into the endpoints.",
         "source_scope_cc_hint": "Sources using the client credentials flow are always private to their creator.",
         "source_readonly_hint": "This sync source is shared with you and can only be viewed.",
         "copy": "Copy",

data/web/templates/modals/imapsync_source.twig

@@ -9,6 +9,29 @@
       <div class="modal-body">
         <form class="form-horizontal" data-cached-form="true" role="form" data-id="add_imapsync_source">
           <input type="hidden" value="0" name="active">
+          <div class="row mb-3">
+            <label class="control-label col-sm-2 text-sm-end">{{ lang.syncjobs.source_presets }}</label>
+            <div class="col-sm-10 small">
+              {% if mailcow_cc_role != 'user' %}
+              <div class="d-flex flex-wrap column-gap-3">
+                <a href="#" class="imapsync-source-preset" data-preset="m365_app">Microsoft 365 ({{ lang.syncjobs.source_flow_badge_app }})</a>
+                <a href="#" class="imapsync-source-preset" data-preset="m365_user">Microsoft 365 ({{ lang.syncjobs.source_flow_badge_user }})</a>
+                <a href="#" class="imapsync-source-preset" data-preset="google_user">Google ({{ lang.syncjobs.source_flow_badge_user }})</a>
+              </div>
+              {% endif %}
+              <div class="d-flex flex-wrap column-gap-3">
+                <a href="#" class="imapsync-source-preset" data-preset="gmail">Gmail</a>
+                <a href="#" class="imapsync-source-preset" data-preset="gmx">GMX</a>
+                <a href="#" class="imapsync-source-preset" data-preset="webde">WEB.DE</a>
+                <a href="#" class="imapsync-source-preset" data-preset="tonline">T-Online</a>
+                <a href="#" class="imapsync-source-preset" data-preset="icloud">iCloud</a>
+                <a href="#" class="imapsync-source-preset" data-preset="yahoo">Yahoo</a>
+                <a href="#" class="imapsync-source-preset" data-preset="ionos">IONOS</a>
+                <a href="#" class="imapsync-source-preset" data-preset="strato">STRATO</a>
+              </div>
+              <span class="text-muted">{{ lang.syncjobs.source_presets_hint }}</span>
+            </div>
+          </div>
           <div class="row mb-2">
             <label class="control-label col-sm-2 text-sm-end" for="addsrc_name">{{ lang.syncjobs.source_name }}</label>
             <div class="col-sm-10">
@@ -101,6 +124,13 @@
                 </select>
               </div>
             </div>
+            <div class="row mb-2 imapsync-source-tenant-row" style="display:none;">
+              <label class="control-label col-sm-2 text-sm-end" for="addsrc_tenant">{{ lang.syncjobs.source_tenant }}</label>
+              <div class="col-sm-10">
+                <input type="text" class="form-control imapsync-source-tenant" id="addsrc_tenant" placeholder="contoso.onmicrosoft.com">
+                <small class="text-muted">{{ lang.syncjobs.source_tenant_hint }}</small>
+              </div>
+            </div>
             <div class="row mb-2 imapsync-source-authcode-block" style="display:none;">
               <label class="control-label col-sm-2 text-sm-end" for="addsrc_oauth_authorize_endpoint">{{ lang.syncjobs.source_authorize_endpoint }}</label>
               <div class="col-sm-10">