[Web] Updated lang.es-es.json

Co-authored-by: ciri-cuervo <nicolascirigliano@gmail.com>

.gitignore

@@ -78,6 +78,4 @@ refresh_images.sh
 update_diffs/
 create_cold_standby.sh
 !data/conf/nginx/mailcow_auth.conf
-data/conf/postfix/postfix-tlspol
-tests/php/vendor/
-tests/php/.phpunit.cache/
+data/conf/postfix/postfix-tlspol

_modules/scripts/new_options.sh

@@ -111,16 +111,16 @@ adapt_new_options() {
             echo "SQL_PORT=127.0.0.1:13306" >> mailcow.conf
             ;;
         API_KEY)
-            echo '# Read-write service API key, shown read-only in the web UI' >> mailcow.conf
+            echo '# Create or override API key for web UI' >> mailcow.conf
             echo "#API_KEY=" >> mailcow.conf
             ;;
         API_KEY_READ_ONLY)
-            echo '# Read-only service API key, shown read-only in the web UI' >> mailcow.conf
+            echo '# Create or override read-only API key for web UI' >> mailcow.conf
             echo "#API_KEY_READ_ONLY=" >> mailcow.conf
             ;;
         API_ALLOW_FROM)
-            echo '# Must be set for API_KEY and API_KEY_READ_ONLY to be active' >> mailcow.conf
-            echo '# Comma separated list of IPs/networks' >> mailcow.conf
+            echo '# Must be set for API_KEY to be active' >> mailcow.conf
+            echo '# IPs only, no networks (networks can be set via UI)' >> mailcow.conf
             echo "#API_ALLOW_FROM=" >> mailcow.conf
             ;;
         SNAT_TO_SOURCE)

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -412,63 +412,6 @@ class DockerApi:
 
         sogo_return = container.exec_run(["/bin/bash", "-c", f"sogo-tool rename-user '{old_username}' '{new_username}'"], user='sogo')
         return self.exec_run_handler('generic', sogo_return)
-  # api call: container_post - post_action: exec - cmd: sogo - task: subscribe
-  # Subscribes users to a folder of the owner, or unsubscribes them; values are passed as environment and
-  # arguments, never through the shell
-  def container_post__exec__sogo__subscribe(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    for container in self.sync_docker_client.containers.list(filters=filters):
-      owner = str(request_json.get('owner', ''))
-      folder = str(request_json.get('folder', ''))
-      users = [str(user) for user in request_json.get('users', []) if str(user)]
-      if not owner or not users or not re.match(r'^(Calendar|Contacts)/[A-Za-z0-9._-]+$', folder):
-        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner, folder and users required' }, indent=4), media_type="application/json")
-      script = 'for user in "$@"; do sogo-tool manage-acl "$SOGO_ACTION" "$SOGO_OWNER" "$SOGO_FOLDER" "$user" || exit 1; done'
-      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe"] + users, user='sogo',
-        environment={ 'SOGO_ACTION': 'unsubscribe' if request_json.get('unsubscribe') else 'subscribe', 'SOGO_OWNER': owner, 'SOGO_FOLDER': folder })
-      return self.exec_run_handler('generic', subscribe_return)
-  # api call: container_post - post_action: exec - cmd: sogo - task: init_user
-  # Creates the personal folders of users that never logged in to SOGo (e.g. shared mailboxes)
-  def container_post__exec__sogo__init_user(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    for container in self.sync_docker_client.containers.list(filters=filters):
-      users = [str(user) for user in request_json.get('users', []) if str(user)]
-      if not users:
-        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
-      init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
-      return self.exec_run_handler('generic', init_return)
-  # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
-  # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
-  def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    for container in self.sync_docker_client.containers.list(filters=filters):
-      owner = str(request_json.get('owner', ''))
-      users = [str(user) for user in request_json.get('users', []) if str(user)]
-      if not owner or not users or any(c in owner for c in '/*%'):
-        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner and users required' }, indent=4), media_type="application/json")
-      if request_json.get('unsubscribe'):
-        # Without access the user no longer sees the folders, their names come from the owner
-        script = ('set -o pipefail; for user in "$@"; do doveadm mailbox list -u "$SUB_OWNER" | while IFS= read -r folder; do '
-          'case "$folder" in Shared|Shared/*) continue;; esac; doveadm mailbox unsubscribe -u "$user" "Shared/$SUB_OWNER/$folder" || exit 1; '
-          'done || exit 1; done')
-      else:
-        # No visible folder means no access yet (ACL, auth cache): fail, so the subscription is retried
-        script = ('for user in "$@"; do folders=$(doveadm mailbox list -u "$user" "Shared/$SUB_OWNER/*") && [ -n "$folders" ] || exit 1; '
-          'while IFS= read -r folder; do doveadm mailbox subscribe -u "$user" "$folder" || exit 1; done <<< "$folders"; done')
-      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe_shared"] + users, environment={ 'SUB_OWNER': owner })
-      return self.exec_run_handler('generic', subscribe_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: get_acl
   def container_post__exec__doveadm__get_acl(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -496,8 +439,7 @@ class DockerApi:
           if len(acls) >= 2:
             for acl in acls[1:]:
               user_id, rights = acl.split(maxsplit=1)
-              # user=<name> as plain name, other identifiers (group=...) keep their type
-              user_id = user_id.split('=', 1)[1] if user_id.startswith('user=') else user_id
+              user_id = user_id.split('=')[1]
               mailbox_seen.append(mailbox)
               formatted_acls.append({ 'user': id, 'id': user_id, 'mailbox': mailbox, 'rights': rights.split() })
         elif "Shared" in shared_folder and "/" in shared_folder:
@@ -534,7 +476,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' 'user={id}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -568,50 +510,8 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' 'user={id}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
-  # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
-  # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
-  # environment, never through the shell
-  def container_post__exec__doveadm__set_acl_all(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    for container in self.sync_docker_client.containers.list(filters=filters):
-      user = request_json.get('user', '')
-      id = self.acl_identifier(request_json.get('id', ''))
-      rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
-      if not user or id in ('user=', 'group='):
-        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
-      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
-      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
-        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
-      acl_return = container.exec_run(["/bin/bash", "-c", script],
-        environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
-      return self.exec_run_handler('generic', acl_return)
-  # api call: container_post - post_action: exec - cmd: doveadm - task: auth_cache_flush
-  # Drops cached userdb results (e.g. acl_groups) of the given users, all users if none are given
-  def container_post__exec__doveadm__auth_cache_flush(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    for container in self.sync_docker_client.containers.list(filters=filters):
-      users = [str(user) for user in request_json.get('users', []) if str(user)]
-      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
-      return self.exec_run_handler('generic', flush_return)
-
-  # Rights doveadm acl set accepts from the API
-  acl_rights = ["admin", "create", "delete", "expunge", "insert", "lookup", "post", "read", "write", "write-deleted", "write-seen"]
-
-  # ACL identifier for doveadm: "user=<name>" for a plain name, "user=..." / "group=..." as given
-  def acl_identifier(self, id):
-    if id.startswith('user=') or id.startswith('group='):
-      return id
-    return 'user=' + id
 
 
   # Collect host stats

data/Dockerfiles/dovecot/Dockerfile

@@ -68,6 +68,7 @@ RUN addgroup -g 5000 vmail \
   perl-package-stash-xs \
   perl-par-packer \
   perl-parse-recdescent \
+  perl-lockfile-simple \
   libproc2 \
   perl-readonly \
   perl-regexp-common \
@@ -119,7 +120,7 @@ COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
-COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
+COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -127,8 +127,7 @@ cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 # Autogenerated by mailcow
 driver = mysql
 connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
-# acl_groups: active mailbox groups of the user, IMAP grants are ACLs on group=mcg_<id>
-user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule, (SELECT GROUP_CONCAT(CONCAT('mcg_', gm.group_id)) FROM mailbox_group_member gm JOIN mailbox_group g ON g.id = gm.group_id AND g.active = 1 WHERE gm.username = mailbox.username) AS acl_groups FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
+user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
 iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
 EOF
 
@@ -280,7 +279,7 @@ chmod g+rw /dev/console
 chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_runner.py \
+  /usr/local/bin/imapsync_runner.pl \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -305,15 +304,6 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 # Envsubst maildir_gc
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
-# Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
-DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
-  echo "Waiting for schema update..."
-  sleep 5
-  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-done
-echo "DB schema is ${DBV_NOW}"
-
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do
   if [[ ! -z $(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"${DBNAME}\" AND TABLE_NAME = 'versions'") ]]; then

data/Dockerfiles/dovecot/imapsync_runner.pl

@@ -0,0 +1,196 @@
+#!/usr/bin/perl
+
+use DBI;
+use LockFile::Simple qw(lock trylock unlock);
+use Proc::ProcessTable;
+use Data::Dumper qw(Dumper);
+use IPC::Run 'run';
+use File::Temp;
+use Try::Tiny;
+use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT);
+
+sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
+my $t = Proc::ProcessTable->new;
+my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table};
+if ($imapsync_running ge 1)
+{
+  print "imapsync is active, exiting...";
+  exit;
+}
+
+sub qqw($) {
+  my @params = ();
+  my @values = split(/(?=--)/, $_[0]);
+  foreach my $val (@values) {
+    my @tmpparam = split(/ /, $val, 2);
+    foreach my $tmpval (@tmpparam) {
+        if ($tmpval ne '') {
+          push @params, $tmpval;
+        }
+    }
+  }
+  foreach my $val (@params) {
+    $val=trim($val);
+  }
+  return @params;
+}
+
+$run_dir="/tmp";
+$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
+$lock_file = $run_dir . "/imapsync_busy";
+$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
+$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
+$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
+  mysql_auto_reconnect => 1,
+  mysql_enable_utf8mb4 => 1
+});
+$dbh->do("UPDATE imapsync SET is_running = 0");
+
+sub sig_handler {
+  # Send die to force exception in "run"
+  die "sig_handler received signal, preparing to exit...\n";
+};
+
+open my $file, '<', "/etc/sogo/sieve.creds"; 
+my $creds = <$file>; 
+close $file;
+my ($master_user, $master_pass) = split /:/, $creds;
+my $sth = $dbh->prepare("SELECT id,
+  user1,
+  user2,
+  host1,
+  authmech1,
+  password1,
+  exclude,
+  port1,
+  enc1,
+  delete2duplicates,
+  maxage,
+  subfolder2,
+  delete1,
+  delete2,
+  automap,
+  skipcrossduplicates,
+  maxbytespersecond,
+  custom_params,
+  subscribeall,
+  timeout1,
+  timeout2,
+  dry
+    FROM imapsync
+      WHERE active = 1
+        AND is_running = 0
+        AND (
+          UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
+          OR
+          last_run IS NULL)
+  ORDER BY last_run");
+
+$sth->execute();
+my $row;
+
+while ($row = $sth->fetchrow_arrayref()) {
+
+  $id                  = @$row[0];
+  $user1               = @$row[1];
+  $user2               = @$row[2];
+  $host1               = @$row[3];
+  $authmech1           = @$row[4];
+  $password1           = @$row[5];
+  $exclude             = @$row[6];
+  $port1               = @$row[7];
+  $enc1                = @$row[8];
+  $delete2duplicates   = @$row[9];
+  $maxage              = @$row[10];
+  $subfolder2          = @$row[11];
+  $delete1             = @$row[12];
+  $delete2             = @$row[13];
+  $automap             = @$row[14];
+  $skipcrossduplicates = @$row[15];
+  $maxbytespersecond   = @$row[16];
+  $custom_params       = @$row[17];
+  $subscribeall        = @$row[18];
+  $timeout1            = @$row[19];
+  $timeout2            = @$row[20];
+  $dry                 = @$row[21];
+
+  if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
+
+  my $template = $run_dir . '/imapsync.XXXXXXX';
+  my $passfile1 = File::Temp->new(TEMPLATE => $template);
+  my $passfile2 = File::Temp->new(TEMPLATE => $template);
+  
+  binmode( $passfile1, ":utf8" );
+  
+  print $passfile1 "$password1\n";
+  print $passfile2 trim($master_pass) . "\n";
+
+  my @custom_params_a = qqw($custom_params);
+  my $custom_params_ref = \@custom_params_a;
+
+  my $generated_cmds = [ "/usr/local/bin/imapsync",
+  "--tmpdir", "/tmp",
+  "--nofoldersizes",
+  "--addheader",
+  ($timeout1 le "0" ? () : ('--timeout1', $timeout1)),
+  ($timeout2 le "0" ? () : ('--timeout2', $timeout2)),
+  ($exclude eq "" ? () : ("--exclude", $exclude)),
+  ($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)),
+  ($maxage eq "0" ? () : ('--maxage', $maxage)),
+  ($maxbytespersecond eq "0" ? () : ('--maxbytespersecond', $maxbytespersecond)),
+  ($delete2duplicates ne "1" ? () : ('--delete2duplicates')),
+  ($subscribeall  ne "1" ? () : ('--subscribeall')),
+  ($delete1 ne "1" ? () : ('--delete')),
+  ($delete2 ne "1" ? () : ('--delete2')),
+  ($automap ne "1" ? () : ('--automap')),
+  ($skipcrossduplicates ne "1" ? () : ('--skipcrossduplicates')),
+  (!defined($enc1) ? () : ($enc1)),
+  "--host1", $host1,
+  "--user1", $user1,
+  "--passfile1", $passfile1->filename,
+  "--port1", $port1,
+  "--host2", "localhost",
+  "--user2", $user2 . '*' . trim($master_user),
+  "--passfile2", $passfile2->filename,
+  ($dry eq "1" ? ('--dry') : ()),
+  '--no-modulesversion',
+  '--noreleasecheck'];
+
+  try {
+    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
+    $is_running->bind_param( 1, ${id} );
+    $is_running->execute();
+
+    run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
+
+    # check exit code and status
+    ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
+
+    $success = 0;
+    if (defined $exit_code && $exit_code == 0) {
+      $success = 1;
+    }
+
+    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
+    $update->bind_param( 1, ${stdout} );
+    $update->bind_param( 2, ${success} );
+    $update->bind_param( 3, ${exit_status} );
+    $update->bind_param( 4, ${id} );
+    $update->execute();
+  } catch {
+    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
+    $update->bind_param( 1, ${id} );
+    $update->execute();
+  } finally {
+    $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
+    $update->bind_param( 1, ${id} );
+    $update->execute();
+  };
+
+
+}
+
+$sth->finish();
+$dbh->disconnect();
+
+$lockmgr->unlock($lock_file);

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -1,331 +0,0 @@
-#!/usr/bin/python3
-
-import fcntl
-import json
-import os
-import re
-import signal
-import subprocess
-import sys
-import tempfile
-import time
-import MySQLdb
-import MySQLdb.cursors
-
-run_dir = '/tmp'
-# Max size of imapsync output stored in returned_text (the tail holds the summary)
-max_output = 16 * 1024 * 1024
-lock_file = run_dir + '/imapsync_busy.lock'
-# An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
-token_min_valid = 900
-# Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
-token_sync_interval = 60
-
-# Next due job: higher prio first, never-run jobs first, then oldest run
-next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
-  i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
-  i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
-  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
-    FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
-      WHERE i.active = 1
-        AND s.active = 1
-        AND i.is_running = 0
-        AND (UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(i.last_run) > i.mins_interval * 60 OR i.last_run IS NULL)
-        {skip}
-  ORDER BY i.prio DESC, i.last_run IS NOT NULL, i.last_run ASC, i.id ASC
-  LIMIT 1"""
-# Atomic claim, a job is only started when this flips is_running 0 -> 1
-claim_sql = """UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL
-  WHERE id = %s AND is_running = 0 AND active = 1"""
-# Access token: client_credentials share the source token, authorization_code use the token of
-# source + local mailbox (user2) + remote user (user1). issued/error describe the last refresh.
-source_token_sql = """SELECT oauth_access_token AS access_token, oauth_token_expires AS token_expires,
-  UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued, oauth_last_refresh_error AS error
-  FROM imapsync_source WHERE id = %s"""
-user_token_sql = """SELECT access_token, token_expires, UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued,
-  last_refresh_error AS error FROM imapsync_source_oauth_token
-  WHERE source_id = %s AND owner = %s AND username = %s"""
-
-conn = None
-# Popen => job of running workers
-running = {}
-# Jobs released without a run (token not ready), skipped for the rest of this runner round
-deferred = set()
-stopping = False
-
-def num(value):
-  try:
-    return int(value)
-  except (TypeError, ValueError):
-    return 0
-
-def unlink(files):
-  for path in files:
-    try:
-      os.unlink(path)
-    except OSError:
-      pass
-
-def imapsync_running():
-  for pid in filter(str.isdigit, os.listdir('/proc')):
-    try:
-      with open('/proc/' + pid + '/cmdline', 'rb') as f:
-        cmdline = f.read().replace(b'\0', b' ')
-    except OSError:
-      continue
-    if re.search(rb'imapsync\s', cmdline, re.I):
-      return True
-  return False
-
-def connect():
-  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
-    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
-
-# Reconnect when the server dropped the connection (syncs can outlast wait_timeout)
-def query(sql, args=None):
-  global conn
-  try:
-    conn.ping()
-  except MySQLdb.Error:
-    conn = connect()
-  cur = conn.cursor(MySQLdb.cursors.DictCursor)
-  cur.execute(sql, args)
-  return cur
-
-# Store a result and release the job
-def finish_job(job_id, text, success, exit_status):
-  try:
-    query("UPDATE imapsync SET returned_text = %s, success = %s, exit_status = %s, last_run = NOW(), is_running = 0 WHERE id = %s",
-      (text, success, exit_status, job_id))
-  except MySQLdb.Error:
-    query("UPDATE imapsync SET success = 0, last_run = NOW(), is_running = 0 WHERE id = %s", (job_id,))
-
-# Release a job without a run; last_run stays, so the next runner round retries it
-def defer_job(job_id, text, exit_status):
-  query("UPDATE imapsync SET returned_text = %s, success = 0, exit_status = %s, is_running = 0 WHERE id = %s",
-    (text, exit_status, job_id))
-  deferred.add(job_id)
-
-def oauth_token(j):
-  if j['oauth_flow'] == 'authorization_code':
-    return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
-  return query(source_token_sql, (j['source_id'],)).fetchone() or {}
-
-# Valid for token_min_valid, or half the lifetime of a short-lived token (known when its last refresh succeeded)
-def token_ready(row):
-  token, expires, issued = row.get('access_token'), row.get('token_expires'), row.get('issued')
-  if not token or expires is None:
-    return False
-  need = token_min_valid
-  if issued and not row.get('error') and expires > int(issued):
-    need = min(need, (expires - int(issued)) // 2)
-  return expires - time.time() >= need
-
-# Write a refreshed token over the token file of a running job (atomic, same path)
-def sync_token_file(oauth):
-  token = oauth_token(oauth['job']).get('access_token')
-  if not token or token == oauth['token']:
-    return
-  fd, tmp = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
-  try:
-    with os.fdopen(fd, 'w', encoding='utf-8') as f:
-      f.write(token + '\n')
-    os.replace(tmp, oauth['file'])
-    oauth['token'] = token
-  except OSError:
-    unlink([tmp])
-
-# Max concurrent imapsync processes and global per-process bandwidth cap in bytes/s (0 = off),
-# re-read before every claim round so changes apply to long runs
-def read_settings():
-  settings = {r['name']: r['value'] for r in query("SELECT name, value FROM imapsync_settings")}
-  return max(num(settings.get('max_parallel')), 1), max(num(settings.get('max_bytes_per_second')), 0)
-
-# Build temp files and argv for a claimed job. Returns None when the job was finished without running imapsync.
-def prepare_job(j, global_bw, files):
-  def tmpfile(content):
-    fd, path = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
-    files.append(path)
-    with os.fdopen(fd, 'w', encoding='utf-8') as f:
-      f.write(content)
-    return path
-
-  auth_type = j['auth_type'] or 'PLAIN'
-  oauth = None
-  if auth_type == 'XOAUTH2':
-    row = oauth_token(j)
-    if not token_ready(row):
-      defer_job(j['id'], "OAuth access token for the configured source is missing or expires too soon; refresh cron will retry.",
-        'OAUTH_TOKEN_NOT_READY')
-      return None
-    oauth = {'job': j, 'token': row['access_token'], 'file': tmpfile(row['access_token'] + '\n')}
-    auth_args = ['--authmech1', 'XOAUTH2', '--oauthaccesstoken1', oauth['file']]
-  else:
-    auth_args = ['--passfile1', tmpfile(j['password1'] + '\n')]
-    # imapsync auto-picks PLAIN, only pass --authmech1 for LOGIN/CRAM-MD5
-    if auth_type != 'PLAIN':
-      auth_args += ['--authmech1', auth_type]
-
-  # Custom params: JSON array of {o,v}, validated by the backend on write. One argv token per pair,
-  # "--opt" or "--opt=value" (value glued to its option, no shell)
-  custom_params = []
-  try:
-    pairs = json.loads(j['custom_params'] or '[]')
-  except ValueError:
-    pairs = []
-  for p in pairs if isinstance(pairs, list) else []:
-    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
-      continue
-    value = '' if p.get('v') is None else str(p['v'])
-    custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
-
-  # Effective per-process bandwidth cap (bytes/s): per-job value, capped by the global limit
-  eff_bw = num(j['maxbytespersecond'])
-  if global_bw > 0 and (eff_bw <= 0 or eff_bw > global_bw):
-    eff_bw = global_bw
-
-  # --nolog: the output is stored in returned_text, imapsync's own log dir would be the (read-only) cwd
-  cmd = ['/usr/local/bin/imapsync', '--tmpdir', '/tmp', '--nolog', '--nofoldersizes', '--addheader']
-  if num(j['timeout1']) > 0:
-    cmd += ['--timeout1', str(j['timeout1'])]
-  if num(j['timeout2']) > 0:
-    cmd += ['--timeout2', str(j['timeout2'])]
-  if j['exclude']:
-    cmd += ['--exclude', j['exclude']]
-  if j['subfolder2']:
-    cmd += ['--subfolder2', j['subfolder2']]
-  if num(j['maxage']) != 0:
-    cmd += ['--maxage', str(j['maxage'])]
-  if eff_bw > 0:
-    cmd += ['--maxbytespersecond', str(eff_bw)]
-  for col, opt in (('delete2duplicates', '--delete2duplicates'), ('subscribeall', '--subscribeall'),
-      ('delete1', '--delete1'), ('delete2', '--delete2'), ('automap', '--automap'),
-      ('skipcrossduplicates', '--skipcrossduplicates')):
-    if j[col] == 1:
-      cmd.append(opt)
-  enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
-  if enc1:
-    cmd.append(enc1)
-  cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
-  cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
-  if j['dry'] == 1:
-    cmd.append('--dry')
-  cmd += ['--no-modulesversion', '--noreleasecheck'] + custom_params
-
-  return {'id': j['id'], 'files': files, 'outfile': tmpfile(''), 'cmd': cmd, 'oauth': oauth}
-
-# Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
-def start_job(job):
-  with open(job['outfile'], 'wb') as out:
-    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
-
-# Store the result of a finished worker and remove its temp files
-def reap_job(job):
-  stdout = b''
-  try:
-    with open(job['outfile'], 'rb') as f:
-      size = os.fstat(f.fileno()).st_size
-      if size > max_output:
-        f.seek(size - max_output)
-        stdout = b'[output truncated, last %d bytes]\n' % max_output
-      stdout += f.read()
-  except OSError:
-    pass
-  unlink(job['files'])
-  text = stdout.decode('utf-8', 'replace')
-  # Last match: the echoed command line at the top may contain user controlled values
-  m = re.findall(r'Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)', text)
-  exit_code, exit_status = m[-1] if m else (None, None)
-  if exit_code is None:
-    text = 'Could not start or finish imapsync\n' + text
-  finish_job(job['id'], text, 1 if exit_code is not None and int(exit_code) == 0 else 0, exit_status)
-
-# Stop claiming new jobs, pass the signal on to the running imapsync processes
-def stop(signum, frame):
-  global stopping
-  stopping = True
-  for proc in list(running):
-    proc.send_signal(signal.SIGTERM)
-
-# Released by the kernel when the process ends, a crashed run leaves no stale lock
-lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
-try:
-  fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-except OSError:
-  sys.exit("can't lock " + lock_file)
-
-if imapsync_running():
-  print('imapsync is active, exiting...')
-  sys.exit(0)
-
-for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
-  signal.signal(sig, stop)
-
-conn = connect()
-# Stale flags from crashed runs, safe only here: under the lock and with no imapsync running
-query("UPDATE imapsync SET is_running = 0")
-
-with open('/etc/sogo/sieve.creds') as f:
-  master_user, master_pass = [s.strip() for s in f.readline().split(':', 1)]
-
-# Worker pool: keep up to max_parallel workers busy, claim the next due job whenever a slot is free
-# (after every worker exit and at least every 10 seconds while workers run)
-last_token_sync = time.monotonic()
-try:
-  while True:
-    for proc, job in list(running.items()):
-      if proc.poll() is not None:
-        del running[proc]
-        reap_job(job)
-
-    if time.monotonic() - last_token_sync >= token_sync_interval:
-      last_token_sync = time.monotonic()
-      for job in running.values():
-        if job['oauth']:
-          try:
-            sync_token_file(job['oauth'])
-          except MySQLdb.Error as e:
-            print('imapsync job %d: %s' % (job['id'], e), file=sys.stderr)
-
-    max_parallel, global_bw = read_settings()
-    # Deferred and finished jobs drop out of next_sql, only failed claims are capped
-    failed_claims = 0
-    while not stopping and len(running) < max_parallel and failed_claims < 100:
-      skip = ('AND i.id NOT IN (%s)' % ', '.join(['%s'] * len(deferred))) if deferred else ''
-      j = query(next_sql.format(skip=skip), tuple(deferred) or None).fetchone()
-      if not j:
-        break
-      if query(claim_sql, (j['id'],)).rowcount != 1:
-        failed_claims += 1
-        continue
-
-      files = []
-      try:
-        job = prepare_job(j, global_bw, files)
-      except Exception as e:
-        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
-        finish_job(j['id'], 'Could not start or finish imapsync', 0, None)
-        job = None
-      if not job:
-        unlink(files)
-        continue
-      try:
-        running[start_job(job)] = job
-      except OSError as e:
-        # Release the claim, the job is picked up again later
-        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
-        unlink(files)
-        query("UPDATE imapsync SET is_running = 0 WHERE id = %s", (j['id'],))
-        break
-
-    if not running:
-      break
-    # A worker exit ends the wait early
-    for _ in range(50):
-      if any(proc.poll() is not None for proc in running):
-        break
-      time.sleep(0.2)
-finally:
-  # Never leave password/token files behind
-  for job in running.values():
-    unlink(job['files'])

data/Dockerfiles/netfilter/docker-entrypoint.sh

@@ -17,8 +17,8 @@ if [ $nftables_found -gt $iptables_found ]; then
 fi
 
 if [ $nftables_found -eq 0 ] && [ $nftables_found -eq $iptables_found ]; then
-  nftables_lines=$(nft list ruleset 2>/dev/null | wc -l)
-  iptables_lines=$(iptables-save 2>/dev/null | wc -l)
+  nftables_lines=$(nft list ruleset | wc -l)
+  iptables_lines=$(iptables-save | wc -l)
   if [ $nftables_lines -gt $iptables_lines ]; then
     backend=nftables
   else

data/Dockerfiles/netfilter/main.py

@@ -1,57 +1,548 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import time
-import signal
-import redis
-from modules.Logger import Logger
-from modules.Netfilter import Netfilter
-
-chain_name = "MAILCOW"
-
-def connect_redis(logger):
-  if os.getenv('REDIS_SLAVEOF_IP', ''):
-    host, port = os.getenv('REDIS_SLAVEOF_IP'), os.getenv('REDIS_SLAVEOF_PORT', '')
-  else:
-    host, port = os.getenv('IPV4_NETWORK', '172.22.1') + '.249', 6379
-  while True:
-    try:
-      # log lines are not always valid UTF-8
-      r = redis.StrictRedis(host=host, port=port, db=0, password=os.environ['REDISPASS'], decode_responses=True,
-                            encoding_errors='replace', socket_connect_timeout=5, socket_timeout=30)
-      r.ping()
-      return r
-    except Exception as ex:
-      logger.logWarn('Redis connection failed: %s - trying again in 3 seconds' % ex)
-      time.sleep(3)
-
-if __name__ == '__main__':
-  logger = Logger()
-
-  backend = sys.argv[1]
-  if backend == "nftables":
-    from modules.NFTables import NFTables
-    logger.logInfo('Using NFTables backend')
-    tables = NFTables(chain_name, logger)
-  else:
-    from modules.IPTables import IPTables
-    logger.logInfo('Using IPTables backend')
-    logger.logWarn(
-        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
-        "Please switch to nftables on your host to ensure complete compatibility."
-    )
-    time.sleep(5)
-    tables = IPTables(chain_name, logger)
-
-  r = connect_redis(logger)
-  logger.set_redis(r)
-
-  netfilter = Netfilter(r, tables, logger, os.environ)
-  signal.signal(signal.SIGTERM, lambda signum, frame: netfilter.stop.set())
-  signal.signal(signal.SIGINT, lambda signum, frame: netfilter.stop.set())
-
-  netfilter.setup()
-  netfilter.run()
-  netfilter.shutdown()
-  sys.exit(0)
+#!/usr/bin/env python3
+
+DEBUG = False
+
+import re
+import os
+import sys
+import time
+import atexit
+import signal
+import ipaddress
+from collections import Counter
+from random import randint
+from threading import Thread
+from threading import Lock
+import redis
+import json
+import dns.resolver
+import dns.exception
+import uuid
+from modules.Logger import Logger
+from modules.IPTables import IPTables
+from modules.NFTables import NFTables
+
+def logdebug(msg):
+  if DEBUG:
+    logger.logInfo("DEBUG: %s" % msg)
+
+# Globals
+WHITELIST = []
+BLACKLIST = []
+bans = {}
+quit_now = False
+exit_code = 0
+lock = Lock()
+chain_name = "MAILCOW"
+r = None
+pubsub = None
+clear_before_quit = False
+
+def refreshF2boptions():
+  global f2boptions
+  global quit_now
+  global exit_code
+  f2boptions = {}
+
+  if not r.get('F2B_OPTIONS'):
+    f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
+    f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
+    f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
+    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
+    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
+    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
+    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
+  else:
+    try:
+      f2boptions = json.loads(r.get('F2B_OPTIONS'))
+    except ValueError as e:
+      logger.logCrit(
+        'Error loading F2B options: F2B_OPTIONS is not json. Exception: %s' % e)
+      quit_now = True
+      exit_code = 2
+
+  verifyF2boptions(f2boptions)
+  r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
+
+def verifyF2boptions(f2boptions):
+  verifyF2boption(f2boptions, 'ban_time', 1800)
+  verifyF2boption(f2boptions, 'max_ban_time', 10000)
+  verifyF2boption(f2boptions, 'ban_time_increment', True)
+  verifyF2boption(f2boptions, 'max_attempts', 10)
+  verifyF2boption(f2boptions, 'retry_window', 600)
+  verifyF2boption(f2boptions, 'netban_ipv4', 32)
+  verifyF2boption(f2boptions, 'netban_ipv6', 128)
+  verifyF2boption(f2boptions, 'banlist_id', str(uuid.uuid4()))
+  verifyF2boption(f2boptions, 'manage_external', 0)
+
+def verifyF2boption(f2boptions, f2boption, f2bdefault):
+  f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
+
+def refreshF2bregex():
+  global f2bregex
+  global quit_now
+  global exit_code
+  if not r.get('F2B_REGEX'):
+    f2bregex = {}
+    f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
+    f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
+    f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
+    f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
+    f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
+    f2bregex[6] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)'
+    f2bregex[7] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)'
+    f2bregex[8] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
+    f2bregex[9] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
+    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
+  else:
+    try:
+      f2bregex = {}
+      f2bregex = json.loads(r.get('F2B_REGEX'))
+    except ValueError:
+      logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
+      quit_now = True
+      exit_code = 2
+
+def get_ip(address):
+  ip = ipaddress.ip_address(address)
+  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
+    ip = ip.ipv4_mapped
+  if ip.is_private or ip.is_loopback:
+    return False
+
+  return ip
+
+def ban(address):
+  global f2boptions
+  global lock
+  logdebug("ban() called with address=%s" % address)
+  refreshF2boptions()
+  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
+  RETRY_WINDOW = int(f2boptions['retry_window'])
+  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
+  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
+
+  ip = get_ip(address)
+  if not ip:
+    logdebug("No valid IP -- skipping ban()")
+    return
+  address = str(ip)
+  self_network = ipaddress.ip_network(address)
+
+  with lock:
+    temp_whitelist = set(WHITELIST)
+    logdebug("Checking if %s overlaps with any WHITELIST entries" % self_network)
+    if temp_whitelist:
+      for wl_key in temp_whitelist:
+        wl_net = ipaddress.ip_network(wl_key, False)
+        logdebug("Checking overlap between %s and %s" % (self_network, wl_net))
+        if wl_net.overlaps(self_network):
+          logger.logInfo(
+            'Address %s is allowlisted by rule %s' % (self_network, wl_net))
+          return
+
+  net = ipaddress.ip_network(
+    (address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
+  net = str(net)
+  logdebug("Ban net: %s" % net)
+
+  if not net in bans:
+    bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
+    logdebug("Initing new ban counter for %s" % net)
+
+  current_attempt = time.time()
+  logdebug("Current attempt ts=%s, previous: %s, retry_window: %s" %
+           (current_attempt, bans[net]['last_attempt'], RETRY_WINDOW))
+  if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
+    bans[net]['attempts'] = 0
+    logdebug("Ban counter for %s reset as window expired" % net)
+
+  bans[net]['attempts'] += 1
+  bans[net]['last_attempt'] = current_attempt
+  logdebug("%s attempts now %d" % (net, bans[net]['attempts']))
+
+  if bans[net]['attempts'] >= MAX_ATTEMPTS:
+    cur_time = int(round(time.time()))
+    NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
+    logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
+    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
+      with lock:
+        logdebug("Calling tables.banIPv4(%s)" % net)
+        tables.banIPv4(net)
+    elif int(f2boptions['manage_external']) != 1:
+      with lock:
+        logdebug("Calling tables.banIPv6(%s)" % net)
+        tables.banIPv6(net)
+
+    logdebug("Updating F2B_ACTIVE_BANS[%s]=%d" %
+              (net, cur_time + NET_BAN_TIME))
+    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
+  else:
+    logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
+      MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
+
+def unban(net):
+  global lock
+  logdebug("Calling unban() with net=%s" % net)
+  if not net in bans:
+    logger.logInfo(
+      '%s is not banned, skipping unban and deleting from queue (if any)' % net)
+    r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
+    return
+  logger.logInfo('Unbanning %s' % net)
+  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
+    with lock:
+      logdebug("Calling tables.unbanIPv4(%s)" % net)
+      tables.unbanIPv4(net)
+  else:
+    with lock:
+      logdebug("Calling tables.unbanIPv6(%s)" % net)
+      tables.unbanIPv6(net)
+  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
+  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
+  if net in bans:
+    logdebug("Unban for %s, setting attempts=0, ban_counter+=1" % net)
+    bans[net]['attempts'] = 0
+    bans[net]['ban_counter'] += 1
+
+def permBan(net, unban=False):
+  global f2boptions
+  global lock
+
+  is_unbanned = False
+  is_banned = False
+  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
+    with lock:
+      if unban:
+        is_unbanned = tables.unbanIPv4(net)
+      elif int(f2boptions['manage_external']) != 1:
+        is_banned = tables.banIPv4(net)
+  else:
+    with lock:
+      if unban:
+        is_unbanned = tables.unbanIPv6(net)
+      elif int(f2boptions['manage_external']) != 1:
+        is_banned = tables.banIPv6(net)
+
+
+  if is_unbanned:
+    r.hdel('F2B_PERM_BANS', '%s' % net)
+    logger.logCrit('Removed host/network %s from denylist' % net)
+  elif is_banned:
+    r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
+    logger.logCrit('Added host/network %s to denylist' % net)
+
+def clear():
+  global lock
+  logger.logInfo('Clearing all bans')
+  for net in bans.copy():
+    logdebug("Unbanning net: %s" % net)
+    unban(net)
+  with lock:
+    logdebug("Clearing IPv4/IPv6 table")
+    tables.clearIPv4Table()
+    tables.clearIPv6Table()
+    try:
+      if r is not None:
+        r.delete('F2B_ACTIVE_BANS')
+        r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
+
+def watch():
+  global pubsub
+  global quit_now
+  global exit_code
+
+  logger.logInfo('Watching Redis channel F2B_CHANNEL')
+  pubsub.subscribe('F2B_CHANNEL')
+
+  while not quit_now:
+    try:
+      for item in pubsub.listen():
+        refreshF2bregex()
+        for rule_id, rule_regex in f2bregex.items():
+          if item['data'] and item['type'] == 'message':
+            try:
+              result = re.search(rule_regex, item['data'])
+            except re.error:
+              result = False
+            if result:
+              addr = result.group(1)
+              ip = ipaddress.ip_address(addr)
+              if ip.is_private or ip.is_loopback:
+                continue
+              logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
+              ban(addr)
+    except Exception as ex:
+      logger.logWarn('Error reading log line from pubsub: %s' % ex)
+      pubsub = None
+      quit_now = True
+      exit_code = 2
+
+def snat4(snat_target):
+  global lock
+  global quit_now
+
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
+
+def snat6(snat_target):
+  global lock
+  global quit_now
+
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
+
+def autopurge():
+  global f2boptions
+  logdebug("autopurge thread started")
+  while not quit_now:
+    logdebug("autopurge tick")
+    time.sleep(10)
+    refreshF2boptions()
+    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
+    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
+    logdebug("QUEUE_UNBAN: %s" % QUEUE_UNBAN)
+    if QUEUE_UNBAN:
+      for net in QUEUE_UNBAN:
+        logdebug("Autopurge: unbanning queued net: %s" % net)
+        unban(str(net))
+    # Only check expiry for actively banned IPs:
+    active_bans = r.hgetall('F2B_ACTIVE_BANS')
+    now = time.time()
+    for net_str, expire_str in active_bans.items():
+      logdebug("Checking ban expiry for (actively banned): %s" % net_str)
+      # Defensive: always process if timer missing or expired
+      try:
+        expire = float(expire_str)
+      except Exception:
+        logdebug("Invalid expire time for %s; unbanning" % net_str)
+        unban(net_str)
+        continue
+      time_left = expire - now
+      logdebug("Time left for %s: %.1f seconds" % (net_str, time_left))
+      if time_left <= 0:
+        logdebug("Ban expired for %s" % net_str)
+        unban(net_str)
+
+def mailcowChainOrder():
+  global lock
+  global quit_now
+  global exit_code
+  while not quit_now:
+    time.sleep(10)
+    with lock:
+      quit_now, exit_code = tables.checkIPv4ChainOrder()
+      if quit_now: return
+      quit_now, exit_code = tables.checkIPv6ChainOrder()
+
+def calcNetBanTime(ban_counter):
+  global f2boptions
+
+  BAN_TIME = int(f2boptions['ban_time'])
+  MAX_BAN_TIME = int(f2boptions['max_ban_time'])
+  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
+  NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
+  NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
+  return NET_BAN_TIME
+
+def isIpNetwork(address):
+  try:
+    ipaddress.ip_network(address, False)
+  except ValueError:
+    return False
+  return True
+
+def genNetworkList(list):
+  resolver = dns.resolver.Resolver()
+  hostnames = []
+  networks = []
+  for key in list:
+    if isIpNetwork(key):
+      networks.append(key)
+    else:
+      hostnames.append(key)
+  for hostname in hostnames:
+    hostname_ips = []
+    for rdtype in ['A', 'AAAA']:
+      try:
+        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
+      except dns.exception.Timeout:
+        logger.logInfo('Hostname %s timedout on resolve' % hostname)
+        break
+      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+        continue
+      except dns.exception.DNSException as dnsexception:
+        logger.logInfo('%s' % dnsexception)
+        continue
+      for rdata in answer:
+        hostname_ips.append(rdata.to_text())
+    networks.extend(hostname_ips)
+  return set(networks)
+
+def whitelistUpdate():
+  global lock
+  global quit_now
+  global WHITELIST
+  while not quit_now:
+    start_time = time.time()
+    list = r.hgetall('F2B_WHITELIST')
+    new_whitelist = []
+    if list:
+      new_whitelist = genNetworkList(list)
+    with lock:
+      if Counter(new_whitelist) != Counter(WHITELIST):
+        WHITELIST = new_whitelist
+        logger.logInfo('Allowlist was changed, it has %s entries' % len(WHITELIST))
+    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
+
+def blacklistUpdate():
+  global quit_now
+  global BLACKLIST
+  while not quit_now:
+    start_time = time.time()
+    list = r.hgetall('F2B_BLACKLIST')
+    new_blacklist = []
+    if list:
+      new_blacklist = genNetworkList(list)
+    if Counter(new_blacklist) != Counter(BLACKLIST):
+      addban = set(new_blacklist).difference(BLACKLIST)
+      delban = set(BLACKLIST).difference(new_blacklist)
+      BLACKLIST = new_blacklist
+      logger.logInfo('Denylist was changed, it has %s entries' % len(BLACKLIST))
+      if addban:
+        for net in addban:
+          permBan(net=net)
+      if delban:
+        for net in delban:
+          permBan(net=net, unban=True)
+    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
+
+def sigterm_quit(signum, frame):
+  global clear_before_quit
+  logdebug("SIGTERM received, setting clear_before_quit to True and exiting")
+  clear_before_quit = True
+  sys.exit(exit_code)
+
+def before_quit():
+  logdebug("before_quit called, clear_before_quit=%s" % clear_before_quit)
+  if clear_before_quit:
+    clear()
+  if pubsub is not None:
+    pubsub.unsubscribe()
+
+if __name__ == '__main__':
+  logger = Logger()
+  logdebug("Sys.argv: %s" % sys.argv)
+  atexit.register(before_quit)
+  signal.signal(signal.SIGTERM, sigterm_quit)
+
+  backend = sys.argv[1]
+  logdebug("Backend: %s" % backend)
+  if backend == "nftables":
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    logger.logInfo('Using IPTables backend')
+    logger.logWarn(
+        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
+        "Please switch to nftables on your host to ensure complete compatibility."
+    )
+    time.sleep(5)
+    tables = IPTables(chain_name, logger)
+
+  clear()
+  logger.logInfo("Initializing mailcow netfilter chain")
+  tables.initChainIPv4()
+  tables.initChainIPv6()
+
+  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE", "").lower() in ("y", "yes"):
+    logger.logInfo(f"Skipping {chain_name} isolation")
+  else:
+    logger.logInfo(f"Setting {chain_name} isolation")
+    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
+
+  # connect to redis
+  while True:
+    try:
+      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
+      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
+      logdebug(
+        "Connecting redis (SLAVEOF_IP:%s, PORT:%s)" % (redis_slaveof_ip, redis_slaveof_port))
+      if "".__eq__(redis_slaveof_ip):
+        r = redis.StrictRedis(
+          host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
+      else:
+        r = redis.StrictRedis(
+          host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0, password=os.environ['REDISPASS'])
+      r.ping()
+      pubsub = r.pubsub()
+    except Exception as ex:
+      logdebug(
+        'Redis connection failed: %s - trying again in 3 seconds' % (ex))
+      time.sleep(3)
+    else:
+      break
+  logger.set_redis(r)
+  logdebug("Redis connection established, setting up F2B keys")
+
+  if r.exists('F2B_LOG'):
+    logdebug("Renaming F2B_LOG to NETFILTER_LOG")
+    r.rename('F2B_LOG', 'NETFILTER_LOG')
+  r.delete('F2B_ACTIVE_BANS')
+  r.delete('F2B_PERM_BANS')
+
+  refreshF2boptions()
+
+  watch_thread = Thread(target=watch)
+  watch_thread.daemon = True
+  watch_thread.start()
+
+  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
+    try:
+      snat_ip = os.getenv('SNAT_TO_SOURCE')
+      snat_ipo = ipaddress.ip_address(snat_ip)
+      if type(snat_ipo) is ipaddress.IPv4Address:
+        snat4_thread = Thread(target=snat4, args=(snat_ip,))
+        snat4_thread.daemon = True
+        snat4_thread.start()
+    except ValueError:
+      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
+
+  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
+    try:
+      snat_ip = os.getenv('SNAT6_TO_SOURCE')
+      snat_ipo = ipaddress.ip_address(snat_ip)
+      if type(snat_ipo) is ipaddress.IPv6Address:
+        snat6_thread = Thread(target=snat6,args=(snat_ip,))
+        snat6_thread.daemon = True
+        snat6_thread.start()
+    except ValueError:
+      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
+
+  autopurge_thread = Thread(target=autopurge)
+  autopurge_thread.daemon = True
+  autopurge_thread.start()
+
+  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
+  mailcowchainwatch_thread.daemon = True
+  mailcowchainwatch_thread.start()
+
+  blacklistupdate_thread = Thread(target=blacklistUpdate)
+  blacklistupdate_thread.daemon = True
+  blacklistupdate_thread.start()
+
+  whitelistupdate_thread = Thread(target=whitelistUpdate)
+  whitelistupdate_thread.daemon = True
+  whitelistupdate_thread.start()
+
+  while not quit_now:
+    time.sleep(0.5)
+
+  logdebug("Exiting with code %s" % exit_code)
+  sys.exit(exit_code)

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -1,236 +0,0 @@
-import ipaddress
-import time
-
-
-class BanManager:
-  """Failed attempts and automatic bans (Redis F2B_ACTIVE_BANS: network => expiry), the denylist (F2B_PERM_BANS)
-  and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
-  (chains, SNAT), because the backends must not run in parallel."""
-
-  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
-  EXCUSE_TIME = 60
-
-  def __init__(self, r, tables, options, logger, lock, clock=time.time):
-    self.r = r
-    self.tables = tables
-    self.options = options
-    self.logger = logger
-    self.lock = lock
-    self.clock = clock
-    self.counters = {}
-    self.banned = set()
-    self.denied = {}
-    self.allowlist = []
-    self.excused = {}
-
-  def excuse(self, ip):
-    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
-    with self.lock:
-      now = self.clock()
-      if self.excused.get(ip, 0) <= now:
-        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
-          ip, self.EXCUSE_TIME))
-      self.excused[ip] = now + self.EXCUSE_TIME
-
-  def attempt(self, ip):
-    """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
-    options = self.options.values
-    prefix = options['netban_ipv4'] if ip.version == 4 else options['netban_ipv6']
-    net = str(ipaddress.ip_network('%s/%d' % (ip, prefix), strict=False))
-    with self.lock:
-      now = self.clock()
-      allowed = self.allowed_by(ip)
-      if allowed is not None:
-        self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
-        return
-      if self.excused.get(ip, 0) > now:
-        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
-        return
-      if net in self.banned:
-        # log lines of connections from before the ban
-        return
-      counter = self.counter(net)
-      if now - counter['last_attempt'] > options['retry_window']:
-        counter['attempts'] = 0
-      counter['attempts'] += 1
-      counter['last_attempt'] = now
-      if counter['attempts'] < options['max_attempts']:
-        self.logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
-          options['max_attempts'] - counter['attempts'], options['retry_window'], net))
-        return
-
-      ban_time = self.options.ban_time(counter['ban_counter'])
-      self.logger.logCrit('Banning %s for %d minutes' % (net, ban_time / 60))
-      # Redis first: a firewall rule without expiry would never be lifted
-      self.r.hset('F2B_ACTIVE_BANS', net, int(round(now + ban_time)))
-      self.banned.add(net)
-      if options['manage_external'] != 1:
-        self.apply([net])
-
-  def unban(self, net):
-    """Lifts the automatic ban of a network; the rule stays while the denylist bans the same network"""
-    with self.lock:
-      normalized = self.normalize(net)
-      if normalized is None:
-        self.logger.logWarn('Removing invalid network %s from the active bans' % net)
-      else:
-        self.logger.logInfo('Unbanning %s' % net)
-        if normalized not in self.denied:
-          self.remove([normalized])
-      self.r.hdel('F2B_ACTIVE_BANS', net)
-      self.r.hdel('F2B_QUEUE_UNBAN', net)
-      if net in self.banned:
-        self.banned.discard(net)
-        counter = self.counter(net)
-        counter['attempts'] = 0
-        counter['ban_counter'] += 1
-
-  def unban_queued(self, entry):
-    """Unban request of the UI: a banned network, or a network that contains banned ones (allowlisting)"""
-    network = self.network(entry)
-    if network is None:
-      self.logger.logWarn('Removing invalid network %s from the unban queue' % entry)
-      self.r.hdel('F2B_QUEUE_UNBAN', entry)
-      return
-    with self.lock:
-      banned = set(self.r.hkeys('F2B_ACTIVE_BANS')) | self.banned
-      targets = [net for net in banned if self.overlaps(net, network)]
-      for net in targets:
-        self.unban(net)
-      if not targets and str(network) not in self.denied:
-        # a rule without known ban, e.g. left by an older netfilter
-        if self.remove([str(network)]):
-          self.logger.logInfo('Unbanning %s' % network)
-        else:
-          self.logger.logInfo('%s is not banned, removing it from the unban queue' % network)
-      self.r.hdel('F2B_QUEUE_UNBAN', entry)
-
-  def purge(self):
-    """Processes the unban queue of the UI and lifts expired bans"""
-    for entry in self.r.hkeys('F2B_QUEUE_UNBAN'):
-      self.unban_queued(entry)
-    now = self.clock()
-    for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
-      try:
-        expired = self.network(net) is None or float(expires) <= now
-      except (TypeError, ValueError):
-        expired = True
-      if expired:
-        self.unban(net)
-    with self.lock:
-      # bans whose Redis entry is gone, e.g. deleted by hand
-      for net in self.banned - set(self.r.hkeys('F2B_ACTIVE_BANS')):
-        self.unban(net)
-    self.forget_idle(now)
-
-  def restore(self):
-    """Applies the unexpired automatic bans of Redis again after a start of netfilter and drops the expired ones"""
-    now = self.clock()
-    restored = []
-    with self.lock:
-      for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
-        try:
-          valid = self.network(net) is not None and float(expires) > now
-        except (TypeError, ValueError):
-          valid = False
-        if not valid:
-          self.r.hdel('F2B_ACTIVE_BANS', net)
-          continue
-        self.banned.add(net)
-        self.counter(net)
-        restored.append(net)
-      if restored and self.options.values['manage_external'] != 1:
-        self.apply(restored)
-    if restored:
-      self.logger.logInfo('Restored %d active bans' % len(restored))
-
-  def reapply(self):
-    """Applies all bans again, e.g. after the MAILCOW chain had to be recreated"""
-    with self.lock:
-      if self.options.values['manage_external'] != 1:
-        self.apply(list(self.banned) + list(self.denied))
-
-  def set_denylist(self, entries):
-    """Bans the networks of the denylist permanently and lifts the bans of removed entries"""
-    networks = {}
-    for entry in entries:
-      net = self.normalize(entry)
-      if net is None:
-        self.logger.logWarn('Skipping invalid denylist entry %s' % entry)
-        continue
-      networks[net] = entry
-    with self.lock:
-      if networks.keys() == self.denied.keys():
-        return
-      self.logger.logInfo('Denylist was changed, it has %d entries' % len(networks))
-      added = [net for net in networks if net not in self.denied]
-      removed = [net for net in self.denied if net not in networks]
-      if added and self.options.values['manage_external'] != 1:
-        self.apply(added)
-      for net in added:
-        self.r.hset('F2B_PERM_BANS', networks[net], int(round(self.clock())))
-        self.logger.logCrit('Added host/network %s to denylist' % networks[net])
-      # an automatic ban of the same network keeps its rule
-      auto_banned = {self.normalize(net) for net in self.banned}
-      self.remove([net for net in removed if net not in auto_banned])
-      for net in removed:
-        self.r.hdel('F2B_PERM_BANS', self.denied[net])
-        self.logger.logCrit('Removed host/network %s from denylist' % self.denied[net])
-      self.denied = networks
-
-  def set_allowlist(self, entries):
-    networks = []
-    for entry in entries:
-      network = self.network(entry)
-      if network is None:
-        self.logger.logWarn('Skipping invalid allowlist entry %s' % entry)
-        continue
-      networks.append(network)
-    with self.lock:
-      if set(networks) != set(self.allowlist):
-        self.allowlist = networks
-        self.logger.logInfo('Allowlist was changed, it has %d entries' % len(networks))
-
-  def allowed_by(self, ip):
-    address = ipaddress.ip_network(ip)
-    for network in self.allowlist:
-      if network.version == address.version and network.overlaps(address):
-        return network
-    return None
-
-  def counter(self, net):
-    return self.counters.setdefault(net, {'attempts': 0, 'last_attempt': 0.0, 'ban_counter': 0})
-
-  def forget_idle(self, now):
-    """Drops counters of networks that were never banned and had no attempt within the retry window"""
-    window = self.options.values['retry_window']
-    with self.lock:
-      for net in [net for net, counter in self.counters.items()
-                  if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
-        del self.counters[net]
-      for ip in [ip for ip, until in self.excused.items() if until <= now]:
-        del self.excused[ip]
-
-  def apply(self, nets):
-    for net in self.tables.ban(nets):
-      self.logger.logWarn('Could not add the firewall rule for %s' % net)
-
-  def remove(self, nets):
-    return self.tables.unban(nets) if nets else []
-
-  @staticmethod
-  def network(entry):
-    try:
-      return ipaddress.ip_network(entry, strict=False)
-    except (TypeError, ValueError):
-      return None
-
-  @classmethod
-  def normalize(cls, entry):
-    network = cls.network(entry)
-    return str(network) if network is not None else None
-
-  @classmethod
-  def overlaps(cls, net, network):
-    other = cls.network(net)
-    return other is not None and other.version == network.version and other.overlaps(network)

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,7 +1,6 @@
 import iptc
 import time
-import ipaddress
-import contextlib
+import os
 
 class IPTables:
   def __init__(self, chain_name, logger):
@@ -9,10 +8,62 @@ class IPTables:
     self.logger = logger
 
   def initChainIPv4(self):
-    self.init_chain(iptc.Table(iptc.Table.FILTER))
+    if not iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name) in iptc.Table(iptc.Table.FILTER).chains:
+      iptc.Table(iptc.Table.FILTER).create_chain(self.chain_name)
+    for c in ['FORWARD', 'INPUT']:
+      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
+      rule = iptc.Rule()
+      rule.src = '0.0.0.0/0'
+      rule.dst = '0.0.0.0/0'
+      target = iptc.Target(rule, self.chain_name)
+      rule.target = target
+      if rule not in chain.rules:
+        chain.insert_rule(rule)
 
   def initChainIPv6(self):
-    self.init_chain(iptc.Table6(iptc.Table6.FILTER))
+    if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name) in iptc.Table6(iptc.Table6.FILTER).chains:
+      iptc.Table6(iptc.Table6.FILTER).create_chain(self.chain_name)
+    for c in ['FORWARD', 'INPUT']:
+      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
+      rule = iptc.Rule6()
+      rule.src = '::/0'
+      rule.dst = '::/0'
+      target = iptc.Target(rule, self.chain_name)
+      rule.target = target
+      if rule not in chain.rules:
+        chain.insert_rule(rule)
+
+  def checkIPv4ChainOrder(self):
+    filter_table = iptc.Table(iptc.Table.FILTER)
+    filter_table.refresh()
+    return self.checkChainOrder(filter_table)
+
+  def checkIPv6ChainOrder(self):
+    filter_table = iptc.Table6(iptc.Table6.FILTER)
+    filter_table.refresh()
+    return self.checkChainOrder(filter_table)
+
+  def checkChainOrder(self, filter_table):
+    err = False
+    exit_code = None
+
+    forward_chain = iptc.Chain(filter_table, 'FORWARD')
+    input_chain = iptc.Chain(filter_table, 'INPUT')
+    for chain in [forward_chain, input_chain]:
+      target_found = False
+      for position, item in enumerate(chain.rules):
+        if item.target.name == self.chain_name:
+          target_found = True
+          if position > 2:
+            self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
+            err = True
+            exit_code = 2
+      if not target_found:
+        self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
+        err = True
+        exit_code = 2
+
+    return err, exit_code
 
   def clearIPv4Table(self):
     self.clearTable(iptc.Table(iptc.Table.FILTER))
@@ -20,133 +71,68 @@ class IPTables:
   def clearIPv6Table(self):
     self.clearTable(iptc.Table6(iptc.Table6.FILTER))
 
-  @contextlib.contextmanager
-  def transaction(self, table):
-    # One commit for all changes, autocommit rewrites the whole table for every change
-    table.autocommit = False
-    table.refresh()
-    try:
-      yield
-      table.commit()
-    finally:
-      table.refresh()
-      table.autocommit = True
-
-  def new_rule(self, table):
-    return iptc.Rule6() if isinstance(table, iptc.Table6) else iptc.Rule()
-
-  def family(self, table):
-    return 'IPv6' if isinstance(table, iptc.Table6) else 'IPv4'
-
-  def init_chain(self, filter_table):
-    # Empty MAILCOW chain with one jump at the top of the input and forward chains
-    filter_table.refresh()
-    if filter_table.is_chain(self.chain_name):
-      filter_table.flush_entries(self.chain_name)
-    else:
-      filter_table.create_chain(self.chain_name)
-    self.ensure_jumps(filter_table)
-
-  def ensure_chains(self):
-    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
-    recreated = False
-    for filter_table in [iptc.Table(iptc.Table.FILTER), iptc.Table6(iptc.Table6.FILTER)]:
-      filter_table.refresh()
-      if not filter_table.is_chain(self.chain_name):
-        filter_table.create_chain(self.chain_name)
-        self.logger.logWarn(f'MAILCOW {self.family(filter_table)} chain was missing and has been recreated')
-        recreated = True
-      self.ensure_jumps(filter_table)
-    return recreated
-
-  def ensure_jumps(self, filter_table):
-    # Exactly one jump to MAILCOW as first rule: docker inserts its jump (DOCKER-USER) at the top
-    for name in ['INPUT', 'FORWARD']:
-      chain = iptc.Chain(filter_table, name)
-      positions = [position for position, rule in enumerate(chain.rules) if rule.target.name == self.chain_name]
-      if positions == [0]: continue
-
-      with self.transaction(filter_table):
-        for rule in [rule for rule in chain.rules if rule.target.name == self.chain_name]:
-          chain.delete_rule(rule)
-        rule = self.new_rule(filter_table)
-        rule.target = iptc.Target(rule, self.chain_name)
-        chain.insert_rule(rule)
-      if positions:
-        self.logger.logInfo(f'Moved MAILCOW jump to the top of the {self.family(filter_table)} {name} chain, it was at position {positions[0] + 1}')
-      else:
-        self.logger.logInfo(f'Added MAILCOW jump to the {self.family(filter_table)} {name} chain')
-
   def clearTable(self, filter_table):
-    try:
-      with self.transaction(filter_table):
-        if not filter_table.is_chain(self.chain_name):
-          return
-        for chain in filter_table.chains:
-          for rule in chain.rules:
-            if rule.target.name == self.chain_name:
-              chain.delete_rule(rule)
-        filter_table.flush_entries(self.chain_name)
-        filter_table.delete_chain(self.chain_name)
-      self.logger.logInfo(f'Clear completed: {self.family(filter_table)}')
-    except Exception as ex:
-      self.logger.logWarn(f'Error clearing the {self.family(filter_table)} MAILCOW chain: {ex}')
+    filter_table.autocommit = False
+    forward_chain = iptc.Chain(filter_table, "FORWARD")
+    input_chain = iptc.Chain(filter_table, "INPUT")
+    mailcow_chain = iptc.Chain(filter_table, self.chain_name)
+    if mailcow_chain in filter_table.chains:
+      for rule in mailcow_chain.rules:
+        mailcow_chain.delete_rule(rule)
+      for rule in forward_chain.rules:
+        if rule.target.name == self.chain_name:
+          forward_chain.delete_rule(rule)
+      for rule in input_chain.rules:
+        if rule.target.name == self.chain_name:
+          input_chain.delete_rule(rule)
+      filter_table.delete_chain(self.chain_name)
+    filter_table.commit()
+    filter_table.refresh()
+    filter_table.autocommit = True
 
-  def ban_rule_network(self, rule):
-    # Source network of a ban rule, None for every other rule
-    if rule.target.name not in ('REJECT', 'DROP') or rule.matches or rule.in_interface or rule.out_interface:
-      return None
-    try:
-      return ipaddress.ip_network(rule.src, strict=False)
-    except ValueError:
-      return None
+  def banIPv4(self, source):
+    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+    rule = iptc.Rule()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule in chain.rules:
+      return False
+    chain.insert_rule(rule)
+    return True
 
-  def split_tables(self, sources):
-    networks = {4: [], 6: []}
-    for source in sources:
-      network = ipaddress.ip_network(source, strict=False)
-      networks[network.version].append(network)
-    tables = [(iptc.Table(iptc.Table.FILTER), networks[4]), (iptc.Table6(iptc.Table6.FILTER), networks[6])]
-    return [(table, table_networks) for table, table_networks in tables if table_networks]
+  def banIPv6(self, source):
+    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
+    rule = iptc.Rule6()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule in chain.rules:
+      return False
+    chain.insert_rule(rule)
+    return True
 
-  def ban(self, sources):
-    # One REJECT rule per network, existing rules are kept. Returns the networks that could not be banned
-    failed = []
-    for filter_table, networks in self.split_tables(sources):
-      try:
-        with self.transaction(filter_table):
-          chain = iptc.Chain(filter_table, self.chain_name)
-          existing = {self.ban_rule_network(rule) for rule in chain.rules}
-          for network in dict.fromkeys(networks):
-            if network in existing: continue
-            rule = self.new_rule(filter_table)
-            rule.src = str(network)
-            rule.target = iptc.Target(rule, "REJECT")
-            chain.insert_rule(rule)
-      except Exception as ex:
-        self.logger.logWarn(f'iptables: {ex}')
-        failed += networks
-    return [str(network) for network in failed]
+  def unbanIPv4(self, source):
+    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+    rule = iptc.Rule()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule not in chain.rules: 
+      return False
+    chain.delete_rule(rule)
+    return True
 
-  def unban(self, sources):
-    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
-    removed = []
-    for filter_table, networks in self.split_tables(sources):
-      targets = set(networks)
-      try:
-        with self.transaction(filter_table):
-          chain = iptc.Chain(filter_table, self.chain_name)
-          found = set()
-          for rule in chain.rules:
-            network = self.ban_rule_network(rule)
-            if network in targets:
-              chain.delete_rule(rule)
-              found.add(network)
-      except Exception as ex:
-        self.logger.logWarn(f'iptables: {ex}')
-        continue
-      removed += found
-    return [str(network) for network in removed]
+  def unbanIPv6(self, source):
+    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
+    rule = iptc.Rule6()
+    rule.src = source
+    target = iptc.Target(rule, "REJECT")
+    rule.target = target
+    if rule not in chain.rules:
+      return False
+    chain.delete_rule(rule)
+    return True
 
   def snat4(self, snat_target, source):
     try:
@@ -202,7 +188,7 @@ class IPTables:
           if item == new_rule:
             if position != 0:
               chain.delete_rule(new_rule)
-
+    
       table.commit()
       table.autocommit = True
     except:
@@ -229,9 +215,7 @@ class IPTables:
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
     try:
-      table = iptc.Table(iptc.Table.FILTER)
-      table.refresh()
-      chain = iptc.Chain(table, self.chain_name)
+      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
 
       # insert mailcow isolation rule
       rule = iptc.Rule()
@@ -247,7 +231,7 @@ class IPTables:
       chain.insert_rule(rule, position=0)
 
       # insert mailcow isolation exception rule
-      if _allow:
+      if _allow != "":
         rule = iptc.Rule()
         rule.src = _allow
         rule.in_interface = f'!{_interface}'
@@ -265,4 +249,4 @@ class IPTables:
       return True
     except Exception as e:
       self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
-      return False
+      return False

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -1,9 +1,8 @@
 import nftables
 import ipaddress
+import os
 
 class NFTables:
-  ISOLATION_COMMENTS = ('mailcow isolation', 'mailcow isolation allow')
-
   def __init__(self, chain_name, logger):
     self.chain_name = chain_name
     self.logger = logger
@@ -13,15 +12,41 @@ class NFTables:
     self.nft.set_handle_output(True)
     self.nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
                             'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
-    self.mailcow_chains = set()
 
     self.search_current_chains()
 
   def initChainIPv4(self):
-    self.init_chain("ip")
+    self.insert_mailcow_chains("ip")
 
   def initChainIPv6(self):
-    self.init_chain("ip6")
+    self.insert_mailcow_chains("ip6")
+
+  def checkIPv4ChainOrder(self):
+    return self.checkChainOrder("ip")
+
+  def checkIPv6ChainOrder(self):
+    return self.checkChainOrder("ip6")
+
+  def checkChainOrder(self, filter_table):
+    err = False
+    exit_code = None
+
+    for chain in ['input', 'forward']:
+      chain_position = self.check_mailcow_chains(filter_table, chain)
+      if chain_position is None: continue
+
+      if chain_position is False:
+        self.logger.logCrit(f'MAILCOW target not found in {filter_table} {chain} table, restarting container to fix it...')
+        err = True
+        exit_code = 2
+
+      if chain_position > 0:
+        chain_position += 1
+        self.logger.logCrit(f'MAILCOW target is in position {chain_position} in the {filter_table} {chain} table, restarting container to fix it...')
+        err = True
+        exit_code = 2
+
+    return err, exit_code
 
   def clearIPv4Table(self):
     self.clearTable("ip")
@@ -29,19 +54,84 @@ class NFTables:
   def clearIPv6Table(self):
     self.clearTable("ip6")
 
+  def clearTable(self, _family):
+    is_empty_dict = True
+    json_command = self.get_base_dict()
+    chain_handle = self.get_chain_handle(_family, "filter", self.chain_name)
+    # if no handle, the chain doesn't exists
+    if chain_handle is not None:
+      is_empty_dict = False
+      # flush chain
+      mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+      flush_chain = {'flush': {'chain': mailcow_chain}}
+      json_command["nftables"].append(flush_chain)
+
+    # remove rule in forward chain
+    # remove rule in input chain
+    chains_family = [self.nft_chain_names[_family]['filter']['input'],
+                    self.nft_chain_names[_family]['filter']['forward'] ]
+
+    for chain_base in chains_family:
+      if not chain_base: continue
+
+      rules_handle = self.get_rules_handle(_family, "filter", chain_base)
+      if rules_handle is not None:
+        for r_handle in rules_handle:
+          is_empty_dict = False
+          mailcow_rule = {'family':_family,
+                          'table': 'filter',
+                          'chain': chain_base,
+                          'handle': r_handle }
+          delete_rules = {'delete': {'rule': mailcow_rule} }
+          json_command["nftables"].append(delete_rules)
+
+    # remove chain
+    # after delete all rules referencing this chain
+    if chain_handle is not None:
+      mc_chain_handle = {'family':_family,
+                        'table': 'filter',
+                        'name': self.chain_name,
+                        'handle': chain_handle }
+      delete_chain = {'delete': {'chain': mc_chain_handle} }
+      json_command["nftables"].append(delete_chain)
+
+    if is_empty_dict == False:
+      if self.nft_exec_dict(json_command):
+        self.logger.logInfo(f"Clear completed: {_family}")
+
+  def banIPv4(self, source):
+    ban_dict = self.get_ban_ip_dict(source, "ip")
+    return self.nft_exec_dict(ban_dict)
+
+  def banIPv6(self, source):
+    ban_dict = self.get_ban_ip_dict(source, "ip6")
+    return self.nft_exec_dict(ban_dict)
+
+  def unbanIPv4(self, source):
+    unban_dict = self.get_unban_ip_dict(source, "ip")
+    if not unban_dict:
+      return False
+    return self.nft_exec_dict(unban_dict)
+
+  def unbanIPv6(self, source):
+    unban_dict = self.get_unban_ip_dict(source, "ip6")
+    if not unban_dict:
+      return False
+    return self.nft_exec_dict(unban_dict)
+
   def snat4(self, snat_target, source):
     self.snat_rule("ip", snat_target, source)
 
   def snat6(self, snat_target, source):
     self.snat_rule("ip6", snat_target, source)
 
-  def nft_exec_dict(self, query: dict, quiet: bool = False):
+
+  def nft_exec_dict(self, query: dict):
     if not query: return False
 
     rc, output, error = self.nft.json_cmd(query)
     if rc != 0:
-      if not quiet:
-        self.logger.logWarn(f"nftables: {str(error).strip() or f'error {rc}'}")
+      #self.logger.logCrit(f"Nftables Error: {error}")
       return False
 
     # Prevent returning False or empty string on commands that do not produce output
@@ -54,165 +144,118 @@ class NFTables:
     return {'nftables': [{ 'metainfo': { 'json_schema_version': 1} } ] }
 
   def search_current_chains(self):
-    nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
-                       'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
     nft_chain_priority = {'ip': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} },
-                          'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
-    mailcow_chains = set()
+                      'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
 
     # Command: 'nft list chains'
     _list = {'list' : {'chains': 'null'} }
     command = self.get_base_dict()
     command['nftables'].append(_list)
     kernel_ruleset = self.nft_exec_dict(command)
-    if not kernel_ruleset:
-      return
+    if kernel_ruleset:
+      for _object in kernel_ruleset['nftables']:
+        chain = _object.get("chain")
+        if not chain: continue
 
-    for _object in kernel_ruleset['nftables']:
+        _family = chain['family']
+        _table = chain['table']
+        _hook = chain.get("hook")
+        _priority = chain.get("prio")
+        _name = chain['name']
+
+        if _family not in self.nft_chain_names: continue
+        if _table not in self.nft_chain_names[_family]: continue
+        if _hook not in self.nft_chain_names[_family][_table]: continue
+        if _priority is None: continue
+
+        _saved_priority = nft_chain_priority[_family][_table][_hook]
+        if _saved_priority is None or _priority < _saved_priority:
+          # at this point, we know the chain has:
+          # hook and priority set
+          # and it has the lowest priority
+          nft_chain_priority[_family][_table][_hook] = _priority
+          self.nft_chain_names[_family][_table][_hook] = _name
+
+  def search_for_chain(self, kernel_ruleset: dict, chain_name: str):
+    found = False
+    for _object in kernel_ruleset["nftables"]:
       chain = _object.get("chain")
-      if not chain: continue
-
-      _family = chain['family']
-      _table = chain['table']
-      _hook = chain.get("hook")
-      _priority = chain.get("prio")
-      _name = chain['name']
-
-      if _family not in nft_chain_names: continue
-      if _table not in nft_chain_names[_family]: continue
-      if _table == 'filter' and _name == self.chain_name:
-        mailcow_chains.add(_family)
-      if _hook not in nft_chain_names[_family][_table]: continue
-      if _priority is None: continue
-
-      _saved_priority = nft_chain_priority[_family][_table][_hook]
-      if _saved_priority is None or _priority < _saved_priority:
-        # at this point, we know the chain has:
-        # hook and priority set
-        # and it has the lowest priority
-        nft_chain_priority[_family][_table][_hook] = _priority
-        nft_chain_names[_family][_table][_hook] = _name
-
-    self.nft_chain_names = nft_chain_names
-    self.mailcow_chains = mailcow_chains
-
-  def has_filter_chains(self, _family: str):
-    # no base chains: the filter table does not exist, e.g. ip6 if docker has ipv6 disabled
-    return any(self.nft_chain_names[_family]['filter'].values())
-
-  def list_rules(self, _family: str, _table: str, _chain: str):
-    # Command: 'nft list chain {family} {table} {chain}', None if the chain does not exist
-    _list = {'list': {'chain': {'family': _family, 'table': _table, 'name': _chain}}}
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
-    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
-      return None
-    return [_object["rule"] for _object in kernel_ruleset["nftables"] if _object.get("rule")]
+      if not chain:
+        continue
+      ch_name = chain.get("name")
+      if ch_name == chain_name:
+        found = True
+        break
+    return found
 
   def get_chain_dict(self, _family: str, _name: str):
-    # nft (add | create) chain [<family>] <table> <name>
+    # nft (add | create) chain [<family>] <table> <name> 
     _chain_opts = {'family': _family, 'table': 'filter', 'name': _name  }
     _add = {'add': {'chain': _chain_opts} }
     final_chain = self.get_base_dict()
     final_chain["nftables"].append(_add)
     return final_chain
 
-  def get_mailcow_jump_rule(self, _family: str, _chain: str):
-    _expr_opt = [{'counter': {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}},
-                 {'jump': {'target': self.chain_name}}]
+  def get_mailcow_jump_rule_dict(self, _family: str, _chain: str):
+    _jump_rule = self.get_base_dict()
+    _expr_opt=[]
+    _expr_counter = {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}
+    _counter_dict = {'counter': _expr_counter}
+    _expr_opt.append(_counter_dict)
+
+    _jump_opts = {'jump': {'target': self.chain_name} }
+
+    _expr_opt.append(_jump_opts)
+
     _rule_params = {'family': _family,
                     'table': 'filter',
                     'chain': _chain,
                     'expr': _expr_opt,
                     'comment': "mailcow" }
-    return {'insert': {'rule': _rule_params} }
 
-  def is_mailcow_jump(self, rule: dict):
-    # by target, jumps restored by iptables-restore carry no nft comment
-    return any(isinstance(expr, dict) and isinstance(expr.get('jump'), dict) and expr['jump'].get('target') == self.chain_name
-               for expr in rule.get('expr', []))
+    _add_rule = {'insert': {'rule': _rule_params} }
 
-  def init_chain(self, _family: str):
-    # Empty MAILCOW chain with one jump at the top of the input and forward chains
-    if not self.has_filter_chains(_family): return
+    _jump_rule["nftables"].append(_add_rule)
 
-    if self.list_rules(_family, 'filter', self.chain_name) is None:
-      if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
-        self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
-    else:
-      flush_chain = {'flush': {'chain': {'family': _family, 'table': 'filter', 'name': self.chain_name}}}
-      command = self.get_base_dict()
-      command['nftables'].append(flush_chain)
-      self.nft_exec_dict(command)
+    return _jump_rule
 
-    self.ensure_jumps(_family)
-
-  def ensure_chains(self):
-    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
-    self.search_current_chains()
-    recreated = False
-    for _family in ['ip', 'ip6']:
-      if not self.has_filter_chains(_family): continue
-
-      if _family not in self.mailcow_chains:
-        if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
-          self.logger.logWarn(f"MAILCOW {_family} chain was missing and has been recreated")
-          recreated = True
-      self.ensure_jumps(_family)
-    return recreated
-
-  def ensure_jumps(self, _family: str):
-    # Exactly one jump to MAILCOW as first rule: docker inserts its jumps (DOCKER-USER, DOCKER-FORWARD) at the top
-    for hook in ['input', 'forward']:
-      base_chain = self.nft_chain_names[_family]['filter'][hook]
-      if not base_chain: continue
-
-      rules = self.list_rules(_family, 'filter', base_chain)
-      if rules is None: continue
-
-      positions = [position for position, rule in enumerate(rules) if self.is_mailcow_jump(rule)]
-      if positions == [0]: continue
-
-      # delete and insert in one transaction, the chain is never without jump
-      command = self.get_base_dict()
-      for position in positions:
-        _rule_opts = {'family': _family, 'table': 'filter', 'chain': base_chain, 'handle': rules[position]['handle']}
-        command['nftables'].append({'delete': {'rule': _rule_opts}})
-      command['nftables'].append(self.get_mailcow_jump_rule(_family, base_chain))
-      if self.nft_exec_dict(command):
-        if positions:
-          self.logger.logInfo(f"Moved MAILCOW jump to the top of the {_family} {base_chain} chain, it was at position {positions[0] + 1}")
-        else:
-          self.logger.logInfo(f"Added MAILCOW jump to the {_family} {base_chain} chain")
-
-  def clearTable(self, _family: str):
-    # Jumps first, the chain can only be deleted without references. Separate steps, so one failing step
-    # does not roll back the others
-    _list = {'list': {'table': {'family': _family, 'name': 'filter'}}}
+  def insert_mailcow_chains(self, _family: str):
+    nft_input_chain = self.nft_chain_names[_family]['filter']['input']
+    nft_forward_chain = self.nft_chain_names[_family]['filter']['forward']
+    # Command: 'nft list table <family> filter'
+    _table_opts = {'family': _family, 'name': 'filter'}
+    _list = {'list': {'table': _table_opts} }
     command = self.get_base_dict()
     command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
-    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
-      return
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      # chain
+      if not self.search_for_chain(kernel_ruleset, self.chain_name):
+        cadena = self.get_chain_dict(_family, self.chain_name)
+        if self.nft_exec_dict(cadena):
+          self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
 
-    delete_jumps = self.get_base_dict()
-    for _object in kernel_ruleset["nftables"]:
-      rule = _object.get("rule")
-      if rule and self.is_mailcow_jump(rule):
-        _rule_opts = {'family': _family, 'table': 'filter', 'chain': rule['chain'], 'handle': rule['handle']}
-        delete_jumps['nftables'].append({'delete': {'rule': _rule_opts}})
-    if len(delete_jumps['nftables']) > 1:
-      self.nft_exec_dict(delete_jumps)
+      input_jump_found, forward_jump_found = False, False
 
-    if self.list_rules(_family, 'filter', self.chain_name) is None:
-      return
-    mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-    command = self.get_base_dict()
-    command['nftables'].append({'flush': {'chain': mailcow_chain}})
-    command['nftables'].append({'delete': {'chain': mailcow_chain}})
-    if self.nft_exec_dict(command):
-      self.logger.logInfo(f"Clear completed: {_family}")
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+
+        rule = _object["rule"]
+        if nft_input_chain and rule["chain"] == nft_input_chain:
+          if rule.get("comment") and rule["comment"] == "mailcow":
+            input_jump_found = True
+        if nft_forward_chain and rule["chain"] == nft_forward_chain:
+          if rule.get("comment") and rule["comment"] == "mailcow":
+            forward_jump_found = True
+
+      if not input_jump_found:
+        command = self.get_mailcow_jump_rule_dict(_family, nft_input_chain)
+        self.nft_exec_dict(command)
+
+      if not forward_jump_found:
+        command = self.get_mailcow_jump_rule_dict(_family, nft_forward_chain)
+        self.nft_exec_dict(command)
 
   def delete_nat_rule(self, _family:str, _chain: str, _handle:str):
     delete_command = self.get_base_dict()
@@ -242,14 +285,23 @@ class NFTables:
     # no postrouting chain, may occur if docker has ipv6 disabled.
     if not chain_name: return
 
-    rules = self.list_rules(_family, 'nat', chain_name)
-    if rules is None:
+    # Command: nft list chain <family> nat <chain_name>
+    _chain_opts = {'family': _family, 'table': 'nat', 'name': chain_name}
+    _list = {'list':{'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if not kernel_ruleset:
       return
 
     rule_position = 0
     rule_handle = None
     rule_found = False
-    for rule in rules:
+    for _object in kernel_ruleset["nftables"]:
+      if not _object.get("rule"):
+        continue
+
+      rule = _object["rule"]
       if not rule.get("comment") or not rule["comment"] == "mailcow":
         rule_position +=1
         continue
@@ -262,28 +314,23 @@ class NFTables:
     target_net = ipaddress.ip_network(snat_target, strict=False)
 
     if rule_found:
-      try:
-        saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
-        saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
+      saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
+      saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
 
-        daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
-        daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
+      daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
+      daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
 
-        target_ip = rule["expr"][3]["snat"]["addr"]
+      target_ip = rule["expr"][3]["snat"]["addr"]
 
-        saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
-        daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
-        current_target_net = ipaddress.ip_network(target_ip, strict=False)
+      saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
+      daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
+      current_target_net = ipaddress.ip_network(target_ip, strict=False)
 
-        match = all((
-                  dest_net == saddr_net,
-                  dest_net == daddr_net,
-                  target_net == current_target_net
-                ))
-      except (KeyError, IndexError, TypeError, ValueError):
-        # rule of another format, replace it
-        saddr_net = dest_net
-        match = False
+      match = all((
+                dest_net == saddr_net,
+                dest_net == daddr_net,
+                target_net == current_target_net
+              ))
       try:
         if rule_position == 0:
           if not match:
@@ -330,13 +377,49 @@ class NFTables:
       except:
         self.logger.logCrit(f"Error running SNAT on {_family}, retrying...")
 
+  def get_chain_handle(self, _family: str, _table: str, chain_name: str):
+    chain_handle = None
+    # Command: 'nft list chains {family}'
+    _list = {'list': {'chains': {'family': _family} } }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("chain"):
+          continue
+        chain = _object["chain"]
+        if chain["family"] == _family and chain["table"] == _table and chain["name"] == chain_name:
+          chain_handle = chain["handle"]
+          break
+    return chain_handle
+
   def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
-    rules = self.list_rules(_family, _table, chain_name) or []
-    return [rule["handle"] for rule in rules if rule.get("comment") == _comment_filter]
+    rule_handle = []
+    # Command: 'nft list chain {family} {table} {chain_name}'
+    _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
+    _list = {'list': {'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+
+        rule = _object["rule"]
+        if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
+          if rule.get("comment") and rule["comment"] == _comment_filter:
+            rule_handle.append(rule["handle"])
+    return rule_handle
+
+  def get_ban_ip_dict(self, ipaddr: str, _family: str):
+    json_command = self.get_base_dict()
 
-  def get_ban_rule(self, network, _family: str):
     expr_opt = []
-    right_dict = {'prefix': {'addr': str(network.network_address), 'len': int(network.prefixlen) } }
+    ipaddr_net = ipaddress.ip_network(ipaddr, strict=False)
+    right_dict = {'prefix': {'addr': str(ipaddr_net.network_address), 'len': int(ipaddr_net.prefixlen) } }
 
     left_dict = {'payload': {'protocol': _family, 'field': 'saddr'} }
     match_dict = {'op': '==', 'left': left_dict, 'right': right_dict }
@@ -349,115 +432,228 @@ class NFTables:
 
     rule_dict = {'family': _family, 'table': "filter", 'chain': self.chain_name, 'expr': expr_opt}
 
-    return {'insert': {'rule': rule_dict} }
+    base_dict = {'insert': {'rule': rule_dict} }
+    json_command["nftables"].append(base_dict)
 
-  def ban_rule_network(self, rule: dict, _family: str):
-    # Source network of a ban rule (saddr match and drop), None for every other rule
-    if rule.get("comment") in self.ISOLATION_COMMENTS:
-      return None
-    exprs = [expr for expr in rule.get("expr", []) if isinstance(expr, dict)]
-    if not any('drop' in expr or 'reject' in expr for expr in exprs):
-      return None
-    for expr in exprs:
-      match = expr.get("match")
-      if not isinstance(match, dict) or match.get("op") != "==":
-        continue
-      left = match.get("left")
-      payload = left.get("payload") if isinstance(left, dict) else None
-      if not isinstance(payload, dict) or payload.get("protocol") != _family or payload.get("field") != "saddr":
-        continue
-      right = match.get("right")
-      try:
-        if isinstance(right, dict) and isinstance(right.get("prefix"), dict):
-          return ipaddress.ip_network(f'{right["prefix"]["addr"]}/{right["prefix"]["len"]}', strict=False)
-        if isinstance(right, str):
-          return ipaddress.ip_network(right, strict=False)
-      except (KeyError, ValueError):
-        return None
-    return None
+    return json_command
 
-  def split_families(self, sources):
-    families = {}
-    for source in sources:
-      network = ipaddress.ip_network(source, strict=False)
-      families.setdefault('ip' if network.version == 4 else 'ip6', []).append(network)
-    return families
+  def get_unban_ip_dict(self, ipaddr:str, _family: str):
+    json_command = self.get_base_dict()
+    # Command: 'nft list chain {s_family} filter  MAILCOW'
+    _chain_opts = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+    _list = {'list': {'chain': _chain_opts} }
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    rule_handle = None
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
 
-  def ban(self, sources):
-    # One drop rule per network, existing rules are kept. Returns the networks that could not be banned
-    failed = []
-    for _family, networks in self.split_families(sources).items():
-      rules = self.list_rules(_family, 'filter', self.chain_name)
-      if rules is None:
-        failed += networks
-        continue
-      existing = {self.ban_rule_network(rule, _family) for rule in rules}
-      missing = [network for network in dict.fromkeys(networks) if network not in existing]
-      if not missing: continue
+        rule = _object["rule"]["expr"][0]["match"]
+        if not "payload" in rule["left"]:
+          continue
+        left_opt = rule["left"]["payload"]
+        if not left_opt["protocol"] == _family:
+          continue
+        if not left_opt["field"] =="saddr":
+          continue
 
-      command = self.get_base_dict()
-      for network in missing:
-        command['nftables'].append(self.get_ban_rule(network, _family))
-      if not self.nft_exec_dict(command):
-        failed += missing
-    return [str(network) for network in failed]
+        # ip currently banned
+        rule_right = rule["right"]
+        if isinstance(rule_right, dict):
+          current_rule_ip = rule_right["prefix"]["addr"] + '/' + str(rule_right["prefix"]["len"])
+        else:
+          current_rule_ip = rule_right
+        current_rule_net = ipaddress.ip_network(current_rule_ip)
 
-  def unban(self, sources):
-    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
-    removed = []
-    for _family, networks in self.split_families(sources).items():
-      rules = self.list_rules(_family, 'filter', self.chain_name)
-      if not rules: continue
+        # ip to ban
+        candidate_net = ipaddress.ip_network(ipaddr, strict=False)
 
-      targets = set(networks)
-      command = self.get_base_dict()
-      found = set()
-      for rule in rules:
-        network = self.ban_rule_network(rule, _family)
-        if network in targets:
-          _rule_opts = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule['handle']}
-          command['nftables'].append({'delete': {'rule': _rule_opts}})
-          found.add(network)
-      if found and self.nft_exec_dict(command):
-        removed += found
-    return [str(network) for network in removed]
+        if current_rule_net == candidate_net:
+          rule_handle = _object["rule"]["handle"]
+          break
+
+      if rule_handle is not None:
+        mailcow_rule = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule_handle}
+        delete_rule = {'delete': {'rule': mailcow_rule} }
+        json_command["nftables"].append(delete_rule)
+      else:
+        return False
+
+    return json_command
+
+  def check_mailcow_chains(self, family: str, chain: str):
+    position = 0
+    rule_found = False
+    chain_name = self.nft_chain_names[family]['filter'][chain]
+
+    if not chain_name: return None
+
+    _chain_opts = {'family': family, 'table': 'filter', 'name': chain_name}
+    _list = {'list': {'chain': _chain_opts}}
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command)
+    if kernel_ruleset:
+      for _object in kernel_ruleset["nftables"]:
+        if not _object.get("rule"):
+          continue
+        rule = _object["rule"]
+        if rule.get("comment") and rule["comment"] == "mailcow":
+          rule_found = True
+          break
+
+        position+=1
+
+    return position if rule_found else False
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
-    # One rule per port: iptables-nft cannot show an anonymous set, a host that saves and restores the ruleset
-    # with iptables-save (e.g. netfilter-persistent) would get a drop for all ports
-    for comment in self.ISOLATION_COMMENTS:
-      for handle in self.get_rules_handle("ip", "filter", self.chain_name, comment):
-        self.delete_filter_rule("ip", self.chain_name, handle)
+    family = "ip"
+    table = "filter"
+    comment_filter_drop = "mailcow isolation"
+    comment_filter_allow = "mailcow isolation allow"
+    json_command = self.get_base_dict()
 
-    if not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[0], {"drop": None})):
+    # Delete old mailcow isolation rules
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
+    for handle in handles:
+      self.delete_filter_rule(family, self.chain_name, handle)
+
+    # insert mailcow isolation rule
+    _match_dict_drop = [
+      {
+        "match": {
+          "op": "!=",
+          "left": {
+            "meta": {
+              "key": "iifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "meta": {
+              "key": "oifname"
+            }
+          },
+          "right": _interface
+        }
+      },
+      {
+        "match": {
+          "op": "==",
+          "left": {
+            "payload": {
+              "protocol": "tcp",
+              "field": "dport"
+            }
+          },
+          "right": {
+            "set": _dports
+          }
+        }
+      },
+      {
+        "counter": {
+          "packets": 0,
+          "bytes": 0
+        }
+      },
+      {
+        "drop": None
+      }
+    ]
+    rule_drop = { "insert": { "rule": {
+      "family": family,
+      "table": table,
+      "chain": self.chain_name,
+      "comment": comment_filter_drop,
+      "expr": _match_dict_drop
+    }}}
+    json_command["nftables"].append(rule_drop)
+
+    # insert mailcow isolation allow rule
+    if _allow != "":
+      _match_dict_allow = [
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "ip",
+                "field": "saddr"
+              }
+            },
+            "right": _allow
+          }
+        },
+        {
+          "match": {
+            "op": "!=",
+            "left": {
+              "meta": {
+                "key": "iifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "meta": {
+                "key": "oifname"
+              }
+            },
+            "right": _interface
+          }
+        },
+        {
+          "match": {
+            "op": "==",
+            "left": {
+              "payload": {
+                "protocol": "tcp",
+                "field": "dport"
+              }
+            },
+            "right": {
+              "set": _dports
+            }
+          }
+        },
+        {
+          "counter": {
+            "packets": 0,
+            "bytes": 0
+          }
+        },
+        {
+          "accept": None
+        }
+      ]
+      rule_allow = { "insert": { "rule": {
+        "family": family,
+        "table": table,
+        "chain": self.chain_name,
+        "comment": comment_filter_allow,
+        "expr": _match_dict_allow
+      }}}
+      json_command["nftables"].append(rule_allow)
+
+    success = self.nft_exec_dict(json_command)
+    if success == False:
       self.logger.logCrit(f"Error adding {self.chain_name} isolation")
       return False
 
-    # separate, an invalid address must not prevent the drop rules. Inserted last, so they are above them
-    if _allow and not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[1], {"accept": None}, _allow)):
-      self.logger.logCrit(f"Error adding {self.chain_name} isolation exception for {_allow}")
-      return False
-
-    return True
-
-  def get_isolation_dict(self, _interface:str, _dports:list, _comment:str, _verdict:dict, _saddr:str = ""):
-    json_command = self.get_base_dict()
-    for port in _dports:
-      expr = []
-      if _saddr:
-        expr.append({"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": _saddr}})
-      expr += [
-        {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": _interface}},
-        {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": _interface}},
-        {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": int(port)}},
-        {"counter": {"packets": 0, "bytes": 0}},
-        _verdict
-      ]
-      json_command["nftables"].append({"insert": {"rule": {
-        "family": "ip",
-        "table": "filter",
-        "chain": self.chain_name,
-        "comment": _comment,
-        "expr": expr
-      }}})
-    return json_command
+    return True

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -1,200 +0,0 @@
-import ipaddress
-import threading
-import dns.exception
-import dns.resolver
-from modules.BanManager import BanManager
-from modules.Options import Options
-from modules.Regexes import Regexes
-
-
-class Netfilter:
-  """Runs the workers of netfilter. Every worker is a loop that logs errors and goes on, so neither a broken log
-  line nor a Redis restart can stop the expiry of bans."""
-
-  ISOLATION_INTERFACE = 'br-mailcow'
-  ISOLATION_PORTS = [3306, 6379, 8983, 12345]
-
-  def __init__(self, r, tables, logger, env):
-    self.r = r
-    self.tables = tables
-    self.logger = logger
-    self.env = env
-    # reentrant, bans.reapply() runs within the chain check
-    self.lock = threading.RLock()
-    self.stop = threading.Event()
-    self.options = Options(logger)
-    self.regexes = Regexes(logger)
-    self.bans = BanManager(r, tables, self.options, logger, self.lock)
-    self.snat4_target = self.snat_target('SNAT_TO_SOURCE', 4)
-    self.snat6_target = self.snat_target('SNAT6_TO_SOURCE', 6)
-    self.resolved = {}
-    self.workers = {
-      'watch': (self.watch, 3),
-      'purge': (self.purge, 10),
-      'chains': (self.check_chains, 10),
-      'lists': (self.update_lists, 60),
-    }
-    if self.snat4_target or self.snat6_target:
-      self.workers['snat'] = (self.snat, 10)
-    self.threads = {}
-
-  def setup(self):
-    if self.r.exists('F2B_LOG'):
-      self.r.rename('F2B_LOG', 'NETFILTER_LOG')
-    # filled again by the lists worker
-    self.r.delete('F2B_PERM_BANS')
-    self.options.refresh(self.r)
-    self.regexes.refresh(self.r)
-    with self.lock:
-      self.tables.clearIPv4Table()
-      self.tables.clearIPv6Table()
-      self.logger.logInfo('Initializing mailcow netfilter chain')
-      self.tables.initChainIPv4()
-      self.tables.initChainIPv6()
-      self.isolate()
-    self.bans.restore()
-
-  def run(self):
-    for name in self.workers:
-      self.start(name)
-    while not self.stop.wait(5):
-      for name, thread in list(self.threads.items()):
-        if not thread.is_alive():
-          self.logger.logCrit('Worker %s stopped unexpectedly, restarting it' % name)
-          self.start(name)
-
-  def shutdown(self):
-    """Removes the firewall rules, the active bans stay in Redis and are restored on the next start"""
-    self.stop.set()
-    for thread in self.threads.values():
-      thread.join(timeout=5)
-    with self.lock:
-      self.logger.logInfo('Clearing all bans')
-      self.tables.clearIPv4Table()
-      self.tables.clearIPv6Table()
-    try:
-      self.r.delete('F2B_PERM_BANS')
-    except Exception as ex:
-      self.logger.logWarn('Error clearing redis key F2B_PERM_BANS: %s' % ex)
-
-  def start(self, name):
-    task, interval = self.workers[name]
-    thread = threading.Thread(target=self.loop, args=(name, task, interval), name=name, daemon=True)
-    self.threads[name] = thread
-    thread.start()
-
-  def loop(self, name, task, interval):
-    while not self.stop.is_set():
-      try:
-        task()
-      except Exception as ex:
-        self.logger.logWarn('Error in worker %s: %s' % (name, ex))
-      self.stop.wait(interval)
-
-  def watch(self):
-    pubsub = self.r.pubsub(ignore_subscribe_messages=True)
-    try:
-      pubsub.subscribe('F2B_CHANNEL')
-      self.logger.logInfo('Watching Redis channel F2B_CHANNEL')
-      while not self.stop.is_set():
-        message = pubsub.get_message(timeout=1.0)
-        if message and message['type'] == 'message' and message['data']:
-          self.handle_line(message['data'])
-    finally:
-      try:
-        pubsub.close()
-      except Exception:
-        pass
-
-  def handle_line(self, line):
-    # a broken line must not end the watcher
-    try:
-      excused = Regexes.temporary_failure(line)
-      if excused is not None:
-        ip = Regexes.parse_ip(excused)
-        if self.external(ip):
-          self.bans.excuse(ip)
-        return
-      result = self.regexes.match(line)
-      if result is None:
-        return
-      rule_id, address = result
-      ip = Regexes.parse_ip(address)
-      if ip is None:
-        self.logger.logWarn('Rule id %s matched an invalid address %s (%s)' % (rule_id, address, line))
-        return
-      if not self.external(ip):
-        return
-      self.logger.logWarn('%s matched rule id %s (%s)' % (ip, rule_id, line))
-      self.bans.attempt(ip)
-    except Exception as ex:
-      self.logger.logWarn('Error processing log line %r: %s' % (line, ex))
-
-  @staticmethod
-  def external(ip):
-    # private and loopback addresses are never banned
-    return ip is not None and not (ip.is_private or ip.is_loopback)
-
-  def purge(self):
-    self.options.refresh(self.r)
-    self.regexes.refresh(self.r)
-    self.bans.purge()
-
-  def check_chains(self):
-    with self.lock:
-      if self.tables.ensure_chains():
-        self.isolate()
-        self.bans.reapply()
-
-  def snat(self):
-    with self.lock:
-      if self.snat4_target:
-        self.tables.snat4(self.snat4_target, self.env.get('IPV4_NETWORK', '172.22.1') + '.0/24')
-      if self.snat6_target:
-        self.tables.snat6(self.snat6_target, self.env.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
-
-  def update_lists(self):
-    self.bans.set_allowlist(self.resolve(self.r.hkeys('F2B_WHITELIST')))
-    self.bans.set_denylist(self.resolve(self.r.hkeys('F2B_BLACKLIST')))
-
-  def isolate(self):
-    if self.env.get('DISABLE_NETFILTER_ISOLATION_RULE', '').lower() in ('y', 'yes'):
-      self.logger.logInfo('Skipping MAILCOW isolation')
-      return
-    self.logger.logInfo('Setting MAILCOW isolation')
-    self.tables.create_mailcow_isolation_rule(self.ISOLATION_INTERFACE, self.ISOLATION_PORTS, self.env.get('MAILCOW_REPLICA_IP', ''))
-
-  def resolve(self, entries):
-    """Networks of the entries; hostnames are resolved, on DNS errors to their last known addresses"""
-    resolver = dns.resolver.Resolver()
-    networks = set()
-    for entry in entries:
-      if BanManager.network(entry) is not None:
-        networks.add(entry)
-        continue
-      addresses = set()
-      for rdtype in ['A', 'AAAA']:
-        try:
-          answer = resolver.resolve(qname=entry, rdtype=rdtype, lifetime=3)
-        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-          continue
-        except dns.exception.DNSException as ex:
-          self.logger.logInfo('Could not resolve %s: %s' % (entry, ex))
-          addresses = self.resolved.get(entry, set())
-          break
-        addresses.update(rdata.to_text() for rdata in answer)
-      self.resolved[entry] = addresses
-      networks |= addresses
-    return networks
-
-  def snat_target(self, name, version):
-    value = self.env.get(name, '')
-    if not value or value == 'n':
-      return None
-    try:
-      if ipaddress.ip_address(value).version == version:
-        return value
-    except ValueError:
-      pass
-    self.logger.logWarn('%s is not a valid IPv%d address' % (value, version))
-    return None

data/Dockerfiles/netfilter/modules/Options.py

@@ -1,120 +0,0 @@
-import json
-import uuid
-
-
-class Options:
-  """Fail2ban options of the mailcow UI (Redis F2B_OPTIONS). The values are replaced as a whole and never changed
-  in place, so a thread that took them (options.values) keeps a consistent set."""
-
-  DEFAULTS = {
-    'ban_time': 1800,
-    'max_ban_time': 10000,
-    'ban_time_increment': True,
-    'max_attempts': 10,
-    'retry_window': 600,
-    'netban_ipv4': 32,
-    'netban_ipv6': 128,
-    'manage_external': 0,
-  }
-  # Single keys of old installations, read while F2B_OPTIONS does not exist
-  LEGACY_KEYS = {
-    'ban_time': 'F2B_BAN_TIME',
-    'max_ban_time': 'F2B_MAX_BAN_TIME',
-    'ban_time_increment': 'F2B_BAN_TIME_INCREMENT',
-    'max_attempts': 'F2B_MAX_ATTEMPTS',
-    'retry_window': 'F2B_RETRY_WINDOW',
-    'netban_ipv4': 'F2B_NETBAN_IPV4',
-    'netban_ipv6': 'F2B_NETBAN_IPV6',
-  }
-
-  def __init__(self, logger):
-    self.logger = logger
-    self.values = self.normalize({})
-
-  def refresh(self, r):
-    """Reads the options of the UI; options it lacks are added to Redis once, existing ones are never overwritten"""
-    raw = r.get('F2B_OPTIONS')
-    if raw:
-      try:
-        stored = json.loads(raw)
-      except ValueError as ex:
-        self.logger.logWarn('F2B_OPTIONS is no valid JSON, keeping the current options: %s' % ex)
-        return
-      if not isinstance(stored, dict):
-        self.logger.logWarn('F2B_OPTIONS is no JSON object, keeping the current options')
-        return
-    else:
-      stored = {key: r.get(name) for key, name in self.LEGACY_KEYS.items()}
-      stored = {key: value for key, value in stored.items() if value is not None}
-
-    values = self.normalize(stored)
-    self.values = values
-    # First start or old installation: store the complete set, else only what the UI has not saved yet
-    missing = {key: value for key, value in values.items() if not raw or stored.get(key) is None}
-    if missing:
-      self.add_missing(r, missing)
-
-  def ban_time(self, ban_counter):
-    """Seconds of the next ban of a network that was banned ban_counter times before"""
-    values = self.values
-    ban_time = values['ban_time']
-    if values['ban_time_increment']:
-      # the exponent is capped, max_ban_time limits the result anyway
-      ban_time = ban_time * 2 ** min(ban_counter, 32)
-    return max(values['ban_time'], min(ban_time, values['max_ban_time']))
-
-  @classmethod
-  def normalize(cls, stored):
-    values = dict(stored)
-    for key, default in cls.DEFAULTS.items():
-      if isinstance(default, bool):
-        values[key] = cls.to_bool(stored.get(key), default)
-      else:
-        values[key] = cls.to_int(stored.get(key), default)
-    values['ban_time'] = max(1, values['ban_time'])
-    values['max_ban_time'] = max(1, values['max_ban_time'])
-    values['max_attempts'] = max(1, values['max_attempts'])
-    values['retry_window'] = max(1, values['retry_window'])
-    # A short prefix would ban whole regions or everything (/0)
-    values['netban_ipv4'] = min(32, max(8, values['netban_ipv4']))
-    values['netban_ipv6'] = min(128, max(8, values['netban_ipv6']))
-    values['manage_external'] = 1 if values['manage_external'] == 1 else 0
-    if not values.get('banlist_id'):
-      values['banlist_id'] = str(uuid.uuid4())
-    return values
-
-  @staticmethod
-  def to_int(value, default):
-    try:
-      return int(value)
-    except (TypeError, ValueError):
-      return default
-
-  @staticmethod
-  def to_bool(value, default):
-    if isinstance(value, bool):
-      return value
-    if isinstance(value, (int, float)):
-      return value != 0
-    if isinstance(value, str):
-      return value.strip().lower() in ('1', 'true', 'yes', 'on')
-    return default
-
-  def add_missing(self, r, missing):
-    # WATCH keeps a save of the UI between reading and writing
-    def fill(pipe):
-      raw = pipe.get('F2B_OPTIONS')
-      current = json.loads(raw) if raw else {}
-      if not isinstance(current, dict):
-        return
-      added = {key: value for key, value in missing.items() if current.get(key) is None}
-      if not added:
-        return
-      current.update(added)
-      pipe.multi()
-      pipe.set('F2B_OPTIONS', json.dumps(current, ensure_ascii=False))
-
-    try:
-      r.transaction(fill, 'F2B_OPTIONS')
-    except Exception as ex:
-      self.logger.logWarn('Could not add default options to F2B_OPTIONS: %s' % ex)

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -1,88 +0,0 @@
-import ipaddress
-import json
-import re
-
-
-class Regexes:
-  """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
-
-  DEFAULTS = {
-    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
-    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
-    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
-    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
-    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
-    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
-    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
-    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
-    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
-  }
-  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
-  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
-  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
-
-  def __init__(self, logger):
-    self.logger = logger
-    self.rules = []
-    self.source = None
-
-  def refresh(self, r):
-    """Compiles the rules of the UI again when they changed; invalid rules are skipped"""
-    raw = r.get('F2B_REGEX')
-    if not raw:
-      raw = json.dumps(self.DEFAULTS, ensure_ascii=False)
-      r.set('F2B_REGEX', raw, nx=True)
-    if raw == self.source:
-      return
-    try:
-      stored = json.loads(raw)
-    except ValueError as ex:
-      self.logger.logWarn('F2B_REGEX is no valid JSON, keeping the current rules: %s' % ex)
-      return
-    if not isinstance(stored, dict):
-      self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
-      return
-
-    rules = []
-    for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
-      try:
-        rules.append((str(rule_id), re.compile(regex)))
-      except (re.error, TypeError) as ex:
-        self.logger.logWarn('Skipping regex of rule id %s: %s' % (rule_id, ex))
-    self.rules = rules
-    self.source = raw
-
-  def match(self, line):
-    """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
-    for rule_id, regex in self.rules:
-      result = regex.search(line)
-      if result and result.lastindex and result.group(1):
-        return rule_id, result.group(1)
-    return None
-
-  @classmethod
-  def temporary_failure(cls, line):
-    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
-    result = cls.TEMPORARY_FAILURE.search(line)
-    return result.group(1) if result else None
-
-  @staticmethod
-  def parse_ip(text):
-    """IP address of a captured address, IPv4-mapped IPv6 as IPv4; a trailing colon (e.g. "1.2.3.4:") is
-    tolerated. None if the text is no address."""
-    for candidate in (text, text.rstrip(':')):
-      try:
-        ip = ipaddress.ip_address(candidate)
-      except ValueError:
-        continue
-      if ip.version == 6 and ip.ipv4_mapped:
-        return ip.ipv4_mapped
-      return ip
-    return None
-
-  @staticmethod
-  def sort_key(rule_id):
-    try:
-      return (0, int(rule_id))
-    except (TypeError, ValueError):
-      return (1, str(rule_id))

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+function array_by_comma { local IFS=","; echo "$*"; }
+
 # Wait for containers
 while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for SQL..."
@@ -130,6 +132,33 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   done
   fi
 
+  # Set API options if env vars are not empty
+  if [[ ${API_ALLOW_FROM} != "invalid" ]] && [[ ! -z ${API_ALLOW_FROM} ]]; then
+    IFS=',' read -r -a API_ALLOW_FROM_ARR <<< "${API_ALLOW_FROM}"
+    declare -a VALIDATED_API_ALLOW_FROM_ARR
+    REGEX_IP6='^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$'
+    REGEX_IP4='^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/([0-9]|[1-2][0-9]|3[0-2]))?$'
+    for IP in "${API_ALLOW_FROM_ARR[@]}"; do
+      if [[ ${IP} =~ ${REGEX_IP6} ]] || [[ ${IP} =~ ${REGEX_IP4} ]]; then
+        VALIDATED_API_ALLOW_FROM_ARR+=("${IP}")
+      fi
+    done
+    VALIDATED_IPS=$(array_by_comma ${VALIDATED_API_ALLOW_FROM_ARR[*]})
+    if [[ ! -z ${VALIDATED_IPS} ]]; then
+      if [[ ${API_KEY} != "invalid" ]] && [[ ! -z ${API_KEY} ]]; then
+        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
+DELETE FROM api WHERE access = 'rw';
+INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY}", "1", "${VALIDATED_IPS}", "rw");
+EOF
+      fi
+      if [[ ${API_KEY_READ_ONLY} != "invalid" ]] && [[ ! -z ${API_KEY_READ_ONLY} ]]; then
+        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
+DELETE FROM api WHERE access = 'ro';
+INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY_READ_ONLY}", "1", "${VALIDATED_IPS}", "ro");
+EOF
+      fi
+    fi
+  fi
 
   # Create events (master only, STATUS for event on slave will be SLAVESIDE_DISABLED)
   mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -13,11 +13,13 @@ do
   sleep 3
 done
 
-# Wait for updated schema (DB_VERSION from docker-compose.yml)
+# Wait for updated schema
 DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
+while [[ "${DBV_NOW}" != "${DBV_NEW}" ]]; do
   echo "Waiting for schema update..."
   DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+  DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
   sleep 5
 done
 echo "DB schema is ${DBV_NOW}"

data/conf/dovecot/auth/mailcowauth.php

@@ -101,16 +101,7 @@ if ($result === false){
   // Init Identity Provider
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
-  try {
-    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
-  } catch (\Mailcow\Auth\IdpUnavailable $e) {
-    // temporary failure for Dovecot, no failed login
-    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
-    http_response_code(503); // Service Unavailable
-    echo json_encode($return);
-    session_destroy();
-    exit;
-  }
+  $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
   if ($result) {
     error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
     set_sasl_log($post['username'], $post['real_rip'], $post['service']);

data/conf/dovecot/auth/passwd-verify.lua

@@ -41,12 +41,6 @@ function auth_password_verify(request, password)
     return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unreachable"
   end
 
-  -- 5xx: the backend or the identity provider cannot answer, a temporary failure and no failed login
-  if c >= 500 then
-    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
-    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unavailable"
-  end
-
   if c ~= 200 and c ~= 401 then
     dovecot.i_info("HTTP request failed with " .. tostring(c) .. " for user " .. request.user)
     return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Upstream error"

data/conf/dovecot/dovecot.conf

@@ -67,11 +67,9 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
-# cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
-# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
 passdb {
   driver = lua
-  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes
 }
 # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
 service doveadm {

data/conf/phpfpm/crons/group-grants.php

@@ -1,65 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-  PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-} catch (PDOException $e) {
-  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  } else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-} catch (Exception $e) {
-  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-function logMsg($priority, $message, $task = "Group Shares") {
-  global $redis;
-  $redis->lPush('CRON_LOG', json_encode(array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message,
-  )));
-}
-
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
-
-// One run at a time, released when the process ends
-$lock = fopen('/tmp/group-grants.lock', 'c');
-if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
-  exit(0);
-}
-
-$sharing = \Mailcow\MailboxGroup\MailboxGroupModule::create($pdo, \Mailcow\Core\Context::system())->sharing;
-
-// Every minute: subscriptions of changed groups; failed ones are retried and logged by the full run
-if ((int)date('i') % 15 !== 0) {
-  $sharing->syncSubscriptions();
-  exit(0);
-}
-
-// Every 15 minutes: IMAP ACLs on new top-level folders and failed grants, SOGo grants of owners that opened SOGo meanwhile
-foreach ($sharing->reconcile() as $warning) {
-  logMsg("warning", $warning[0] . ": " . html_entity_decode(implode(", ", array_slice($warning, 1))));
-}

data/conf/phpfpm/crons/identity-sync.php

@@ -1,95 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-  PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-} catch (PDOException $e) {
-  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  } else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-} catch (Exception $e) {
-  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-function logMsg($priority, $message, $task = "Identity Sync") {
-  global $redis;
-  $redis->lPush('CRON_LOG', json_encode(array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message,
-  )));
-}
-
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-// The legacy mailbox functions run as admin
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-// One run at a time, released when the process ends
-$lock = fopen('/tmp/identity-sync.lock', 'c');
-if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
-  exit(0);
-}
-
-// Users, offboarding and groups of Keycloak or LDAP; requested runs first, else every sync_interval minutes.
-// The IdP is only contacted when a run is due.
-$iam_settings = identity_provider('get');
-$identity = \Mailcow\Identity\IdentityModule::create($pdo, \Mailcow\Core\Context::system(), $iam_settings, function () {
-  return identity_provider('init');
-});
-$run = $identity->sync->runDue();
-if ($run !== null) {
-  $summary = $run['summary'];
-  if ($run['status'] == 'error') {
-    logMsg("err", ucfirst($run['mode']) . " failed: " . $summary['error']);
-  } else {
-    $counts = array();
-    foreach ($summary['counts'] as $key => $count) {
-      if ($count > 0) $counts[] = $key . "=" . $count;
-    }
-    logMsg(($run['status'] == 'blocked') ? "warning" : "info", ucfirst($run['mode']) . " " . $run['status'] . ": " . (empty($counts) ? "no changes" : implode(", ", $counts)));
-    foreach ($summary['details']['conflicts'] as $conflict) {
-      logMsg("warning", $conflict);
-    }
-    foreach ($summary['errors'] ?? array() as $error) {
-      logMsg("err", $error);
-    }
-  }
-}
-session_destroy();

data/conf/phpfpm/crons/imapsync-oauth-refresh.php

@@ -1,73 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-  PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-} catch (PDOException $e) {
-  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  } else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-} catch (Exception $e) {
-  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
-  exit(1);
-}
-
-function logMsg($priority, $message, $task = "imapsync OAuth refresh") {
-  global $redis;
-  $redis->lPush('CRON_LOG', json_encode(array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message,
-  )));
-}
-
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-
-// File-lock to prevent concurrent runs
-$lock_file = '/tmp/imapsync-oauth-refresh.lock';
-if (file_exists($lock_file)) {
-  $pid = (int)trim(@file_get_contents($lock_file));
-  if ($pid > 0 && posix_kill($pid, 0)) {
-    logMsg("info", "Previous refresh still running (pid=$pid), exiting");
-    exit(0);
-  }
-  @unlink($lock_file);
-}
-file_put_contents($lock_file, getmypid());
-
-try {
-  // client_credentials source tokens, unused per-user tokens (1 day grace), authorization_code per-user tokens;
-  // failing tokens are retried 15 minutes after their last attempt (C2), errors are sanitized
-  $imapsync = \Mailcow\Imapsync\ImapsyncModule::create($pdo, \Mailcow\Core\Context::system(), [
-    'hostname' => (string)getenv('MAILCOW_HOSTNAME'),
-  ]);
-  foreach ($imapsync->oauth->refreshDue() as $e) {
-    logMsg($e['priority'], $e['message']);
-  }
-} finally {
-  @unlink($lock_file);
-}

data/conf/phpfpm/crons/keycloak-sync.php

@@ -0,0 +1,231 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+// Init database
+//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  logMsg("err", $e->getMessage());
+  session_destroy();
+  exit;
+}
+
+// Init Redis
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  }
+  else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+}
+catch (Exception $e) {
+  echo "Exiting: " . $e->getMessage();
+  session_destroy();
+  exit;
+}
+
+function logMsg($priority, $message, $task = "Keycloak Sync") {
+  global $redis;
+
+  $finalMsg = array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message
+  );
+  $redis->lPush('CRON_LOG', json_encode($finalMsg));
+}
+
+// Load core functions first
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+
+$_SESSION['mailcow_cc_username'] = "admin";
+$_SESSION['mailcow_cc_role'] = "admin";
+$_SESSION['acl']['tls_policy'] = "1";
+$_SESSION['acl']['quarantine_notification'] = "1";
+$_SESSION['acl']['quarantine_category'] = "1";
+$_SESSION['acl']['ratelimit'] = "1";
+$_SESSION['acl']['sogo_access'] = "1";
+$_SESSION['acl']['protocol_access'] = "1";
+$_SESSION['acl']['mailbox_relayhost'] = "1";
+$_SESSION['acl']['unlimited_quota'] = "1";
+
+$iam_settings = identity_provider('get');
+if ($iam_settings['authsource'] != "keycloak" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
+  session_destroy();
+  exit;
+}
+
+// Set pagination variables
+$start = 0;
+$max = 100;
+
+// lock sync if already running
+$lock_file = '/tmp/iam-sync.lock';
+if (file_exists($lock_file)) {
+  $lock_file_parts = explode("\n", file_get_contents($lock_file));
+  $pid = $lock_file_parts[0];
+  if (count($lock_file_parts) > 1){
+    $last_execution = $lock_file_parts[1];
+    $elapsed_time = (time() - $last_execution) / 60;
+    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
+      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
+      session_destroy();
+      exit;
+    }
+  }
+
+  if (posix_kill($pid, 0)) {
+    logMsg("warning", "Sync is already running");
+    session_destroy();
+    exit;
+  } else {
+    unlink($lock_file);
+  }
+}
+$lock_file_handle = fopen($lock_file, 'w');
+fwrite($lock_file_handle, getmypid());
+fclose($lock_file_handle);
+
+// Init Keycloak Provider
+$iam_provider = identity_provider('init');
+
+// Loop until all users have been retrieved
+while (true) {
+  // Get admin access token
+  $admin_token = identity_provider("get-keycloak-admin-token");
+
+  // Make the API request to retrieve the users
+  $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users?first=$start&max=$max";
+  $ch = curl_init();
+  curl_setopt($ch, CURLOPT_URL, $url);
+  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+  curl_setopt($ch, CURLOPT_HTTPHEADER, [
+    "Content-Type: application/json",
+    "Authorization: Bearer " . $admin_token
+  ]);
+  $response = curl_exec($ch);
+  $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+  curl_close($ch);
+
+  if ($code != 200){
+    logMsg("err", "Received HTTP {$code}");
+    session_destroy();
+    exit;
+  }
+  try {
+    $response = json_decode($response, true);
+  } catch (Exception $e) {
+    logMsg("err", $e->getMessage());
+    break;
+  }
+  if (!is_array($response)){
+    logMsg("err", "Received malformed response from keycloak api");
+    break;
+  }
+  if (count($response) == 0) {
+    break;
+  }
+
+  // Process the batch of users
+  foreach ($response as $user) {
+    if (empty($user['email'])){
+      logMsg("warning", "No email address in keycloak found for user " . $user['name']);
+      continue;
+    }
+
+    // try get mailbox user
+    $stmt = $pdo->prepare("SELECT
+      mailbox.*,
+      domain.active AS d_active
+      FROM `mailbox`
+      INNER JOIN domain on mailbox.domain = domain.domain
+      WHERE `kind` NOT REGEXP 'location|thing|group'
+        AND `username` = :user");
+    $stmt->execute(array(':user' => $user['email']));
+    $row = $stmt->fetch(PDO::FETCH_ASSOC);
+
+    // check if matching attribute mapping exists
+    $user_template = $user['attributes']['mailcow_template'][0];
+    $mapper_key = array_search($user_template, $iam_settings['mappers']);
+
+    $_SESSION['access_all_exception'] = '1';
+    if (!$row && intval($iam_settings['import_users']) == 1){
+      if ($mapper_key === false){
+        if (!empty($iam_settings['default_template'])) {
+          $mbox_template = $iam_settings['default_template'];
+          logMsg("warning", "Using default template for user " . $user['email']);
+        } else {
+          logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
+          continue;
+        }
+      } else {
+        $mbox_template = $iam_settings['templates'][$mapper_key];
+      }
+      // mailbox user does not exist, create...
+      logMsg("info", "Creating user " . $user['email']);
+      $create_res = mailbox('add', 'mailbox_from_template', array(
+        'domain' => explode('@', $user['email'])[1],
+        'local_part' => explode('@', $user['email'])[0],
+        'name' => $user['firstName'] . " " . $user['lastName'],
+        'authsource' => 'keycloak',
+        'template' => $mbox_template
+      ));
+      if (!$create_res){
+        logMsg("err", "Could not create user " . $user['email']);
+        continue;
+      }
+    } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "keycloak") {
+      if ($mapper_key === false){
+        logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
+        continue;
+      }
+      $mbox_template = $iam_settings['templates'][$mapper_key];
+      // mailbox user does exist, sync attribtues...
+      logMsg("info", "Syncing attributes for user " . $user['email']);
+      mailbox('edit', 'mailbox_from_template', array(
+        'username' => $user['email'],
+        'name' => $user['firstName'] . " " . $user['lastName'],
+        'template' => $mbox_template
+      ));
+    } else {
+      // skip mailbox user
+      logMsg("info", "Skipping user " . $user['email']);
+    }
+    $_SESSION['access_all_exception'] = '0';
+
+    sleep(0.025);
+  }
+
+  // Update the pagination variables for the next batch
+  $start += $max;
+  sleep(1);
+}
+
+logMsg("info", "DONE!");
+// add last execution time to lock file
+$lock_file_handle = fopen($lock_file, 'w');
+fwrite($lock_file_handle, getmypid() . "\n" . time());
+fclose($lock_file_handle);
+session_destroy();

data/conf/phpfpm/crons/ldap-sync.php

@@ -0,0 +1,198 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+// Init database
+//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  logMsg("err", $e->getMessage());
+  session_destroy();
+  exit;
+}
+
+// Init Redis
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  }
+  else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+}
+catch (Exception $e) {
+  echo "Exiting: " . $e->getMessage();
+  session_destroy();
+  exit;
+}
+
+function logMsg($priority, $message, $task = "LDAP Sync") {
+  global $redis;
+
+  $finalMsg = array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message
+  );
+  $redis->lPush('CRON_LOG', json_encode($finalMsg));
+}
+
+// Load core functions first
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+
+$_SESSION['mailcow_cc_username'] = "admin";
+$_SESSION['mailcow_cc_role'] = "admin";
+$_SESSION['acl']['tls_policy'] = "1";
+$_SESSION['acl']['quarantine_notification'] = "1";
+$_SESSION['acl']['quarantine_category'] = "1";
+$_SESSION['acl']['ratelimit'] = "1";
+$_SESSION['acl']['sogo_access'] = "1";
+$_SESSION['acl']['protocol_access'] = "1";
+$_SESSION['acl']['mailbox_relayhost'] = "1";
+$_SESSION['acl']['unlimited_quota'] = "1";
+
+$iam_settings = identity_provider('get');
+if ($iam_settings['authsource'] != "ldap" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
+  session_destroy();
+  exit;
+}
+
+// Set pagination variables
+$start = 0;
+$max = 100;
+
+// lock sync if already running
+$lock_file = '/tmp/iam-sync.lock';
+if (file_exists($lock_file)) {
+  $lock_file_parts = explode("\n", file_get_contents($lock_file));
+  $pid = $lock_file_parts[0];
+  if (count($lock_file_parts) > 1){
+    $last_execution = $lock_file_parts[1];
+    $elapsed_time = (time() - $last_execution) / 60;
+    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
+      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
+      session_destroy();
+      exit;
+    }
+  }
+
+  if (posix_kill($pid, 0)) {
+    logMsg("warning", "Sync is already running");
+    session_destroy();
+    exit;
+  } else {
+    unlink($lock_file);
+  }
+}
+$lock_file_handle = fopen($lock_file, 'w');
+fwrite($lock_file_handle, getmypid());
+fclose($lock_file_handle);
+
+// Init Provider
+$iam_provider = identity_provider('init');
+
+// Get ldap users
+$ldap_query = $iam_provider->query();
+if (!empty($iam_settings['filter'])) {
+  $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
+}
+$response = $ldap_query->where($iam_settings['username_field'], "*")
+  ->where($iam_settings['attribute_field'], "*")
+  ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname'])
+  ->paginate($max);
+
+// Process the users
+foreach ($response as $user) {
+  // try get mailbox user
+  $stmt = $pdo->prepare("SELECT
+    mailbox.*,
+    domain.active AS d_active
+    FROM `mailbox`
+    INNER JOIN domain on mailbox.domain = domain.domain
+    WHERE `kind` NOT REGEXP 'location|thing|group'
+      AND `username` = :user");
+  $stmt->execute(array(':user' => $user[$iam_settings['username_field']][0]));
+  $row = $stmt->fetch(PDO::FETCH_ASSOC);
+
+  // check if matching attribute mapping exists
+  $user_template = $user[$iam_settings['attribute_field']][0];
+  $mapper_key = array_search($user_template, $iam_settings['mappers']);
+
+  if (empty($user[$iam_settings['username_field']][0])){
+    logMsg("warning", "Skipping user " . $user['displayname'][0] . " due to empty LDAP ". $iam_settings['username_field'] . " property.");
+    continue;
+  }
+
+  $_SESSION['access_all_exception'] = '1';
+  if (!$row && intval($iam_settings['import_users']) == 1){
+    if ($mapper_key === false){
+      if (!empty($iam_settings['default_template'])) {
+        $mbox_template = $iam_settings['default_template'];
+      } else {
+        logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
+        continue;
+      }
+    } else {
+      $mbox_template = $iam_settings['templates'][$mapper_key];
+    }
+    // mailbox user does not exist, create...
+    logMsg("info", "Creating user " .  $user[$iam_settings['username_field']][0]);
+    $create_res = mailbox('add', 'mailbox_from_template', array(
+      'domain' => explode('@',  $user[$iam_settings['username_field']][0])[1],
+      'local_part' => explode('@',  $user[$iam_settings['username_field']][0])[0],
+      'name' => $user['displayname'][0],
+      'authsource' => 'ldap',
+      'template' => $mbox_template
+    ));
+    if (!$create_res){
+      logMsg("err", "Could not create user " . $user[$iam_settings['username_field']][0]);
+      continue;
+    }
+  } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "ldap") {
+    if ($mapper_key === false){
+      logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
+      continue;
+    }
+    $mbox_template = $iam_settings['templates'][$mapper_key];
+    // mailbox user does exist, sync attribtues...
+    logMsg("info", "Syncing attributes for user " . $user[$iam_settings['username_field']][0]);
+    mailbox('edit', 'mailbox_from_template', array(
+      'username' =>  $user[$iam_settings['username_field']][0],
+      'name' => $user['displayname'][0],
+      'template' => $mbox_template
+    ));
+  } else {
+    // skip mailbox user
+    logMsg("info", "Skipping user " .  $user[$iam_settings['username_field']][0]);
+  }
+  $_SESSION['access_all_exception'] = '0';
+
+  sleep(0.025);
+}
+
+logMsg("info", "DONE!");
+// add last execution time to lock file
+$lock_file_handle = fopen($lock_file, 'w');
+fwrite($lock_file_handle, getmypid() . "\n" . time());
+fclose($lock_file_handle);
+session_destroy();

data/web/admin/mailbox.php

@@ -9,8 +9,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 
-$js_minifier->add('/web/js/site/imapsync.js');
-$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -40,12 +38,10 @@ $template_data = [
   'global_filters' => mailbox('get', 'global_filter_details'),
   'domains' => $domains,
   'mailboxes' => $mailboxes,
-  'mbox_templates' => mailbox('get', 'mailbox_templates'),
   'lang_mailbox' => json_encode($lang['mailbox']),
   'lang_rl' => json_encode($lang['ratelimit']),
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
-  'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/system.php

@@ -12,7 +12,6 @@ $fido2_data = fido2(array("action" => "get_friendly_names"));
 $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
-$js_minifier->add('/web/js/site/identity.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -94,6 +93,10 @@ $template_data = [
   'tfa_id' => @$_SESSION['tfa_id'],
   'fido2_cid' => @$_SESSION['fido2_cid'],
   'fido2_data' => $fido2_data,
+  'api' => [
+    'ro' => admin_api('ro', 'get'),
+    'rw' => admin_api('rw', 'get'),
+  ],
   'dkim_domains' => $dkim_domains,
   'dkim_domains_with_keys' => $dkim_domains_with_keys,
   'dkim_blind_domains' => $dkim_blind_domains,
@@ -115,7 +118,6 @@ $template_data = [
   'ui_themes' => customize('get', 'ui_themes'),
   'ui_theme' => customize('get', 'ui_theme'),
   'password_complexity' => password_complexity('get'),
-  'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',

data/web/domainadmin/mailbox.php

@@ -9,8 +9,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 
-$js_minifier->add('/web/js/site/imapsync.js');
-$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -44,7 +42,6 @@ $template_data = [
   'lang_rl' => json_encode($lang['ratelimit']),
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
-  'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/edit.php

@@ -99,27 +99,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
         $template = 'edit/oauth2client.twig';
         $template_data = ['oauth2client' => $oauth2client];
     }
-    elseif (isset($_GET['mailbox-group']) &&
-      is_numeric($_GET['mailbox-group'])) {
-        $result = \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->getGroup($_GET['mailbox-group']);
-        $template = 'edit/mailbox-group.twig';
-        // Member candidates: mailboxes of the domains the caller manages
-        $mailboxes = [];
-        foreach (mailbox('get', 'domains') as $domain) {
-          foreach (mailbox('get', 'mailboxes', $domain) as $mailbox) {
-            $mailboxes[] = $mailbox;
-          }
-        }
-        $template_data = [
-          'mailboxes' => $mailboxes,
-          'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
-        ];
-    }
-    elseif (isset($_GET['api-key']) &&
-      is_numeric($_GET['api-key'])) {
-        $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
-        $template = 'edit/api-key.twig';
-    }
     elseif (isset($_GET['aliasdomain']) &&
       is_valid_domain_name(html_entity_decode(rawurldecode($_GET["aliasdomain"]))) &&
       !empty($_GET["aliasdomain"])) {
@@ -157,8 +136,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'get_tls_policy' => $get_tls_policy,
           'rlyhosts' => $rlyhosts,
           'sender_acl_handles' => mailbox('get', 'sender_acl_handles', $mailbox),
-          'mailbox_groups' => \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->getMemberships($mailbox),
-          'identity_link' => \Mailcow\Identity\IdentityModule::fromGlobals()->api->getLink($mailbox),
           'user_acls' => acl('get', 'user', $mailbox),
           'mailbox_details' => $result,
           'iam_settings' => $iam_settings,
@@ -227,15 +204,9 @@ if (isset($_SESSION['mailcow_cc_role'])) {
     if (isset($_GET['syncjob']) &&
       is_numeric($_GET['syncjob'])) {
         $id = $_GET["syncjob"];
-        $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id);
+        $result = mailbox('get', 'syncjob_details', $id);
         $template = 'edit/syncjob.twig';
       }
-    elseif (isset($_GET['syncjob_source']) &&
-      is_numeric($_GET['syncjob_source'])) {
-        $id = $_GET["syncjob_source"];
-        $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSource($id);
-        $template = 'edit/imapsync_source.twig';
-    }
     elseif (isset($_GET['filter']) &&
       is_numeric($_GET['filter'])) {
         $id = $_GET["filter"];
@@ -254,8 +225,6 @@ else {
   $template_data['access_denied'] = true;
 }
 
-$js_minifier->add('/web/js/site/imapsync.js');
-$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/edit.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 
@@ -264,6 +233,5 @@ $template_data['return_to'] = $_SESSION['return_to'];
 $template_data['lang_user'] = json_encode($lang['user']);
 $template_data['lang_admin'] = json_encode($lang['admin']);
 $template_data['lang_datatables'] = json_encode($lang['datatables']);
-$template_data['imapsync_redirect_uri'] = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri();
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/inc/ajax/container_ctrl.php

@@ -10,10 +10,6 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
 	exit();
 }
-// Read-only API keys must not control containers
-if (!empty($_SESSION['mailcow_cc_api']) && ($_SESSION['mailcow_cc_api_access'] ?? '') != 'rw') {
-	exit();
-}
 
 if (preg_match('/^[a-z\-]{0,}-mailcow/', $_GET['service'])) {
   if ($_GET['action'] == "start") {

data/web/inc/ajax/syncjob_logs.php

@@ -6,7 +6,7 @@ if (!isset($_SESSION['mailcow_cc_role'])) {
 }
 
 if (isset($_GET['id']) && is_numeric($_GET['id'])) {
-  if ($details = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob(intval($_GET['id']))) {
+  if ($details = mailbox('get', 'syncjob_details', intval($_GET['id']))) {
     echo (empty($details['log'])) ? '-' : $details['log'];
   }
 }

data/web/inc/footer.inc.php

@@ -33,11 +33,11 @@ if (array_key_exists('pending_tfa_methods', $_SESSION)) {
   if (isset($pending_tfa_authmechs['webauthn'])) {
     $pending_tfa_authmechs['webauthn'] = true;
   }
-  if (!isset($pending_tfa_authmechs['webauthn'])
+  if (!isset($pending_tfa_authmechs['webauthn']) 
       && isset($pending_tfa_authmechs['yubi_otp'])) {
     $pending_tfa_authmechs['yubi_otp'] = true;
   }
-  if (!isset($pending_tfa_authmechs['webauthn'])
+  if (!isset($pending_tfa_authmechs['webauthn']) 
       && !isset($pending_tfa_authmechs['yubi_otp'])
       && isset($pending_tfa_authmechs['totp'])) {
     $pending_tfa_authmechs['totp'] = true;
@@ -72,8 +72,6 @@ $globalVariables = [
   'lang_fido2' => json_encode($lang['fido2']),
   'lang_success' => json_encode($lang['success']),
   'lang_danger' => json_encode($lang['danger']),
-  'lang_syncjobs' => json_encode($lang['syncjobs']),
-  'lang_groups' => json_encode($lang['groups']),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.acl.inc.php

@@ -193,10 +193,6 @@ function acl($_action, $_scope = null, $_data = null, $_extra = null) {
         while ($row = array_shift($acl_all)) {
           $acl['acl'][$row['Field']] = 1;
         }
-        // Read-only API keys must not log in as other accounts (SOGo SSO, dual login)
-        if (!empty($_SESSION['mailcow_cc_api']) && ($_SESSION['mailcow_cc_api_access'] ?? '') != 'rw') {
-          $acl['acl']['login_as'] = 0;
-        }
       }
       elseif ($_SESSION['mailcow_cc_role'] == 'domainadmin' ||
         (isset($_SESSION["dual-login"]["role"]) && $_SESSION["dual-login"]["role"] == 'domainadmin')) {

data/web/inc/functions.admin.inc.php

@@ -103,7 +103,7 @@ function admin($_action, $_data = null) {
             continue;
           }
         }
-        if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new)) || $username_new == 'API') {
+        if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new))) {
           $_SESSION['return'][] = array(
             'type' => 'danger',
             'log' => array(__FUNCTION__, $_action, $_data_log),
@@ -179,9 +179,6 @@ function admin($_action, $_data = null) {
             $stmt = $pdo->prepare("UPDATE `tfa` SET `username` = :username_new WHERE `username` = :username");
             $stmt->execute(array(':username_new' => $username_new, ':username' => $username));
           }
-          if ($username_new != $username) {
-            \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username, $username_new);
-          }
         }
         $_SESSION['return'][] = array(
           'type' => 'success',
@@ -214,7 +211,6 @@ function admin($_action, $_data = null) {
         $stmt->execute(array(
           ':username' => $username,
         ));
-        \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username);
         $stmt = $pdo->prepare("DELETE FROM `domain_admins` WHERE `username` = :username");
         $stmt->execute(array(
           ':username' => $username,

data/web/inc/functions.auth.inc.php

@@ -36,18 +36,7 @@ function check_login($user, $pass, $extra = null) {
 
   // Try validate user
   if (!isset($role) || $role == "user") {
-    try {
-      $result = user_login($user, $pass, $extra);
-    } catch (\Mailcow\Auth\IdpUnavailable $e) {
-      // no failed login, netfilter must not count it
-      error_log("mailcow UI: Identity provider unavailable for " . $user . " by " . $_SERVER['REMOTE_ADDR'] . ": " . $e->getMessage());
-      $_SESSION['return'][] =  array(
-        'type' => 'danger',
-        'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
-        'msg' => 'iam_unavailable'
-      );
-      return false;
-    }
+    $result = user_login($user, $pass, $extra);
     if ($result !== false) {
       $real_rip = ($_SERVER['HTTP_X_REAL_IP'] ?? $_SERVER['REMOTE_ADDR']);
       set_sasl_log($user, $real_rip, $extra['service']);
@@ -273,20 +262,12 @@ function user_login($user, $pass, $extra = null){
 
   // check if user has access to service (imap, smtp, pop3, sieve) if service is set
   $row['attributes'] = json_decode($row['attributes'], true);
-  // Shared mailboxes have no login of their own, members open them through group shares
-  if (($row['attributes']['shared'] ?? '0') == '1') {
-    return false;
-  }
   if ($extra['service'] != 'NONE') {
     $key = strtolower($extra['service']) . "_access";
     if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
       return false;
     }
   }
-  // IdP accounts: refuse inactive mailboxes and domains before asking the IdP (no template side effects)
-  if ($row['authsource'] != 'mailcow' && ($row['active'] != '1' || $row['d_active'] != '1')) {
-    return false;
-  }
   switch ($row['authsource']) {
     case 'keycloak':
       // user authsource is keycloak, try using via rest flow
@@ -481,7 +462,6 @@ function apppass_login($user, $pass, $extra = null){
       AND `mailbox`.`active` = '1'
       AND `domain`.`active` = '1'
       AND `app_passwd`.`active` = '1'
-      AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
       AND `app_passwd`.`mailbox` = :user"
   );
   // fetch password data
@@ -527,14 +507,11 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     return false;
   }
   if (!$iam_provider) {
-    throw new \Mailcow\Auth\IdpUnavailable('Keycloak is not configured completely');
+    return false;
   }
 
   // get access_token for service account of mailcow client
   $admin_token = identity_provider("get-keycloak-admin-token");
-  if (!$admin_token) {
-    throw new \Mailcow\Auth\IdpUnavailable('Could not get an admin token from Keycloak');
-  }
 
   // get the mailcow_password attribute from keycloak user
   $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users";
@@ -548,23 +525,15 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
       'Authorization: Bearer ' . $admin_token,
       'Content-Type: application/json'
   ));
-  if (intval($iam_settings['ignore_ssl_error']) == 1) {
-    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
-    curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
-  }
-  $users = json_decode(curl_exec($curl), true);
+  $user_res = json_decode(curl_exec($curl), true)[0];
   $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
   curl_close($curl);
-  // the password is checked below, a failed request is never a wrong password
   if ($code != 200) {
-    throw new \Mailcow\Auth\IdpUnavailable('Keycloak returned HTTP ' . $code);
-  }
-  // exact=true can still return several users with that email
-  if (!is_array($users) || count($users) != 1) {
-    return false;
-  }
-  $user_res = $users[0];
-  if (empty($user_res['enabled'])) {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $user, '*', 'Identity Provider returned HTTP ' . $code),
+      'msg' => 'generic_server_error'
+    );
     return false;
   }
   if (!isset($user_res['attributes']['mailcow_password']) || !is_array($user_res['attributes']['mailcow_password'])){
@@ -591,28 +560,18 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   }
 
   // get mapped template
-  $name = trim(($user_res['firstName'] ?? '') . ' ' . ($user_res['lastName'] ?? ''));
-  $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
-  $mapped_template = $idp_login->mappedTemplate($user_res['attributes']['mailcow_template'] ?? array());
+  $user_template = $user_res['attributes']['mailcow_template'][0];
+  $mapper_key = array_search($user_template, $iam_settings['mappers']);
 
   if (!$create) {
-    if (!$idp_login->verifyLink('keycloak', $user, (string)($user_res['id'] ?? ''))) {
-      $_SESSION['return'][] =  array(
-        'type' => 'danger',
-        'log' => array(__FUNCTION__, $user, '*', 'The mailbox is linked to another IdP user'),
-        'msg' => 'generic_server_error'
-      );
-      return false;
-    }
     // login success
-    $update_template = $idp_login->templateFor($user, $mapped_template);
-    if ($update_template !== null) {
+    if ($mapper_key !== false) {
       // update user
       $_SESSION['access_all_exception'] = '1';
       mailbox('edit', 'mailbox_from_template', array(
         'username' => $user,
-        'name' => $name,
-        'template' => $update_template
+        'name' => $user_res['name'],
+        'template' => $iam_settings['templates'][$mapper_key]
       ));
       $_SESSION['access_all_exception'] = '0';
     }
@@ -631,7 +590,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     return false;
   }
   // check if matching attribute exist
-  if ($mapped_template === null) {
+  if (empty($iam_settings['mappers']) || !$user_template || $mapper_key === false) {
     if (!empty($iam_settings['default_template'])) {
       $mbox_template = $iam_settings['default_template'];
     } else {
@@ -643,16 +602,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
       return false;
     }
   } else {
-    $mbox_template = $mapped_template;
-  }
-
-  if (!$idp_login->mayCreate('keycloak', $user, (string)($user_res['id'] ?? ''))) {
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $user, '*', 'The IdP user is linked to another mailbox'),
-      'msg' => 'generic_server_error'
-    );
-    return false;
+    $mbox_template = $iam_settings['templates'][$mapper_key];
   }
 
   // create mailbox
@@ -660,7 +610,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   $create_res = mailbox('add', 'mailbox_from_template', array(
     'domain' => explode('@', $user)[1],
     'local_part' => explode('@', $user)[0],
-    'name' => $name,
+    'name' => $user_res['name'],
     'authsource' => 'keycloak',
     'template' => $mbox_template
   ));
@@ -674,7 +624,6 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     );
     return false;
   }
-  $idp_login->storeLink('keycloak', $user, (string)($user_res['id'] ?? ''));
 
   return 'user';
 }
@@ -697,8 +646,7 @@ function ldap_mbox_login($user, $pass, $extra = null){
     return false;
   }
   if (!$iam_provider) {
-    // identity_provider('init') could not connect
-    throw new \Mailcow\Auth\IdpUnavailable('No connection to the LDAP server');
+    return false;
   }
 
   try {
@@ -707,42 +655,47 @@ function ldap_mbox_login($user, $pass, $extra = null){
       $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
     }
     $ldap_query = $ldap_query->where($iam_settings['username_field'], '=', $user)
-      ->select(array_values(array_filter(array_merge([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn'],
-        \Mailcow\Identity\Directory\LdapDirectory::idAttributes($iam_settings['id_attribute'] ?? '')))));
+      ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn']);
 
     $user_res = $ldap_query->firstOrFail();
-  } catch (\LdapRecord\Query\ObjectNotFoundException $e) {
-    return false;
   } catch (Exception $e) {
-    throw new \Mailcow\Auth\IdpUnavailable('LDAP search failed: ' . $e->getMessage(), 0, $e);
+    // clear $_SESSION['return'] to not leak data
+    $_SESSION['return'] = array();
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
+      'msg' => 'generic_server_error'
+    );
+    return false;
   }
-  $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
-  if (!$idp_login->verifyLdapPassword($iam_provider, (string)$user_res['dn'], (string)$pass)) {
+  try {
+    if (!$iam_provider->auth()->attempt($user_res['dn'], $pass)) {
+      return false;
+    }
+  } catch (Exception $e) {
+    // clear $_SESSION['return'] to not leak data
+    $_SESSION['return'] = array();
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
+      'msg' => 'generic_server_error'
+    );
     return false;
   }
 
   // get mapped template
-  $mapped_template = $idp_login->mappedTemplate($user_res[$iam_settings['attribute_field']] ?? array());
-  $external_id = \Mailcow\Identity\Directory\LdapDirectory::entryId($user_res, $iam_settings['id_attribute'] ?? '');
+  $user_template = $user_res[$iam_settings['attribute_field']][0];
+  $mapper_key = array_search($user_template, $iam_settings['mappers']);
 
   if (!$create) {
-    if (!$idp_login->verifyLink('ldap', $user, $external_id)) {
-      $_SESSION['return'][] =  array(
-        'type' => 'danger',
-        'log' => array(__FUNCTION__, $user, '*', 'The mailbox is linked to another IdP user'),
-        'msg' => 'generic_server_error'
-      );
-      return false;
-    }
     // login success
-    $update_template = $idp_login->templateFor($user, $mapped_template);
-    if ($update_template !== null) {
+    if ($mapper_key !== false) {
       // update user
       $_SESSION['access_all_exception'] = '1';
       mailbox('edit', 'mailbox_from_template', array(
         'username' => $user,
         'name' => $user_res['displayname'][0],
-        'template' => $update_template
+        'template' => $iam_settings['templates'][$mapper_key]
       ));
       $_SESSION['access_all_exception'] = '0';
     }
@@ -761,7 +714,7 @@ function ldap_mbox_login($user, $pass, $extra = null){
     return false;
   }
   // check if matching attribute exist
-  if ($mapped_template === null) {
+  if (empty($iam_settings['mappers']) || !$user_template || $mapper_key === false) {
     if (!empty($iam_settings['default_template'])) {
       $mbox_template = $iam_settings['default_template'];
     } else {
@@ -773,16 +726,7 @@ function ldap_mbox_login($user, $pass, $extra = null){
       return false;
     }
   } else {
-    $mbox_template = $mapped_template;
-  }
-
-  if (!$idp_login->mayCreate('ldap', $user, $external_id)) {
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $user, '*', 'The IdP user is linked to another mailbox'),
-      'msg' => 'generic_server_error'
-    );
-    return false;
+    $mbox_template = $iam_settings['templates'][$mapper_key];
   }
 
   // create mailbox
@@ -804,7 +748,6 @@ function ldap_mbox_login($user, $pass, $extra = null){
     );
     return false;
   }
-  $idp_login->storeLink('ldap', $user, $external_id);
 
   return 'user';
 }

data/web/inc/functions.domain_admin.inc.php

@@ -156,7 +156,7 @@ function domain_admin($_action, $_data = null) {
               }
             }
           }
-          if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new)) || $username_new == 'API') {
+          if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new))) {
             $_SESSION['return'][] = array(
               'type' => 'danger',
               'log' => array(__FUNCTION__, $_action, $_data_log),
@@ -183,9 +183,6 @@ function domain_admin($_action, $_data = null) {
             ':username_new' => $username_new,
             ':username' => $username
           ));
-          if ($username_new != $username) {
-            \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username, $username_new);
-          }
           if (!empty($domains)) {
             foreach ($domains as $domain) {
               $stmt = $pdo->prepare("INSERT INTO `domain_admins` (`username`, `domain`, `created`, `active`)
@@ -334,8 +331,6 @@ function domain_admin($_action, $_data = null) {
         $stmt->execute(array(
           ':username' => $username,
         ));
-        // sync sources created by this account must not pass to a re-created account of the same name
-        \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username);
         $_SESSION['return'][] = array(
           'type' => 'success',
           'log' => array(__FUNCTION__, $_action, $_data_log),

data/web/inc/functions.fail2ban.inc.php

@@ -60,14 +60,13 @@ function fail2ban($_action, $_data = null, $_extra = null) {
         $queue_unban = $redis->hGetAll('F2B_QUEUE_UNBAN');
         if (is_array($active_bans)) {
           foreach ($active_bans as $network => $banned_until) {
-            // Expired bans are lifted by netfilter within seconds
-            $difference = max(0, intval($banned_until) - time());
-            $queued_for_unban = ($difference == 0 || (isset($queue_unban[$network]) && $queue_unban[$network] == 1)) ? 1 : 0;
+            $queued_for_unban = (isset($queue_unban[$network]) && $queue_unban[$network] == 1) ? 1 : 0;
+            $difference = $banned_until - time();
             $f2b_options['active_bans'][] = array(
               'queued_for_unban' => $queued_for_unban,
               'network' => $network,
               'ip' => strtok($network,'/'),
-              'banned_until' => sprintf('%02dh %02dm %02ds', intdiv($difference, 3600), intdiv($difference, 60) % 60, $difference % 60)
+              'banned_until' => sprintf('%02dh %02dm %02ds', ($difference/3600), ($difference/60%60), $difference%60)
             );
           }
         }
@@ -109,15 +108,16 @@ function fail2ban($_action, $_data = null, $_extra = null) {
             );
             return false;
           }
-          // netfilter recreates the default rules within 10 seconds
+          // Rules will also be recreated on log events, but rules may seem empty for a second in the UI
+          docker('post', 'netfilter-mailcow', 'restart');
           $fail_count = 0;
           $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
-          while (empty($regex_result) && $fail_count < 15) {
+          while (empty($regex_result) && $fail_count < 10) {
             $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
             $fail_count++;
             sleep(1);
           }
-          if (empty($regex_result)) {
+          if ($fail_count >= 10) {
             $_SESSION['return'][] = array(
               'type' => 'danger',
               'log' => array(__FUNCTION__, $_action, $_data_log),
@@ -141,7 +141,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
           $_SESSION['return'][] = array(
             'type' => 'success',
             'log' => array(__FUNCTION__, $_action, $_data_log),
-            'msg' => 'f2b_modified'
+            'msg' => array('object_modified', htmlspecialchars($network))
           );
           return true;
         }
@@ -172,7 +172,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
               if (valid_network($network)) {
                 try {
                   $redis->hSet('F2B_WHITELIST', $network, 1);
-                  $redis->hDel('F2B_BLACKLIST', $network);
+                  $redis->hDel('F2B_BLACKLIST', $network, 1);
                   $redis->hSet('F2B_QUEUE_UNBAN', $network, 1);
                 }
                 catch (RedisException $e) {
@@ -199,13 +199,14 @@ function fail2ban($_action, $_data = null, $_extra = null) {
               if (valid_network($network) && !in_array($network, array(
                 '0.0.0.0',
                 '0.0.0.0/0',
-                getenv('IPV4_NETWORK') . '.0/24',
-                getenv('IPV4_NETWORK') . '.0',
+                getenv('IPV4_NETWORK') . '0/24',
+                getenv('IPV4_NETWORK') . '0',
                 getenv('IPV6_NETWORK')
               ))) {
                 try {
                   $redis->hSet('F2B_BLACKLIST', $network, 1);
-                  $redis->hDel('F2B_WHITELIST', $network);
+                  $redis->hDel('F2B_WHITELIST', $network, 1);
+                  //$response = docker('post', 'netfilter-mailcow', 'restart');
                 }
                 catch (RedisException $e) {
                   $_SESSION['return'][] = array(
@@ -260,60 +261,62 @@ function fail2ban($_action, $_data = null, $_extra = null) {
       $f2b_options['ban_time'] = ($ban_time < 60) ? 60 : $ban_time;
       $f2b_options['ban_time_increment'] = ($ban_time_increment == 1) ? true : false;
       $f2b_options['max_ban_time'] = ($max_ban_time < 60) ? 60 : $max_ban_time;
-      $f2b_options['netban_ipv4'] = min(32, max(8, $netban_ipv4));
-      $f2b_options['netban_ipv6'] = min(128, max(8, $netban_ipv6));
+      $f2b_options['netban_ipv4'] = ($netban_ipv4 < 8) ? 8 : $netban_ipv4;
+      $f2b_options['netban_ipv6'] = ($netban_ipv6 < 8) ? 8 : $netban_ipv6;
+      $f2b_options['netban_ipv4'] = ($netban_ipv4 > 32) ? 32 : $netban_ipv4;
+      $f2b_options['netban_ipv6'] = ($netban_ipv6 > 128) ? 128 : $netban_ipv6;
       $f2b_options['max_attempts'] = ($max_attempts < 1) ? 1 : $max_attempts;
       $f2b_options['retry_window'] = ($retry_window < 1) ? 1 : $retry_window;
-      $f2b_options['banlist_id'] = (!empty($is_now['banlist_id'])) ? $is_now['banlist_id'] : uuid4();
+      $f2b_options['banlist_id'] = $is_now['banlist_id'];
       $f2b_options['manage_external'] = ($manage_external > 0) ? 1 : 0;
-      $wl_items = array();
-      if (!empty($wl)) {
-        foreach (array_filter(array_map('trim', preg_split("/( |,|;|\n)/", $wl))) as $wl_item) {
-          if (valid_network($wl_item) || valid_hostname($wl_item)) {
-            $wl_items[$wl_item] = 1;
-          }
-          else {
-            $_SESSION['return'][] = array(
-              'type' => 'danger',
-              'log' => array(__FUNCTION__, $_action, $_data_log),
-              'msg' => array('network_host_invalid', $wl_item)
-            );
-          }
-        }
-      }
-      $bl_items = array();
-      if (!empty($bl)) {
-        foreach (array_filter(array_map('trim', preg_split("/( |,|;|\n)/", $bl))) as $bl_item) {
-          if (valid_network($bl_item) && !in_array($bl_item, array(
-            '0.0.0.0',
-            '0.0.0.0/0',
-            getenv('IPV4_NETWORK') . '.0/24',
-            getenv('IPV4_NETWORK') . '.0',
-            getenv('IPV6_NETWORK')
-          ))) {
-            $bl_items[$bl_item] = 1;
-          }
-          else {
-            $_SESSION['return'][] = array(
-              'type' => 'danger',
-              'log' => array(__FUNCTION__, $_action, $_data_log),
-              'msg' => array('network_host_invalid', $bl_item)
-            );
-          }
-        }
-      }
       try {
-        // One transaction, netfilter must never see empty lists in between
-        $redis->multi();
         $redis->Set('F2B_OPTIONS', json_encode($f2b_options));
-        $redis->Del('F2B_WHITELIST', 'F2B_BLACKLIST');
-        if (!empty($wl_items)) {
-          $redis->hMSet('F2B_WHITELIST', $wl_items);
+        $redis->Del('F2B_WHITELIST');
+        $redis->Del('F2B_BLACKLIST');
+        if(!empty($wl)) {
+          $wl_array = array_map('trim', preg_split( "/( |,|;|\n)/", $wl));
+          $wl_array = array_filter($wl_array);
+          if (is_array($wl_array)) {
+            foreach ($wl_array as $wl_item) {
+              if (valid_network($wl_item) || valid_hostname($wl_item)) {
+                $redis->hSet('F2B_WHITELIST', $wl_item, 1);
+              }
+              else {
+                $_SESSION['return'][] = array(
+                  'type' => 'danger',
+                  'log' => array(__FUNCTION__, $_action, $_data_log),
+                  'msg' => array('network_host_invalid', $wl_item)
+                );
+                continue;
+              }
+            }
+          }
         }
-        if (!empty($bl_items)) {
-          $redis->hMSet('F2B_BLACKLIST', $bl_items);
+        if(!empty($bl)) {
+          $bl_array = array_map('trim', preg_split( "/( |,|;|\n)/", $bl));
+          $bl_array = array_filter($bl_array);
+          if (is_array($bl_array)) {
+            foreach ($bl_array as $bl_item) {
+              if (valid_network($bl_item) && !in_array($bl_item, array(
+                '0.0.0.0',
+                '0.0.0.0/0',
+                getenv('IPV4_NETWORK') . '0/24',
+                getenv('IPV4_NETWORK') . '0',
+                getenv('IPV6_NETWORK')
+              ))) {
+                $redis->hSet('F2B_BLACKLIST', $bl_item, 1);
+              }
+              else {
+                $_SESSION['return'][] = array(
+                  'type' => 'danger',
+                  'log' => array(__FUNCTION__, $_action, $_data_log),
+                  'msg' => array('network_host_invalid', $bl_item)
+                );
+                continue;
+              }
+            }
+          }
         }
-        $redis->exec();
       }
       catch (RedisException $e) {
         $_SESSION['return'][] = array(
@@ -345,9 +348,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
       if (is_array($_extra)) {
         $_extra = $_extra[0];
       }
-      // Timing safe, a missing or empty id never matches
-      $banlist_id = (string)($f2b_options['banlist_id'] ?? '');
-      if ($banlist_id === '' || !is_string($_extra) || !hash_equals($banlist_id, $_extra)) {
+      if ($_extra != $f2b_options['banlist_id']){
         http_response_code(404);
         return false;
       }

data/web/inc/functions.inc.php

@@ -1292,13 +1292,8 @@ function set_tfa($_data) {
       $row = $stmt->fetch(PDO::FETCH_ASSOC);
       if ($row) {
         if ($row['authsource'] == 'ldap'){
-          try {
-            if (!ldap_mbox_login($username, $_data["confirm_password"], $iam_settings)) $access_denied = true;
-            else $access_denied = false;
-          } catch (\Mailcow\Auth\IdpUnavailable $e) {
-            $_SESSION['return'][] = array('type' => 'danger', 'log' => array(__FUNCTION__, $_data_log, $e->getMessage()), 'msg' => 'iam_unavailable');
-            return false;
-          }
+          if (!ldap_mbox_login($username, $_data["confirm_password"], $iam_settings)) $access_denied = true;
+          else $access_denied = false;
         } else {
           if (!verify_hash($row['password'], $_data["confirm_password"])) $access_denied = true;
           else $access_denied = false;
@@ -2059,6 +2054,128 @@ function verify_tfa_login($username, $_data) {
     return true;
   }
 }
+function admin_api($access, $action, $data = null) {
+  global $pdo;
+  if ($_SESSION['mailcow_cc_role'] != "admin") {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__),
+      'msg' => 'access_denied'
+    );
+    return false;
+  }
+  if ($access !== "ro" && $access !== "rw") {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__),
+      'msg' => 'invalid access type'
+    );
+    return false;
+  }
+  if ($action == "edit") {
+    $active = (!empty($data['active'])) ? 1 : 0;
+    $skip_ip_check = (isset($data['skip_ip_check'])) ? 1 : 0;
+    $allow_from = array();
+    if (isset($data['allow_from'])) {
+      $allow_from = array_map('trim', preg_split( "/( |,|;|\n)/", $data['allow_from']));
+    }
+    foreach ($allow_from as $key => $val) {
+      if (empty($val)) {
+        unset($allow_from[$key]);
+        continue;
+      }
+      if (valid_network($val) !== true) {
+        $_SESSION['return'][] =  array(
+          'type' => 'warning',
+          'log' => array(__FUNCTION__, $data),
+          'msg' => array('ip_invalid', htmlspecialchars($allow_from[$key]))
+        );
+        unset($allow_from[$key]);
+        continue;
+      }
+    }
+    $allow_from = implode(',', array_unique(array_filter($allow_from)));
+    if (empty($allow_from) && $skip_ip_check == 0) {
+      $_SESSION['return'][] =  array(
+        'type' => 'danger',
+        'log' => array(__FUNCTION__, $data),
+        'msg' => 'ip_list_empty'
+      );
+      return false;
+    }
+    $api_key = implode('-', array(
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3)))
+    ));
+    $stmt = $pdo->query("SELECT `api_key` FROM `api` WHERE `access` = '" . $access . "'");
+    $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
+    if (empty($num_results)) {
+      $stmt = $pdo->prepare("INSERT INTO `api` (`api_key`, `skip_ip_check`, `active`, `allow_from`, `access`)
+        VALUES (:api_key, :skip_ip_check, :active, :allow_from, :access);");
+      $stmt->execute(array(
+        ':api_key' => $api_key,
+        ':skip_ip_check' => $skip_ip_check,
+        ':active' => $active,
+        ':allow_from' => $allow_from,
+        ':access' => $access
+      ));
+    }
+    else {
+      if ($skip_ip_check == 0) {
+        $stmt = $pdo->prepare("UPDATE `api` SET `skip_ip_check` = :skip_ip_check,
+          `active` = :active,
+          `allow_from` = :allow_from
+            WHERE `access` = :access;");
+        $stmt->execute(array(
+          ':active' => $active,
+          ':skip_ip_check' => $skip_ip_check,
+          ':allow_from' => $allow_from,
+          ':access' => $access
+        ));
+      }
+      else {
+        $stmt = $pdo->prepare("UPDATE `api` SET `skip_ip_check` = :skip_ip_check,
+          `active` = :active
+            WHERE `access` = :access;");
+        $stmt->execute(array(
+          ':active' => $active,
+          ':skip_ip_check' => $skip_ip_check,
+          ':access' => $access
+        ));
+      }
+    }
+  }
+  elseif ($action == "regen_key") {
+    $api_key = implode('-', array(
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3))),
+      strtoupper(bin2hex(random_bytes(3)))
+    ));
+    $stmt = $pdo->prepare("UPDATE `api` SET `api_key` = :api_key WHERE `access` = :access");
+    $stmt->execute(array(
+      ':api_key' => $api_key,
+      ':access' => $access
+    ));
+  }
+  elseif ($action == "get") {
+    $stmt = $pdo->query("SELECT * FROM `api` WHERE `access` = '" . $access . "'");
+    $apidata = $stmt->fetch(PDO::FETCH_ASSOC);
+    if ($apidata !== false) {
+      $apidata['allow_from'] = str_replace(',', PHP_EOL, $apidata['allow_from']);
+    }
+    return $apidata;
+  }
+  $_SESSION['return'][] =  array(
+    'type' => 'success',
+    'log' => array(__FUNCTION__, $data),
+    'msg' => 'admin_api_modified'
+  );
+}
 function license($action, $data = null) {
   global $pdo;
   global $redis;
@@ -2365,7 +2482,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
   $data_log = $_data;
   if (isset($data_log['client_secret'])) $data_log['client_secret'] = '*';
   if (isset($data_log['access_token'])) $data_log['access_token'] = '*';
-  if (isset($data_log['bindpass'])) $data_log['bindpass'] = '*';
 
   switch ($_action) {
     case 'get':
@@ -2376,12 +2492,9 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       foreach($rows as $row){
         switch ($row["key"]) {
           case "redirect_url_extra":
-            $settings[$row["key"]] = json_decode($row["value"]);
-          break;
           case "mappers":
           case "templates":
-            // Parallel lists; older rows can be JSON objects, decode as arrays to keep the keys aligned
-            $settings[$row["key"]] = (array)json_decode($row["value"], true);
+            $settings[$row["key"]] = json_decode($row["value"]);
           break;
           case "use_ssl":
           case "use_tls":
@@ -2464,13 +2577,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
 
       $_data['ignore_ssl_error']    = isset($_data['ignore_ssl_error']) ? boolval($_data['ignore_ssl_error']) : false;
       $_data['login_provisioning']  = isset($_data['login_provisioning']) ? boolval($_data['login_provisioning']) : false;
-      // Identity sync: offboarding and groups (Keycloak and LDAP)
-      $sync_settings = array('offboarding', 'offboarding_limit', 'group_sync', 'group_autocreate', 'group_default_domain');
-      $_data['offboarding']          = isset($_data['offboarding']) ? intval($_data['offboarding']) : 0;
-      $_data['offboarding_limit']    = isset($_data['offboarding_limit']) && is_numeric($_data['offboarding_limit']) ? min(100, max(0, intval($_data['offboarding_limit']))) : 10;
-      $_data['group_sync']           = isset($_data['group_sync']) ? intval($_data['group_sync']) : 0;
-      $_data['group_autocreate']     = isset($_data['group_autocreate']) ? intval($_data['group_autocreate']) : 0;
-      $_data['group_default_domain'] = isset($_data['group_default_domain']) ? strtolower(trim($_data['group_default_domain'])) : "";
       switch ($_data['authsource']) {
         case "keycloak":
           $_data['server_url']        = (!empty($_data['server_url'])) ? rtrim($_data['server_url'], '/') : null;
@@ -2479,8 +2585,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $_data['import_users']      = isset($_data['import_users']) ? intval($_data['import_users']) : 0;
           $_data['sync_interval']     = (!empty($_data['sync_interval'])) ? intval($_data['sync_interval']) : 15;
           $_data['sync_interval']     = $_data['sync_interval'] < 1 ? 1 : $_data['sync_interval'];
-          $_data['group_paths']       = isset($_data['group_paths']) ? trim($_data['group_paths']) : "";
-          $required_settings          = array_merge(array('authsource', 'server_url', 'realm', 'client_id', 'client_secret', 'redirect_url', 'version', 'mailpassword_flow', 'periodic_sync', 'import_users', 'sync_interval', 'ignore_ssl_error', 'login_provisioning', 'group_paths'), $sync_settings);
+          $required_settings          = array('authsource', 'server_url', 'realm', 'client_id', 'client_secret', 'redirect_url', 'version', 'mailpassword_flow', 'periodic_sync', 'import_users', 'sync_interval', 'ignore_ssl_error', 'login_provisioning');
         break;
         case "generic-oidc":
           $_data['authorize_url']     = (!empty($_data['authorize_url'])) ? $_data['authorize_url'] : null;
@@ -2501,14 +2606,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $_data['use_tls']           = isset($_data['use_tls']) && !$_data['use_ssl'] ? boolval($_data['use_tls']) : false;
           $_data['sync_interval']     = (!empty($_data['sync_interval'])) ? intval($_data['sync_interval']) : 15;
           $_data['sync_interval']     = $_data['sync_interval'] < 1 ? 1 : $_data['sync_interval'];
-          $_data['id_attribute']      = isset($_data['id_attribute']) ? strtolower(trim($_data['id_attribute'])) : "";
-          $_data['disabled_filter']   = isset($_data['disabled_filter']) ? trim($_data['disabled_filter']) : "";
-          $_data['group_basedn']      = isset($_data['group_basedn']) ? trim($_data['group_basedn']) : "";
-          $_data['group_filter']      = isset($_data['group_filter']) ? trim($_data['group_filter']) : "";
-          $_data['group_name_attribute'] = (!empty($_data['group_name_attribute'])) ? strtolower(trim($_data['group_name_attribute'])) : "cn";
-          $_data['group_member_mode'] = isset($_data['group_member_mode']) && in_array($_data['group_member_mode'], array('memberof', 'member', 'ad_chain')) ? $_data['group_member_mode'] : "memberof";
-          $_data['group_member_attribute'] = (!empty($_data['group_member_attribute'])) ? strtolower(trim($_data['group_member_attribute'])) : "member";
-          $required_settings          = array_merge(array('authsource', 'host', 'port', 'basedn', 'username_field', 'filter', 'attribute_field', 'binddn', 'bindpass', 'periodic_sync', 'import_users', 'sync_interval', 'use_ssl', 'use_tls', 'ignore_ssl_error', 'login_provisioning', 'id_attribute', 'disabled_filter', 'group_basedn', 'group_filter', 'group_name_attribute', 'group_member_mode', 'group_member_attribute'), $sync_settings);
+          $required_settings          = array('authsource', 'host', 'port', 'basedn', 'username_field', 'filter', 'attribute_field', 'binddn', 'bindpass', 'periodic_sync', 'import_users', 'sync_interval', 'use_ssl', 'use_tls', 'ignore_ssl_error', 'login_provisioning');
         break;
       }
 
@@ -2530,10 +2628,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         $stmt->bindParam(':value', $_data[$setting]);
         $stmt->execute();
       }
-      // Stored LDAP ids of another id attribute never match again, the mailboxes are linked by email again
-      if ($_data['authsource'] == 'ldap' && ($iam_settings['id_attribute'] ?? '') != $_data['id_attribute']) {
-        $pdo->query("UPDATE `identity_link` SET `external_id` = NULL WHERE `provider` = 'ldap'");
-      }
       $pdo->commit();
 
       // add redirect_url_extra
@@ -2561,26 +2655,19 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         $_data['mappers'] = (!is_array($_data['mappers'])) ? array($_data['mappers']) : $_data['mappers'];
         $_data['templates'] = (!is_array($_data['templates'])) ? array($_data['templates']) : $_data['templates'];
 
-        // Keep mapper/template pairs together, incomplete rows are dropped
-        $mappers = array();
-        $templates = array();
-        $template_values = array_values($_data['templates']);
-        foreach (array_values($_data['mappers']) as $i => $mapper) {
-          if ((string)$mapper === '' || (string)($template_values[$i] ?? '') === '') {
-            continue;
-          }
-          $mappers[] = $mapper;
-          $templates[] = $template_values[$i];
-        }
-        $mappers = json_encode($mappers);
-        $templates = json_encode($templates);
+        $mappers = array_filter($_data['mappers']);
+        $templates = array_filter($_data['templates']);
+        if (count($mappers) == count($templates)){
+          $mappers = json_encode($mappers);
+          $templates = json_encode($templates);
 
-        $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('mappers', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
-        $stmt->bindParam(':value', $mappers);
-        $stmt->execute();
-        $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('templates', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
-        $stmt->bindParam(':value', $templates);
-        $stmt->execute();
+          $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('mappers', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
+          $stmt->bindParam(':value', $mappers);
+          $stmt->execute();
+          $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('templates', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
+          $stmt->bindParam(':value', $templates);
+          $stmt->execute();
+        }
       }
 
       // delete old access_token
@@ -2705,8 +2792,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       }
 
       $stmt = $pdo->query("DELETE FROM identity_provider;");
-      $stmt = $pdo->query("DELETE FROM identity_link;");
-      $stmt = $pdo->query("DELETE FROM identity_sync_run;");
 
       $_SESSION['return'][] =  array(
         'type' => 'success',
@@ -2827,8 +2912,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       }
 
       // get mapped template
-      $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
-      $mapped_template = $idp_login->mappedTemplate($info['mailcow_template'] ?? array());
+      $user_template = $info['mailcow_template'];
+      $mapper_key = array_search($user_template, $iam_settings['mappers']);
 
       // token valid, get mailbox
       $stmt = $pdo->prepare("SELECT
@@ -2850,23 +2935,13 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           );
           return false;
         }
-        if (!$idp_login->verifyLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
-          clear_session();
-          $_SESSION['return'][] =  array(
-            'type' => 'danger',
-            'log' => array(__FUNCTION__, $info['email'], 'The mailbox is linked to another IdP user'),
-            'msg' => 'login_failed'
-          );
-          return false;
-        }
-        $update_template = $idp_login->templateFor($info['email'], $mapped_template);
-        if ($update_template !== null) {
+        if ($mapper_key !== false) {
           // update user
           $_SESSION['access_all_exception'] = '1';
           mailbox('edit', 'mailbox_from_template', array(
             'username' => $info['email'],
             'name' => $info['name'],
-            'template' => $update_template
+            'template' => $iam_settings['templates'][$mapper_key]
           ));
           $_SESSION['access_all_exception'] = '0';
 
@@ -2881,11 +2956,11 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $stmt->execute(array(':user' => $info['email']));
           $row = $stmt->fetch(PDO::FETCH_ASSOC);
         }
-        if ($row['active'] != 1 || $row['d_active'] != 1 || (json_decode($row['attributes'], true)['shared'] ?? '0') == '1') {
+        if ($row['active'] != 1 || $row['d_active'] != 1) {
           clear_session();
           $_SESSION['return'][] =  array(
             'type' => 'danger',
-            'log' => array(__FUNCTION__, $info['email'], 'Domain or mailbox is inactive or shared'),
+            'log' => array(__FUNCTION__, $info['email'], 'Domain or mailbox is inactive'),
             'msg' => 'login_failed'
           );
           return false;
@@ -2911,7 +2986,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
-      if ($mapped_template === null){
+      if (empty($iam_settings['mappers']) || empty($user_template) || $mapper_key === false){
         if (!empty($iam_settings['default_template'])) {
           $mbox_template = $iam_settings['default_template'];
         } else {
@@ -2924,17 +2999,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           return false;
         }
       } else {
-        $mbox_template = $mapped_template;
-      }
-
-      if (!$idp_login->mayCreate($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
-        clear_session();
-        $_SESSION['return'][] =  array(
-          'type' => 'danger',
-          'log' => array(__FUNCTION__, $info['email'], 'The IdP user is linked to another mailbox'),
-          'msg' => 'login_failed'
-        );
-        return false;
+        $mbox_template = $iam_settings['templates'][$mapper_key];
       }
 
       // create mailbox
@@ -2956,7 +3021,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         );
         return false;
       }
-      $idp_login->storeLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''));
 
       // double check if mailbox and domain is active
       $stmt = $pdo->prepare("SELECT * FROM `mailbox`
@@ -3013,26 +3077,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
-      // The mailbox may have been deleted or deactivated since the login
-      $stmt = $pdo->prepare("SELECT `mailbox`.`authsource` FROM `mailbox`
-        INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
-        WHERE `kind` NOT REGEXP 'location|thing|group'
-          AND `mailbox`.`active` = '1'
-          AND `domain`.`active` = '1'
-          AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
-          AND `username` = :user");
-      $stmt->execute(array(':user' => $info['email']));
-      $row = $stmt->fetch(PDO::FETCH_ASSOC);
-      if (!$row || !in_array($row['authsource'], array('keycloak', 'generic-oidc'))) {
-        clear_session();
-        $_SESSION['return'][] =  array(
-          'type' => 'danger',
-          'log' => array(__FUNCTION__, $info['email'], 'Mailbox or domain is inactive or not of type OIDC'),
-          'msg' => 'refresh_login_failed'
-        );
-        return false;
-      }
-
       set_user_loggedin_session($info['email']);
       $_SESSION['iam_token'] = $plain_token;
       $_SESSION['iam_refresh_token'] = $plain_refreshtoken;

data/web/inc/functions.mailbox.inc.php

@@ -302,6 +302,192 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             'msg' => array('mailbox_modified', $username)
           );
         break;
+        case 'syncjob':
+          if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          if (isset($_data['username']) && filter_var($_data['username'], FILTER_VALIDATE_EMAIL)) {
+            if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data['username'])) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              return false;
+            }
+            else {
+              $username = $_data['username'];
+            }
+          }
+          elseif ($_SESSION['mailcow_cc_role'] == "user") {
+            $username = $_SESSION['mailcow_cc_username'];
+          }
+          else {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'no_user_defined'
+            );
+            return false;
+          }
+          $active               = intval($_data['active']);
+          $subscribeall         = intval($_data['subscribeall']);
+          $delete2duplicates    = intval($_data['delete2duplicates']);
+          $delete1              = intval($_data['delete1']);
+          $delete2              = intval($_data['delete2']);
+          $timeout1             = intval($_data['timeout1']);
+          $timeout2             = intval($_data['timeout2']);
+          $skipcrossduplicates  = intval($_data['skipcrossduplicates']);
+          $automap              = intval($_data['automap']);
+          $dry                  = intval($_data['dry']);
+          $port1                = $_data['port1'];
+          $host1                = strtolower($_data['host1']);
+          $password1            = $_data['password1'];
+          $exclude              = $_data['exclude'];
+          $maxage               = $_data['maxage'];
+          $maxbytespersecond    = $_data['maxbytespersecond'];
+          $subfolder2           = $_data['subfolder2'];
+          $user1                = $_data['user1'];
+          $mins_interval        = $_data['mins_interval'];
+          $enc1                 = $_data['enc1'];
+          $custom_params        = (empty(trim($_data['custom_params']))) ? '' : trim($_data['custom_params']);
+
+          // validate custom params
+          foreach (explode('-', $custom_params) as $param){
+            if(empty($param)) continue;
+
+            // extract option
+            if (str_contains($param, '=')) $param = explode('=', $param)[0];
+            else $param = rtrim($param, ' ');
+            // remove first char if first char is -
+            if ($param[0] == '-') $param = ltrim($param, $param[0]);
+
+            if (str_contains($param, ' ')) {
+              // bad char
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'bad character SPACE'
+              );
+              return false;
+            }
+
+            // check if param is whitelisted
+            if (!in_array(strtolower($param), $GLOBALS["IMAPSYNC_OPTIONS"]["whitelist"])){
+              // bad option
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'bad option '. $param
+              );
+              return false;
+            }
+          }
+          if (empty($subfolder2)) {
+            $subfolder2 = "";
+          }
+          if (!isset($maxage) || !filter_var($maxage, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+            $maxage = "0";
+          }
+          if (!isset($timeout1) || !filter_var($timeout1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+            $timeout1 = "600";
+          }
+          if (!isset($timeout2) || !filter_var($timeout2, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+            $timeout2 = "600";
+          }
+          if (!isset($maxbytespersecond) || !filter_var($maxbytespersecond, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 125000000)))) {
+            $maxbytespersecond = "0";
+          }
+          if (!filter_var($port1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 65535)))) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          if (!filter_var($mins_interval, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 43800)))) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          // if (!is_valid_domain_name($host1)) {
+            // $_SESSION['return'][] = array(
+              // 'type' => 'danger',
+              // 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              // 'msg' => 'access_denied'
+            // );
+            // return false;
+          // }
+          if ($enc1 != "TLS" && $enc1 != "SSL" && $enc1 != "PLAIN") {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          if (@preg_match("/" . $exclude . "/", null) === false) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          $stmt = $pdo->prepare("SELECT '1' FROM `imapsync`
+            WHERE `user2` = :user2 AND `user1` = :user1 AND `host1` = :host1");
+          $stmt->execute(array(':user1' => $user1, ':user2' => $username, ':host1' => $host1));
+          $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
+          if ($num_results != 0) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => array('object_exists', htmlspecialchars($host1 . ' / ' . $user1))
+            );
+            return false;
+          }
+          $stmt = $pdo->prepare("INSERT INTO `imapsync` (`user2`, `exclude`, `delete1`, `delete2`, `timeout1`, `timeout2`, `automap`, `skipcrossduplicates`, `maxbytespersecond`, `subscribeall`, `dry`, `maxage`, `subfolder2`, `host1`, `authmech1`, `user1`, `password1`, `mins_interval`, `port1`, `enc1`, `delete2duplicates`, `custom_params`, `active`)
+            VALUES (:user2, :exclude, :delete1, :delete2, :timeout1, :timeout2, :automap, :skipcrossduplicates, :maxbytespersecond, :subscribeall, :dry, :maxage, :subfolder2, :host1, :authmech1, :user1, :password1, :mins_interval, :port1, :enc1, :delete2duplicates, :custom_params, :active)");
+          $stmt->execute(array(
+            ':user2' => $username,
+            ':custom_params' => $custom_params,
+            ':exclude' => $exclude,
+            ':maxage' => $maxage,
+            ':delete1' => $delete1,
+            ':delete2' => $delete2,
+            ':timeout1' => $timeout1,
+            ':timeout2' => $timeout2,
+            ':automap' => $automap,
+            ':skipcrossduplicates' => $skipcrossduplicates,
+            ':maxbytespersecond' => $maxbytespersecond,
+            ':subscribeall' => $subscribeall,
+            ':dry' => $dry,
+            ':subfolder2' => $subfolder2,
+            ':host1' => $host1,
+            ':authmech1' => 'PLAIN',
+            ':user1' => $user1,
+            ':password1' => $password1,
+            ':mins_interval' => $mins_interval,
+            ':port1' => $port1,
+            ':enc1' => $enc1,
+            ':delete2duplicates' => $delete2duplicates,
+            ':active' => $active,
+          ));
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+            'msg' => array('mailbox_modified', $username)
+          );
+        break;
         case 'domain':
           if ($_SESSION['mailcow_cc_role'] != "admin") {
             $_SESSION['return'][] = array(
@@ -938,8 +1124,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $eas_access = (isset($_data['eas_access'])) ? intval($_data['eas_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['eas_access']);
           $dav_access = (isset($_data['dav_access'])) ? intval($_data['dav_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['dav_access']);
           $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : 0;
-          // Shared mailbox: no login of its own, members open it through group shares
-          $shared = (isset($_data['shared'])) ? intval($_data['shared']) : 0;
           $quarantine_notification = (isset($_data['quarantine_notification'])) ? strval($_data['quarantine_notification']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_notification']);
           $quarantine_category = (isset($_data['quarantine_category'])) ? strval($_data['quarantine_category']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_category']);
           // Validate quarantine_category
@@ -973,7 +1157,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'eas_access' => strval($eas_access),
               'dav_access' => strval($dav_access),
               'relayhost' => strval($relayhost),
-              'shared' => strval($shared),
               'passwd_update' => time(),
               'mailbox_format' => strval($MAILBOX_DEFAULT_ATTRIBUTES['mailbox_format']),
               'quarantine_notification' => strval($quarantine_notification),
@@ -1053,11 +1236,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             );
             return false;
           }
-          if ($authsource == 'mailcow' && $shared == 1 && empty($password) && empty($password2)) {
-            // Nobody logs in to a shared mailbox, a random password keeps it unusable
-            $password_hashed = hash_password(bin2hex(random_bytes(32)));
-          }
-          elseif ($authsource == 'mailcow'){
+          if ($authsource == 'mailcow'){
             if (password_check($password, $password2) !== true) {
               return false;
             }
@@ -1275,16 +1454,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             }
           }
 
-          $result = mailbox('add', 'mailbox', $mailbox_attributes);
-          if ($result !== false) {
-            // Remember the applied template, see edit mailbox_from_template
-            $stmt = $pdo->prepare("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.template', :template) WHERE `username` = :username");
-            $stmt->execute(array(
-              ':template' => $_data['template'],
-              ':username' => strtolower(trim($_data['local_part'])) . '@' . idn_to_ascii(strtolower(trim($_data['domain'])), 0, INTL_IDNA_VARIANT_UTS46)
-            ));
-          }
-          return $result;
+          return mailbox('add', 'mailbox', $mailbox_attributes);
         break;
         case 'mta_sts':
           $domain       = idn_to_ascii(strtolower(trim($_data['domain'])), 0, INTL_IDNA_VARIANT_UTS46);
@@ -2110,6 +2280,202 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             );
           }
         break;
+        case 'syncjob':
+          if (!is_array($_data['id'])) {
+            $ids = array();
+            $ids[] = $_data['id'];
+          }
+          else {
+            $ids = $_data['id'];
+          }
+          if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          foreach ($ids as $id) {
+            $is_now = mailbox('get', 'syncjob_details', $id, array('with_password'));
+            if (!empty($is_now)) {
+              $username = $is_now['user2'];
+              $user1 = (!empty($_data['user1'])) ? $_data['user1'] : $is_now['user1'];
+              $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];
+              $last_run = (isset($_data['last_run'])) ? NULL : $is_now['last_run'];
+              $success = (isset($_data['success'])) ? NULL : $is_now['success'];
+              $delete2duplicates = (isset($_data['delete2duplicates'])) ? intval($_data['delete2duplicates']) : $is_now['delete2duplicates'];
+              $subscribeall = (isset($_data['subscribeall'])) ? intval($_data['subscribeall']) : $is_now['subscribeall'];
+              $dry = (isset($_data['dry'])) ? intval($_data['dry']) : $is_now['dry'];
+              $delete1 = (isset($_data['delete1'])) ? intval($_data['delete1']) : $is_now['delete1'];
+              $delete2 = (isset($_data['delete2'])) ? intval($_data['delete2']) : $is_now['delete2'];
+              $automap = (isset($_data['automap'])) ? intval($_data['automap']) : $is_now['automap'];
+              $skipcrossduplicates = (isset($_data['skipcrossduplicates'])) ? intval($_data['skipcrossduplicates']) : $is_now['skipcrossduplicates'];
+              $port1 = (!empty($_data['port1'])) ? $_data['port1'] : $is_now['port1'];
+              $password1 = (!empty($_data['password1'])) ? $_data['password1'] : $is_now['password1'];
+              $host1 = (!empty($_data['host1'])) ? $_data['host1'] : $is_now['host1'];
+              $subfolder2 = (isset($_data['subfolder2'])) ? $_data['subfolder2'] : $is_now['subfolder2'];
+              $enc1 = (!empty($_data['enc1'])) ? $_data['enc1'] : $is_now['enc1'];
+              $mins_interval = (!empty($_data['mins_interval'])) ? $_data['mins_interval'] : $is_now['mins_interval'];
+              $exclude = (isset($_data['exclude'])) ? $_data['exclude'] : $is_now['exclude'];
+              $custom_params = (isset($_data['custom_params'])) ? $_data['custom_params'] : $is_now['custom_params'];
+              $maxage = (isset($_data['maxage']) && $_data['maxage'] != "") ? intval($_data['maxage']) : $is_now['maxage'];
+              $maxbytespersecond = (isset($_data['maxbytespersecond']) && $_data['maxbytespersecond'] != "") ? intval($_data['maxbytespersecond']) : $is_now['maxbytespersecond'];
+              $timeout1 = (isset($_data['timeout1']) && $_data['timeout1'] != "") ? intval($_data['timeout1']) : $is_now['timeout1'];
+              $timeout2 = (isset($_data['timeout2']) && $_data['timeout2'] != "") ? intval($_data['timeout2']) : $is_now['timeout2'];
+            }
+            else {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+
+            // validate custom params
+            foreach (explode('-', $custom_params) as $param){
+              if(empty($param)) continue;
+
+              // extract option
+              if (str_contains($param, '=')) $param = explode('=', $param)[0];
+              else $param = rtrim($param, ' ');
+              // remove first char if first char is -
+              if ($param[0] == '-') $param = ltrim($param, $param[0]);
+
+              if (str_contains($param, ' ')) {
+                // bad char
+                $_SESSION['return'][] = array(
+                  'type' => 'danger',
+                  'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                  'msg' => 'bad character SPACE'
+                );
+                return false;
+              }
+
+              // check if param is whitelisted
+              if (!in_array(strtolower($param), $GLOBALS["IMAPSYNC_OPTIONS"]["whitelist"])){
+                // bad option
+                $_SESSION['return'][] = array(
+                  'type' => 'danger',
+                  'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                  'msg' => 'bad option '. $param
+                );
+                return false;
+              }
+            }
+            if (empty($subfolder2)) {
+              $subfolder2 = "";
+            }
+            if (!isset($maxage) || !filter_var($maxage, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+              $maxage = "0";
+            }
+            if (!isset($timeout1) || !filter_var($timeout1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+              $timeout1 = "600";
+            }
+            if (!isset($timeout2) || !filter_var($timeout2, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) {
+              $timeout2 = "600";
+            }
+            if (!isset($maxbytespersecond) || !filter_var($maxbytespersecond, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 125000000)))) {
+              $maxbytespersecond = "0";
+            }
+            if (!filter_var($port1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 65535)))) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            if (!filter_var($mins_interval, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 43800)))) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            if (!is_valid_domain_name($host1)) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            if ($enc1 != "TLS" && $enc1 != "SSL" && $enc1 != "PLAIN") {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            if (@preg_match("/" . $exclude . "/", null) === false) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            $stmt = $pdo->prepare("UPDATE `imapsync` SET `delete1` = :delete1,
+              `delete2` = :delete2,
+              `automap` = :automap,
+              `skipcrossduplicates` = :skipcrossduplicates,
+              `maxage` = :maxage,
+              `maxbytespersecond` = :maxbytespersecond,
+              `subfolder2` = :subfolder2,
+              `exclude` = :exclude,
+              `host1` = :host1,
+              `last_run` = :last_run,
+              `success` = :success,
+              `user1` = :user1,
+              `password1` = :password1,
+              `mins_interval` = :mins_interval,
+              `port1` = :port1,
+              `enc1` = :enc1,
+              `delete2duplicates` = :delete2duplicates,
+              `custom_params` = :custom_params,
+              `timeout1` = :timeout1,
+              `timeout2` = :timeout2,
+              `subscribeall` = :subscribeall,
+              `dry` = :dry,
+              `active` = :active
+                WHERE `id` = :id");
+            $stmt->execute(array(
+              ':delete1' => $delete1,
+              ':delete2' => $delete2,
+              ':automap' => $automap,
+              ':skipcrossduplicates' => $skipcrossduplicates,
+              ':id' => $id,
+              ':exclude' => $exclude,
+              ':maxage' => $maxage,
+              ':maxbytespersecond' => $maxbytespersecond,
+              ':subfolder2' => $subfolder2,
+              ':host1' => $host1,
+              ':user1' => $user1,
+              ':password1' => $password1,
+              ':last_run' => $last_run,
+              ':success' => $success,
+              ':mins_interval' => $mins_interval,
+              ':port1' => $port1,
+              ':enc1' => $enc1,
+              ':delete2duplicates' => $delete2duplicates,
+              ':custom_params' => $custom_params,
+              ':timeout1' => $timeout1,
+              ':timeout2' => $timeout2,
+              ':subscribeall' => $subscribeall,
+              ':dry' => $dry,
+              ':active' => $active,
+            ));
+            $_SESSION['return'][] = array(
+              'type' => 'success',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => array('mailbox_modified', $username)
+            );
+          }
+        break;
         case 'filter':
           if (!isset($_SESSION['acl']['filters']) || $_SESSION['acl']['filters'] != "1" ) {
             $_SESSION['return'][] = array(
@@ -2224,14 +2590,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
-            if (!empty($is_now['group_id'])) {
-              $_SESSION['return'][] = array(
-                'type' => 'danger',
-                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                'msg' => array('alias_group_managed', htmlspecialchars($is_now['address']))
-              );
-              continue;
-            }
             if ($_data['expand_alias'] === true || $_data['expand_alias'] == 1) {
               $stmt = $pdo->prepare("SELECT `address` FROM `alias`
                 WHERE `address` = :address
@@ -2394,8 +2752,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 // Delete from sender_acl to prevent duplicates
                 $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE
                   `logged_in_as` = :goto AND
-                  `send_as` = :address AND
-                  `group_id` IS NULL");
+                  `send_as` = :address");
                 $stmt->execute(array(
                   ':goto' => $goto,
                   ':address' => $address
@@ -2429,10 +2786,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':active' => $active,
                 ':id' => $is_now['id']
               ));
-              if ($address != $is_now['address']) {
-                // Send-as grants of groups follow the address
-                \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
-              }
             }
             $_SESSION['return'][] = array(
               'type' => 'success',
@@ -2814,7 +3167,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               (int)$eas_access     = (isset($_data['eas_access']) && hasACLAccess("protocol_access")) ? intval($_data['eas_access']) : intval($is_now['attributes']['eas_access']);
               (int)$dav_access    = (isset($_data['dav_access']) && hasACLAccess("protocol_access")) ? intval($_data['dav_access']) : intval($is_now['attributes']['dav_access']);
               (int)$relayhost       = (isset($_data['relayhost']) && hasACLAccess("mailbox_relayhost")) ? intval($_data['relayhost']) : intval($is_now['attributes']['relayhost']);
-              (int)$shared          = (isset($_data['shared'])) ? intval($_data['shared']) : intval($is_now['attributes']['shared'] ?? 0);
               (int)$quota_m         = (isset_has_content($_data['quota'])) ? intval($_data['quota']) : ($is_now['quota'] / 1048576);
               $name                 = (!empty($_data['name'])) ? ltrim(rtrim($_data['name'], '>'), '<') : $is_now['name'];
               $domain               = $is_now['domain'];
@@ -3037,7 +3389,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $sender_acl_merged = array_merge($sender_acl_domain_admin, $sender_acl_admin);
                 // If merged array still contains "*", set it as only value
                 !in_array('*', $sender_acl_merged) ?: $sender_acl_merged = array('*');
-                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `group_id` IS NULL AND `logged_in_as` = :username");
+                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `logged_in_as` = :username");
                 $stmt->execute(array(
                   ':username' => $username
                 ));
@@ -3069,7 +3421,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 }
               }
               else {
-                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `group_id` IS NULL AND `logged_in_as` = :username");
+                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `logged_in_as` = :username");
                 $stmt->execute(array(
                   ':username' => $username
                 ));
@@ -3110,7 +3462,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   `attributes` = JSON_SET(`attributes`, '$.sieve_access', :sieve_access),
                   `attributes` = JSON_SET(`attributes`, '$.pop3_access', :pop3_access),
                   `attributes` = JSON_SET(`attributes`, '$.relayhost', :relayhost),
-                  `attributes` = JSON_SET(`attributes`, '$.shared', :shared),
                   `attributes` = JSON_SET(`attributes`, '$.smtp_access', :smtp_access),
                   `attributes` = JSON_SET(`attributes`, '$.eas_access', :eas_access),
                   `attributes` = JSON_SET(`attributes`, '$.dav_access', :dav_access),
@@ -3133,7 +3484,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':dav_access' => $dav_access,
                 ':recovery_email' => $pw_recovery_email,
                 ':relayhost' => $relayhost,
-                ':shared' => strval($shared),
                 ':username' => $username,
                 ':authsource' => $authsource
               ));
@@ -3290,12 +3640,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               "imapsync" => ['user2'],
               'bcc_maps' => ['local_dest', 'bcc_dest'],
               'recipient_maps' => ['old_dest', 'new_dest'],
-              'sender_acl' => ['logged_in_as', 'send_as'],
-              'identity_link' => ['username'],
-              'mailbox_group_imap_grant' => ['owner'],
-              'mailbox_group_sogo_grant' => ['owner'],
-              'mailbox_group_sogo_acl' => ['c_uid'],
-              'mailbox_group_subscription' => ['username', 'owner']
+              'sender_acl' => ['logged_in_as', 'send_as']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3307,11 +3652,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               }
             }
 
-            // Sync sources created by / targeting the mailbox and its OAuth tokens (owner); a PDOException rolls back below
-            \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
-            // Group memberships and send-as grants follow, group aliases are rebuilt
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
-
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
               ->execute([
@@ -3409,27 +3749,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attribute_hash = sha1(json_encode($mbox_template_data["attributes"]));
           $is_now = mailbox('get', 'mailbox_details', $_data['username']);
           $name = ltrim(rtrim($_data['name'], '>'), '<');
-          // Last applied template, null for mailboxes created before it was recorded
-          $applied_template = $is_now['attributes']['template'] ?? null;
-          $stmt_template = $pdo->prepare("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.template', :template) WHERE `username` = :username");
-          if ($is_now['attributes']['attribute_hash'] == $attribute_hash && $is_now['name'] == $name) {
-            if ($applied_template === null) {
-              $stmt_template->execute(array(':template' => $_data['template'], ':username' => $_data['username']));
-            }
+          if ($is_now['attributes']['attribute_hash'] == $attribute_hash && $is_now['name'] == $name)
             return true;
-          }
 
           $mbox_template_data = json_decode($mbox_template_data["attributes"], true);
-          // Any manual edit clears attribute_hash: re-applying the same template must not undo a manual (de)activation
-          if ($applied_template === null || $applied_template === $_data['template']) {
-            unset($mbox_template_data['active']);
-          }
-          // Login disabled by the IdP sync is only lifted by the sync
-          $stmt = $pdo->prepare("SELECT `idp_disabled` FROM `identity_link` WHERE `username` = :username");
-          $stmt->execute(array(':username' => $_data['username']));
-          if (intval($stmt->fetchColumn()) == 1) {
-            unset($mbox_template_data['active']);
-          }
           $mbox_template_data['attribute_hash'] = $attribute_hash;
           $mbox_template_data['name'] = $name;
           $quarantine_attributes = array('username' => $_data['username']);
@@ -3472,7 +3795,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           if ($result === false) return $result;
           $result = acl('edit', 'user', $acl_attributes);
           if ($result === false) return $result;
-          $stmt_template->execute(array(':template' => $_data['template'], ':username' => $_data['username']));
 
           $_SESSION['return'] = array();
           return true;
@@ -3585,8 +3907,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ":template" => $_data["template"] ,
               ":attributes" => json_encode($attr)
             ));
-            // Mailboxes with this effective group template get the new settings
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->templateChanged((int)$id);
           }
 
 
@@ -4018,7 +4338,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
           // Return array $data['sender_acl_domains/addresses']['ro'] with read-only objects
           // Return array $data['sender_acl_domains/addresses']['rw'] with read-write objects (can be deleted)
-          $stmt = $pdo->prepare("SELECT REPLACE(`send_as`, '@', '') AS `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND `group_id` IS NULL AND (`send_as` LIKE '@%' OR `send_as` = '*')");
+          $stmt = $pdo->prepare("SELECT REPLACE(`send_as`, '@', '') AS `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND (`send_as` LIKE '@%' OR `send_as` = '*')");
           $stmt->execute(array(':logged_in_as' => $_data));
           $domain_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($domain_row = array_shift($domain_rows)) {
@@ -4037,7 +4357,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               $data['sender_acl_domains']['rw'][] = $domain_row['send_as'];
             }
           }
-          $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND `group_id` IS NULL AND (`send_as` NOT LIKE '@%' AND `send_as` != '*')");
+          $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND (`send_as` NOT LIKE '@%' AND `send_as` != '*')");
           $stmt->execute(array(':logged_in_as' => $_data));
           $address_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($address_row = array_shift($address_rows)) {
@@ -4283,6 +4603,68 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
           return false;
         break;
+        case 'syncjob_details':
+          $syncjobdetails = array();
+          if (!is_numeric($_data)) {
+            return false;
+          }
+          if (isset($_extra) && in_array('no_log', $_extra)) {
+            $field_query = $pdo->query('SHOW FIELDS FROM `imapsync` WHERE FIELD NOT IN ("returned_text", "password1")');
+            $fields = $field_query->fetchAll(PDO::FETCH_ASSOC);
+            while($field = array_shift($fields)) {
+              $shown_fields[] = $field['Field'];
+            }
+            $stmt = $pdo->prepare("SELECT " . implode(',', (array)$shown_fields) . ",
+              `active`
+                FROM `imapsync` WHERE id = :id");
+          }
+          elseif (isset($_extra) && in_array('with_password', $_extra)) {
+            $stmt = $pdo->prepare("SELECT *,
+              `active`
+                FROM `imapsync` WHERE id = :id");
+          }
+          else {
+            $field_query = $pdo->query('SHOW FIELDS FROM `imapsync` WHERE FIELD NOT IN ("password1")');
+            $fields = $field_query->fetchAll(PDO::FETCH_ASSOC);
+            while($field = array_shift($fields)) {
+              $shown_fields[] = $field['Field'];
+            }
+            $stmt = $pdo->prepare("SELECT " . implode(',', (array)$shown_fields) . ",
+              `active`
+                FROM `imapsync` WHERE id = :id");
+          }
+          $stmt->execute(array(':id' => $_data));
+          $syncjobdetails = $stmt->fetch(PDO::FETCH_ASSOC);
+          if (!empty($syncjobdetails['returned_text'])) {
+            $syncjobdetails['log'] = $syncjobdetails['returned_text'];
+          }
+          else {
+            $syncjobdetails['log'] = '';
+          }
+          unset($syncjobdetails['returned_text']);
+          if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $syncjobdetails['user2'])) {
+            return false;
+          }
+          return $syncjobdetails;
+        break;
+        case 'syncjobs':
+          $syncjobdata = array();
+          if (isset($_data) && filter_var($_data, FILTER_VALIDATE_EMAIL)) {
+            if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data)) {
+              return false;
+            }
+          }
+          else {
+            $_data = $_SESSION['mailcow_cc_username'];
+          }
+          $stmt = $pdo->prepare("SELECT `id` FROM `imapsync` WHERE `user2` = :username");
+          $stmt->execute(array(':username' => $_data));
+          $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
+          while($row = array_shift($rows)) {
+            $syncjobdata[] = $row['id'];
+          }
+          return $syncjobdata;
+        break;
         case 'spam_score':
           $curl = curl_init();
           curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, '/var/lib/rspamd/rspamd.sock');
@@ -4495,9 +4877,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             `sogo_visible`,
             `sender_allowed`,
             `created`,
-            `modified`,
-            `group_id`,
-            (SELECT `name` FROM `mailbox_group` WHERE `mailbox_group`.`id` = `alias`.`group_id`) AS `group_name`
+            `modified`
               FROM `alias`
                   WHERE (`id` = :id OR `address` = :address) AND `address` != `goto`");
           $stmt->execute(array(
@@ -4530,9 +4910,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $aliasdata['sogo_visible'] = $row['sogo_visible'];
           $aliasdata['sogo_visible_int'] = $row['sogo_visible'];
           $aliasdata['sender_allowed'] = $row['sender_allowed'];
-          // Distribution address of a mailbox group, read-only here
-          $aliasdata['group_id'] = $row['group_id'];
-          $aliasdata['group_name'] = $row['group_name'];
           $aliasdata['created'] = $row['created'];
           $aliasdata['modified'] = $row['modified'];
           if (!hasDomainAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $aliasdata['domain'])) {
@@ -4973,7 +5350,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $mailboxdata['is_relayed'] = $row['backupmx'];
             // Internal send-as ACL for this mailbox, mirrors the sender_acl field accepted by edit/mailbox
             $mailboxdata['sender_acl'] = array();
-            $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :username AND `external` = '0' AND `group_id` IS NULL");
+            $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :username AND `external` = '0'");
             $stmt->execute(array(':username' => $_data));
             $sender_acl_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
             foreach ($sender_acl_rows as $sender_acl_row) {
@@ -5117,6 +5494,46 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
     break;
     case 'delete':
       switch ($_type) {
+        case 'syncjob':
+          if (!is_array($_data['id'])) {
+            $ids = array();
+            $ids[] = $_data['id'];
+          }
+          else {
+            $ids = $_data['id'];
+          }
+          if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          foreach ($ids as $id) {
+            if (!is_numeric($id)) {
+              return false;
+            }
+            $stmt = $pdo->prepare("SELECT `user2` FROM `imapsync` WHERE id = :id");
+            $stmt->execute(array(':id' => $id));
+            $user2 = $stmt->fetch(PDO::FETCH_ASSOC)['user2'];
+            if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $user2)) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => 'access_denied'
+              );
+              continue;
+            }
+            $stmt = $pdo->prepare("DELETE FROM `imapsync` WHERE `id`= :id");
+            $stmt->execute(array(':id' => $id));
+            $_SESSION['return'][] = array(
+              'type' => 'success',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => array('deleted_syncjob', $id)
+            );
+          }
+        break;
         case 'filter':
           if (!is_array($_data['id'])) {
             $ids = array();
@@ -5347,11 +5764,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':domain' => $domain,
             ));
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
-            $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
-            $stmt->execute(array(
-              ':domain' => $domain,
-            ));
             $stmt = $pdo->prepare("DELETE FROM `domain_admins` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,
@@ -5475,14 +5887,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
-            if (!empty($alias_data['group_id'])) {
-              $_SESSION['return'][] = array(
-                'type' => 'danger',
-                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                'msg' => array('alias_group_managed', htmlspecialchars($alias_data['address']))
-              );
-              continue;
-            }
 
             // Track affected mailboxes for SOGo update (capture before deletion)
             if (!empty($alias_data['goto'])) {
@@ -5503,7 +5907,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':alias_address' => $alias_data['address']
             ));
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressDeleted($alias_data['address']);
             $_SESSION['return'][] = array(
               'type' => 'success',
               'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -5544,7 +5947,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':alias_domain' => $alias_domain,
             ));
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($alias_domain);
             $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `domain` = :alias_domain");
             $stmt->execute(array(
               ':alias_domain' => $alias_domain,
@@ -5627,8 +6029,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 'msg' => 'Could not move maildir to garbage collector: variables local_part and/or domain empty'
               );
             }
-            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->mailboxDeleted($username);
-            $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `goto` = :username AND `group_id` IS NULL");
+            $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `goto` = :username");
             $stmt->execute(array(
               ':username' => $username
             ));
@@ -5670,8 +6071,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':username' => $username
             ));
-            // Tokens, scope targets and own sync sources of the mailbox
-            \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username);
             $stmt = $pdo->prepare("DELETE FROM `filterconf` WHERE `object` = :username");
             $stmt->execute(array(
               ':username' => $username

data/web/inc/init_db.inc.php

@@ -4,20 +4,7 @@ function init_db_schema()
   try {
     global $pdo;
 
-    // Defined once as DB_VERSION in docker-compose.yml
-    $db_version = (string)getenv('DB_VERSION');
-    if ($db_version === '') {
-      if (php_sapi_name() == "cli") {
-        echo "DB initialization failed: DB_VERSION is not set" . PHP_EOL;
-      } else {
-        $_SESSION['return'][] = array(
-          'type' => 'danger',
-          'log' => array(__FUNCTION__),
-          'msg' => 'Database not initialized: DB_VERSION is not set.'
-        );
-      }
-      return false;
-    }
+    $db_version = "18082026_1200";
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -187,18 +174,19 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "api" => array(
+      "alias" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
-          "api_key" => "VARCHAR(255) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "allow_from" => "VARCHAR(512) NOT NULL",
-          "skip_ip_check" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "address" => "VARCHAR(255) NOT NULL",
+          "goto" => "TEXT NOT NULL",
+          "domain" => "VARCHAR(255) NOT NULL",
           "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE NOW(0)",
-          "access" => "ENUM('ro', 'rw') NOT NULL DEFAULT 'rw'",
-          // 1 = defined in mailcow.conf, read-only in UI and API
-          "service" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "private_comment" => "TEXT",
+          "public_comment" => "TEXT",
+          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
         ),
         "keys" => array(
@@ -206,7 +194,41 @@ function init_db_schema()
             "" => array("id")
           ),
           "unique" => array(
-            "api_key" => array("api_key")
+            "address" => array("address")
+          ),
+          "key" => array(
+            "domain" => array("domain")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "api" => array(
+        "cols" => array(
+          "api_key" => "VARCHAR(255) NOT NULL",
+          "allow_from" => "VARCHAR(512) NOT NULL",
+          "skip_ip_check" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE NOW(0)",
+          "access" => "ENUM('ro', 'rw') NOT NULL DEFAULT 'rw'",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("api_key")
+          ),
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "sender_acl" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "logged_in_as" => "VARCHAR(255) NOT NULL",
+          "send_as" => "VARCHAR(255) NOT NULL",
+          "external" => "TINYINT(1) NOT NULL DEFAULT '0'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
           )
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
@@ -389,360 +411,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "mailbox_group" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          // distribution address, maintained as alias row with group_id
-          "address" => "VARCHAR(255) NULL DEFAULT NULL",
-          // members may send as the address
-          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "template_id" => "INT NULL DEFAULT NULL",
-          // highest priority wins when a mailbox is in several groups with a template
-          "priority" => "INT NOT NULL DEFAULT '0'",
-          // IdP group id: set = memberships are synced from the IdP
-          "idp_ref" => "VARCHAR(255) NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "domain_name" => array("domain", "name"),
-            "address" => array("address"),
-            "idp_ref" => array("idp_ref")
-          ),
-          "fkey" => array(
-            "fk_mailbox_group_domain" => array(
-              "col" => "domain",
-              "ref" => "domain.domain",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mailbox_group_template" => array(
-              "col" => "template_id",
-              "ref" => "templates.id",
-              "delete" => "SET NULL",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "mailbox_group_member" => array(
-        "cols" => array(
-          "group_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL",
-          "source" => "ENUM('manual', 'idp') NOT NULL DEFAULT 'manual'",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("group_id", "username")
-          ),
-          "key" => array(
-            "username" => array("username")
-          ),
-          "fkey" => array(
-            "fk_mailbox_group_member_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mailbox_group_member_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "mailbox_group_send_as" => array(
-        "cols" => array(
-          "group_id" => "INT NOT NULL",
-          // address or @domain the members may send as
-          "address" => "VARCHAR(255) NOT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("group_id", "address")
-          ),
-          "key" => array(
-            "address" => array("address")
-          ),
-          "fkey" => array(
-            "fk_mailbox_group_send_as_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "mailbox_group_imap_grant" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "group_id" => "INT NOT NULL",
-          // mailbox whose folders the members may access
-          "owner" => "VARCHAR(255) NOT NULL",
-          "rights" => "ENUM('read', 'write', 'full') NOT NULL DEFAULT 'read'",
-          // 1 = the group-grants cron subscribes the members to the folders (once per member)
-          "subscribe" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // removing = the ACL is deleted in Dovecot, then the row
-          "state" => "ENUM('pending', 'applied', 'failed', 'removing') NOT NULL DEFAULT 'pending'",
-          "last_error" => "TEXT",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "group_owner" => array("group_id", "owner")
-          ),
-          "key" => array(
-            "owner" => array("owner")
-          ),
-          "fkey" => array(
-            "fk_mailbox_group_imap_grant_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mailbox_group_imap_grant_owner" => array(
-              "col" => "owner",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "mailbox_group_sogo_grant" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "group_id" => "INT NOT NULL",
-          "owner" => "VARCHAR(255) NOT NULL",
-          "folder_type" => "ENUM('Calendar', 'Contacts') NOT NULL DEFAULT 'Calendar'",
-          "folder" => "VARCHAR(255) NOT NULL DEFAULT 'personal'",
-          "access" => "ENUM('read', 'write') NOT NULL DEFAULT 'read'",
-          // 1 = the group-grants cron subscribes the members to the folder (once per member)
-          "subscribe" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // pending = the owner has not opened SOGo yet, the folder does not exist
-          "state" => "ENUM('pending', 'applied', 'failed') NOT NULL DEFAULT 'pending'",
-          "last_error" => "TEXT",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "group_folder" => array("group_id", "owner", "folder_type", "folder")
-          ),
-          "key" => array(
-            "owner" => array("owner")
-          ),
-          "fkey" => array(
-            "fk_mailbox_group_sogo_grant_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mailbox_group_sogo_grant_owner" => array(
-              "col" => "owner",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // sogo_acl rows written for a grant, so only those are ever removed again. No FK: rows of grants
-      // removed by a cascade stay until their sogo_acl rows are deleted (GroupRepository::cleanupSogoAcl())
-      "mailbox_group_sogo_acl" => array(
-        "cols" => array(
-          "acl_id" => "INT NOT NULL",
-          "grant_id" => "INT NOT NULL",
-          "c_uid" => "VARCHAR(255) NOT NULL",
-          "c_role" => "VARCHAR(80) NOT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("acl_id")
-          ),
-          "key" => array(
-            "grant_id" => array("grant_id")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // IMAP and SOGo subscriptions made by the group-grants cron. No FK: rows of grants removed by a cascade stay
-      // until the members are unsubscribed again
-      "mailbox_group_subscription" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "kind" => "ENUM('imap', 'sogo') NOT NULL",
-          // mailbox_group_imap_grant.id or mailbox_group_sogo_grant.id
-          "grant_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL",
-          "owner" => "VARCHAR(255) NOT NULL",
-          // SOGo: Calendar/<id> or Contacts/<id>, IMAP: empty (all folders of the owner)
-          "folder" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "grant_username" => array("kind", "grant_id", "username")
-          ),
-          "key" => array(
-            "username" => array("username")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "identity_link" => array(
-        "cols" => array(
-          "username" => "VARCHAR(255) NOT NULL",
-          "provider" => "VARCHAR(20) NOT NULL",
-          // Keycloak user id (= OIDC sub), LDAP objectGUID/entryUUID/... or dn:<dn>, OIDC sub; NULL = not linked yet
-          "external_id" => "VARCHAR(255) NULL DEFAULT NULL",
-          // 1 = the sync set the mailbox to login disabled (active = 2)
-          "idp_disabled" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // first complete sync run that did not list the user
-          "missing_since" => "DATETIME NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("username")
-          ),
-          "unique" => array(
-            "provider_external_id" => array("provider", "external_id")
-          ),
-          "fkey" => array(
-            "fk_identity_link_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "identity_sync_run" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "provider" => "VARCHAR(20) NOT NULL DEFAULT ''",
-          // run = scheduled or requested, preview = dry run, force = without the deactivation limit
-          "mode" => "ENUM('run', 'preview', 'force') NOT NULL DEFAULT 'run'",
-          "status" => "ENUM('pending', 'running', 'ok', 'blocked', 'error') NOT NULL DEFAULT 'pending'",
-          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "started" => "DATETIME NULL DEFAULT NULL",
-          "finished" => "DATETIME NULL DEFAULT NULL",
-          "summary" => "MEDIUMTEXT"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "status" => array("status")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // alias and sender_acl come after mailbox_group (foreign keys)
-      "alias" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "address" => "VARCHAR(255) NOT NULL",
-          "goto" => "TEXT NOT NULL",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "private_comment" => "TEXT",
-          "public_comment" => "TEXT",
-          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          // set = distribution address of a mailbox group, read-only in the alias UI
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "address" => array("address"),
-            "group_id" => array("group_id")
-          ),
-          "key" => array(
-            "domain" => array("domain")
-          ),
-          "fkey" => array(
-            "fk_alias_mailbox_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "sender_acl" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "logged_in_as" => "VARCHAR(255) NOT NULL",
-          "send_as" => "VARCHAR(255) NOT NULL",
-          "external" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // set = granted through a mailbox group, rebuilt by the group module
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "group_id" => array("group_id")
-          ),
-          "fkey" => array(
-            "fk_sender_acl_mailbox_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "sieve_filters" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -1052,8 +720,7 @@ function init_db_schema()
           "mailbox_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
+          "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1088,153 +755,35 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "imapsync_source" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "scope" => "ENUM('all','domain','user') NOT NULL DEFAULT 'user'",
-          "host1" => "VARCHAR(255) NOT NULL",
-          "port1" => "SMALLINT UNSIGNED NOT NULL",
-          "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'",
-          "auth_type" => "ENUM('PLAIN','LOGIN','CRAM-MD5','XOAUTH2') DEFAULT 'PLAIN'",
-          "oauth_flow" => "ENUM('client_credentials','authorization_code') NOT NULL DEFAULT 'client_credentials'",
-          "oauth_token_endpoint" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_authorize_endpoint" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_userinfo_endpoint" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_client_id" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_client_secret" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_scope" => "VARCHAR(500) DEFAULT NULL",
-          "oauth_extra_params" => "TEXT DEFAULT NULL",
-          "oauth_access_token" => "TEXT DEFAULT NULL",
-          "oauth_token_expires" => "BIGINT UNSIGNED DEFAULT NULL",
-          "oauth_last_refresh_error" => "TEXT DEFAULT NULL",
-          "oauth_last_refresh_attempt" => "DATETIME NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "uniq_creator_name" => array("created_by", "name")
-          ),
-          "key" => array(
-            "idx_created_by" => array("created_by"),
-            "idx_scope" => array("scope")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "imapsync_source_domain" => array(
-        "cols" => array(
-          "source_id" => "INT NOT NULL",
-          "domain" => "VARCHAR(255) NOT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("source_id", "domain")
-          ),
-          "key" => array(
-            "idx_domain" => array("domain")
-          ),
-          "fkey" => array(
-            "fk_imapsync_source_domain" => array(
-              "col" => "source_id",
-              "ref" => "imapsync_source.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "imapsync_source_user" => array(
-        "cols" => array(
-          "source_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("source_id", "username")
-          ),
-          "key" => array(
-            "idx_username" => array("username")
-          ),
-          "fkey" => array(
-            "fk_imapsync_source_user" => array(
-              "col" => "source_id",
-              "ref" => "imapsync_source.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "imapsync_source_oauth_token" => array(
-        "cols" => array(
-          "source_id" => "INT NOT NULL",
-          "owner" => "VARCHAR(255) NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL",
-          "access_token" => "TEXT DEFAULT NULL",
-          "refresh_token" => "TEXT DEFAULT NULL",
-          "token_expires" => "BIGINT UNSIGNED DEFAULT NULL",
-          "last_refresh_error" => "TEXT DEFAULT NULL",
-          "oauth_last_refresh_attempt" => "DATETIME NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("source_id", "owner", "username")
-          ),
-          "key" => array(
-            "idx_owner" => array("owner")
-          ),
-          "fkey" => array(
-            "fk_imapsync_source_oauth_token" => array(
-              "col" => "source_id",
-              "ref" => "imapsync_source.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "imapsync" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
           "user2" => "VARCHAR(255) NOT NULL",
-          "source_id" => "INT NOT NULL",
+          "host1" => "VARCHAR(255) NOT NULL",
+          "authmech1" => "ENUM('PLAIN','LOGIN','CRAM-MD5') DEFAULT 'PLAIN'",
           "regextrans2" => "VARCHAR(255) DEFAULT ''",
           "authmd51" => "TINYINT(1) NOT NULL DEFAULT 0",
           "domain2" => "VARCHAR(255) NOT NULL DEFAULT ''",
           "subfolder2" => "VARCHAR(255) NOT NULL DEFAULT ''",
           "user1" => "VARCHAR(255) NOT NULL",
-          "password1" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "password1" => "VARCHAR(255) NOT NULL",
           "exclude" => "VARCHAR(500) NOT NULL DEFAULT ''",
           "maxage" => "SMALLINT NOT NULL DEFAULT '0'",
           "mins_interval" => "SMALLINT UNSIGNED NOT NULL DEFAULT '0'",
           "maxbytespersecond" => "VARCHAR(50) NOT NULL DEFAULT '0'",
+          "port1" => "SMALLINT UNSIGNED NOT NULL",
+          "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'",
           "delete2duplicates" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "delete1" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "delete2" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "automap" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "skipcrossduplicates" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "custom_params" => "TEXT",
-          // original legacy custom_params string, only set when options were dropped during migration (never used by the runner)
-          "custom_params_legacy" => "TEXT DEFAULT NULL",
+          "custom_params" => "VARCHAR(512) NOT NULL DEFAULT ''",
           "timeout1" => "SMALLINT NOT NULL DEFAULT '600'",
           "timeout2" => "SMALLINT NOT NULL DEFAULT '600'",
           "subscribeall" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "dry" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "is_running" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "prio" => "INT NOT NULL DEFAULT 0",
           "returned_text" => "LONGTEXT",
           "last_run" => "TIMESTAMP NULL DEFAULT NULL",
           "success" => "TINYINT(1) UNSIGNED DEFAULT NULL",
@@ -1246,32 +795,6 @@ function init_db_schema()
         "keys" => array(
           "primary" => array(
             "" => array("id")
-          ),
-          "key" => array(
-            "idx_source_id" => array("source_id"),
-            "idx_prio" => array("prio")
-          ),
-          "fkey" => array(
-            "fk_imapsync_source" => array(
-              "col" => "source_id",
-              "ref" => "imapsync_source.id",
-              "delete" => "RESTRICT",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "imapsync_settings" => array(
-        "cols" => array(
-          "name" => "VARCHAR(64) NOT NULL",
-          "value" => "VARCHAR(255) NOT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("name")
           )
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
@@ -1661,128 +1184,11 @@ function init_db_schema()
         }
       }
 
-      // Migrate legacy imapsync jobs (host/port/enc per job) to domain-scoped legacy sources
-      // Re-entrant: only rows with source_id IS NULL are migrated, the legacy columns are kept while any are left
-      if ($table == 'imapsync') {
-        $stmt = $pdo->query("SHOW TABLES LIKE 'imapsync'");
-        if (count($stmt->fetchAll(PDO::FETCH_ASSOC)) != 0) {
-          $imapsync_cols = array_column($pdo->query("SHOW COLUMNS FROM `imapsync`")->fetchAll(PDO::FETCH_ASSOC), 'Field', 'Field');
-          if (isset($imapsync_cols['host1'])) {
-            if (!isset($imapsync_cols['source_id'])) {
-              $pdo->exec("ALTER TABLE `imapsync` ADD COLUMN `source_id` INT DEFAULT NULL");
-            }
-            if (!isset($imapsync_cols['custom_params_legacy'])) {
-              $pdo->exec("ALTER TABLE `imapsync` ADD COLUMN `custom_params_legacy` TEXT DEFAULT NULL");
-            }
-            // Converted JSON may not fit into the old VARCHAR(512)
-            $pdo->exec("ALTER TABLE `imapsync` MODIFY COLUMN `custom_params` TEXT");
-            $imapsync_allow = \Mailcow\Imapsync\Model\CustomParams::allowlist($GLOBALS['IMAPSYNC_OPTIONS'] ?? [], $GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? []);
-            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
-            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `auth_type`, `active`)
-              VALUES (:name, '', '', 'domain', :host, :port, :enc, :auth, 1)");
-            $source_domain_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)");
-            $job_upd = $pdo->prepare("UPDATE `imapsync` SET `source_id` = :source_id, `custom_params` = :cp, `custom_params_legacy` = :cp_legacy,
-              `dry` = GREATEST(`dry`, :dry) WHERE `id` = :id AND `source_id` IS NULL");
-            $stmt = $pdo->query("SELECT `id`, `user2`, `host1`, `port1`, `enc1`, `custom_params` FROM `imapsync` WHERE `source_id` IS NULL");
-            foreach ($stmt->fetchAll(PDO::FETCH_ASSOC) as $row) {
-              // Parse the legacy custom_params string like the old runner did (all tokens up to the next option are its value)
-              // into pairs valid under $IMAPSYNC_OPTIONS, a token after a flag is never its value.
-              // --authmech1 PLAIN|LOGIN|CRAM-MD5 becomes the source auth type, --dry the job's dry flag (never run a simulation for real).
-              // Anything else that does not fit is dropped, the original string is then kept in custom_params_legacy.
-              $pairs = array();
-              $auth = null;
-              $dry = false;
-              $dropped = false;
-              $tokens = preg_split('/\s+/', trim((string)$row['custom_params']), -1, PREG_SPLIT_NO_EMPTY);
-              for ($i = 0, $n = count($tokens); $i < $n; $i++) {
-                if ($tokens[$i][0] !== '-') {
-                  // stray value without option
-                  $dropped = true;
-                  continue;
-                }
-                @list($opt, $val) = explode('=', ltrim($tokens[$i], '-'), 2);
-                $opt = strtolower($opt);
-                $inline = ($val !== null);
-                $rest = array();
-                while ($i + 1 < $n && $tokens[$i + 1][0] !== '-') {
-                  $rest[] = $tokens[++$i];
-                }
-                $rule = ($opt === 'authmech1') ? true : ($imapsync_allow[$opt] ?? null);
-                if ($opt === 'dry' || $rule === false) {
-                  if ($inline || !empty($rest)) $dropped = true;
-                  if ($opt === 'dry') {
-                    $dry = true;
-                  } elseif (!$inline) {
-                    $pairs[] = array('o' => $opt, 'v' => '');
-                  }
-                  continue;
-                }
-                // unknown option, or "--opt=a b" (the old runner cut the value at the first space)
-                if ($rule === null || ($inline && !empty($rest))) {
-                  $dropped = true;
-                  continue;
-                }
-                $val = $inline ? $val : implode(' ', $rest);
-                if ($opt === 'authmech1' && $auth === null && in_array(strtoupper($val), array('PLAIN', 'LOGIN', 'CRAM-MD5'), true)) {
-                  $auth = strtoupper($val);
-                } elseif ($opt === 'authmech1' || trim($val) === ''
-                  || (is_string($rule) && preg_match('~^(?:' . str_replace('~', '\~', $rule) . ')$~D', $val) !== 1)) {
-                  $dropped = true;
-                } else {
-                  $pairs[] = array('o' => $opt, 'v' => $val);
-                }
-              }
-              // Old runner ignored the authmech1 column, only a custom --authmech1 was effective
-              $auth = $auth ?? 'PLAIN';
-              $host = strtolower(trim((string)$row['host1']));
-              $port = (intval($row['port1']) >= 1 && intval($row['port1']) <= 65535) ? intval($row['port1']) : 143;
-              $enc = in_array($row['enc1'], array('TLS', 'SSL', 'PLAIN'), true) ? $row['enc1'] : 'TLS';
-              $at = strrchr((string)$row['user2'], '@');
-              $domain = ($at === false) ? '' : strtolower(substr($at, 1));
-              // One source per (host, port, enc, auth, domain), name always unique and <= 100 chars
-              $name = 'legacy-' . mb_substr($host, 0, 60) . '-' . $port . '-' . substr(sha1($host . '|' . $port . '|' . $enc . '|' . $auth . '|' . $domain), 0, 8);
-              $source_sel->execute(array(':name' => $name));
-              if (($source_id = $source_sel->fetchColumn()) === false) {
-                $source_ins->execute(array(':name' => $name, ':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth));
-                $source_sel->execute(array(':name' => $name));
-                if (($source_id = $source_sel->fetchColumn()) === false) continue;
-              }
-              if ($domain !== '') {
-                $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
-              }
-              $job_upd->execute(array(
-                ':source_id' => $source_id,
-                ':cp' => json_encode($pairs, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE),
-                ':cp_legacy' => $dropped ? $row['custom_params'] : null,
-                ':dry' => $dry ? 1 : 0,
-                ':id' => $row['id']
-              ));
-            }
-
-            // Never throw on leftovers: keep source_id nullable and the legacy columns, retried with the next db_version
-            $pending = intval($pdo->query("SELECT COUNT(*) FROM `imapsync` WHERE `source_id` IS NULL")->fetchColumn());
-            if ($pending > 0) {
-              $properties['cols']['source_id'] = "INT DEFAULT NULL";
-              $legacy_col_types = array(
-                'host1' => "VARCHAR(255) DEFAULT NULL",
-                'port1' => "SMALLINT UNSIGNED DEFAULT NULL",
-                'enc1' => "ENUM('TLS','SSL','PLAIN') DEFAULT NULL",
-                'authmech1' => "ENUM('PLAIN','LOGIN','CRAM-MD5') DEFAULT NULL"
-              );
-              foreach ($legacy_col_types as $col => $type) {
-                if (isset($imapsync_cols[$col])) $properties['cols'][$col] = $type;
-              }
-              error_log('mailcow: imapsync migration: ' . $pending . ' sync job(s) could not be assigned to a source and will not run');
-            }
-          }
-        }
-      }
-
       $stmt = $pdo->query("SHOW TABLES LIKE '" . $table . "'");
       $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
       if ($num_results != 0) {
         $stmt = $pdo->prepare("SELECT CONCAT('ALTER TABLE `', `table_schema`, '`.', `table_name`, ' DROP FOREIGN KEY ', `constraint_name`, ';') AS `FKEY_DROP` FROM `information_schema`.`table_constraints`
-          WHERE `constraint_type` = 'FOREIGN KEY' AND `table_schema` = DATABASE() AND `table_name` = :table;");
+          WHERE `constraint_type` = 'FOREIGN KEY' AND `table_name` = :table;");
         $stmt->execute(array(':table' => $table));
         $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
         while ($row = array_shift($rows)) {
@@ -1943,29 +1349,20 @@ function init_db_schema()
         AND (JSON_CONTAINS_PATH(`call`, 'one', '$[2].password')
           OR JSON_CONTAINS_PATH(`call`, 'one', '$[2].password2'));");
 
-    // Mask syncjob secrets in old log entries (data and attr arguments of the logged call)
-    $mask_paths = array();
-    $unmasked = array();
-    foreach (array(3, 4) as $idx) {
-      foreach (array('password1', 'password2', 'oauth_client_secret', 'oauth_extra_params') as $key) {
-        $mask_paths[] = "'$[" . $idx . "]." . $key . "'";
-        // only rows still holding a value other than the string '*' (objects/arrays/numbers included), missing paths are NULL
-        $unmasked[] = "JSON_UNQUOTE(JSON_EXTRACT(`call`, '$[" . $idx . "]." . $key . "')) <> '*'";
-      }
-    }
-    $pdo->exec("UPDATE `logs` SET `call` = JSON_REPLACE(`call`, " . implode(", '*', ", $mask_paths) . ", '*')
-      WHERE JSON_VALID(`call`)
-        AND (JSON_VALUE(`call`, '$[0]') = 'syncjob'
-          OR JSON_VALUE(`call`, '$[0]') LIKE 'imapsync%'
-          OR (JSON_VALUE(`call`, '$[0]') = 'mailbox' AND JSON_VALUE(`call`, '$[2]') = 'syncjob'))
-        AND (" . implode(" OR ", $unmasked) . ");");
+    // Mitigate imapsync argument injection issue
+    $pdo->query("UPDATE `imapsync` SET `custom_params` = ''
+      WHERE `custom_params` LIKE '%pipemess%'
+        OR custom_params LIKE '%skipmess%'
+        OR custom_params LIKE '%delete2foldersonly%'
+        OR custom_params LIKE '%delete2foldersbutnot%'
+        OR custom_params LIKE '%regexflag%'
+        OR custom_params LIKE '%pipemess%'
+        OR custom_params LIKE '%regextrans2%'
+        OR custom_params LIKE '%maxlinelengthcmd%';");
 
     // Migrate webauthn tfa
     $stmt = $pdo->query("ALTER TABLE `tfa` MODIFY COLUMN `authmech` ENUM('yubi_otp', 'u2f', 'hotp', 'totp', 'webauthn')");
 
-    // Syncjobs: seed global settings
-    $pdo->query("INSERT IGNORE INTO `imapsync_settings` (`name`, `value`) VALUES ('max_parallel', '1')");
-
     // Inject admin if not exists
     $stmt = $pdo->query("SELECT NULL FROM `admin`");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -2133,18 +1530,6 @@ function init_db_schema()
     // fill quarantine.qhash
     $pdo->query("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE ISNULL(`qhash`)");
 
-    // Group rows that lost their group_id (init_db of an older version ran) belong to their group again, so the
-    // group can update its alias and revokes send-as rights of former members. An active group without rows of its
-    // own for a grant only exists after such a downgrade; manual rows next to group rows stay manual.
-    $pdo->query("UPDATE `alias` a JOIN `mailbox_group` g ON g.`address` = a.`address` SET a.`group_id` = g.`id` WHERE a.`group_id` IS NULL");
-    $pdo->query("UPDATE `sender_acl` s
-      JOIN `mailbox_group_member` m ON m.`username` = s.`logged_in_as`
-      JOIN `mailbox_group` g ON g.`id` = m.`group_id` AND g.`active` = 1
-      JOIN `mailbox_group_send_as` ga ON ga.`group_id` = m.`group_id` AND ga.`address` = s.`send_as`
-      SET s.`group_id` = m.`group_id`
-      WHERE s.`group_id` IS NULL AND s.`external` = 0
-        AND NOT EXISTS (SELECT 1 FROM (SELECT `group_id` FROM `sender_acl` WHERE `group_id` IS NOT NULL) own WHERE own.`group_id` = m.`group_id`)");
-
   } catch (PDOException $e) {
     if (php_sapi_name() == "cli") {
       echo "DB initialization failed: " . print_r($e, true) . PHP_EOL;
@@ -2159,10 +1544,6 @@ function init_db_schema()
 }
 if (php_sapi_name() == "cli") {
   include '/web/inc/vars.inc.php';
-  if (file_exists('/web/inc/vars.local.inc.php')) {
-    include '/web/inc/vars.local.inc.php';
-  }
-  require_once '/web/inc/lib/vendor/autoload.php';
   include '/web/inc/functions.inc.php';
   include '/web/inc/functions.docker.inc.php';
   // $now = new DateTime();
@@ -2180,7 +1561,7 @@ if (php_sapi_name() == "cli") {
     //PDO::MYSQL_ATTR_INIT_COMMAND => "SET time_zone = '" . $offset . "', group_concat_max_len = 3423543543;",
   ];
   $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-  $stmt = $pdo->query("SELECT COUNT('OK') AS OK_C FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND (TABLE_NAME = 'sogo_view' OR TABLE_NAME = '_sogo_static_view');");
+  $stmt = $pdo->query("SELECT COUNT('OK') AS OK_C FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view' OR TABLE_NAME = '_sogo_static_view';");
   $res = $stmt->fetch(PDO::FETCH_ASSOC);
   if (intval($res['OK_C']) === 2) {
     // Be more precise when replacing into _sogo_static_view, col orders may change
@@ -2200,13 +1581,4 @@ if (php_sapi_name() == "cli") {
     // Dunno
   }
   init_db_schema();
-  // Service API keys from mailcow.conf (API_KEY, API_KEY_READ_ONLY, API_ALLOW_FROM)
-  try {
-    $api_keys = \Mailcow\ApiKey\ApiKeyModule::create($pdo, \Mailcow\Core\Context::system())->keys;
-    foreach ($api_keys->syncServiceKeys(getenv()) as $warning) {
-      echo "API service keys: " . $warning . PHP_EOL;
-    }
-  } catch (PDOException $e) {
-    echo "API service keys: sync failed: " . $e->getMessage() . PHP_EOL;
-  }
 }

data/web/inc/lib/composer.json

@@ -13,10 +13,5 @@
         "stevenmaguire/oauth2-keycloak": "^4.0",
         "league/oauth2-client": "^2.7",
         "bacon/bacon-qr-code": "^2.0"
-    },
-    "autoload": {
-        "psr-4": {
-            "Mailcow\\": "mailcow/"
-        }
     }
 }

data/web/inc/lib/mailcow/ApiKey/ApiKeyModule.php

@@ -1,40 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\ApiKey;
-
-use Mailcow\ApiKey\Controller\ApiKeyApiController;
-use Mailcow\ApiKey\Repository\ApiKeyRepository;
-use Mailcow\ApiKey\Service\ApiKeyService;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use PDO;
-
-/**
- * Composition root of the API key module.
- *
- *   process_add_return(ApiKeyModule::fromGlobals()->api->addKey($attr));
- *   ApiKeyModule::create($pdo, Context::system())->keys->syncServiceKeys(getenv());  // php-fpm start
- */
-class ApiKeyModule
-{
-    /** Request keys whose values never reach a log */
-    private const SECRET_KEYS = ['api_key'];
-
-    public function __construct(public readonly ApiKeyService $keys, public readonly ApiKeyApiController $api)
-    {
-    }
-
-    public static function create(PDO $pdo, Context $context): self
-    {
-        $keys = new ApiKeyService(new ApiKeyRepository($pdo), $context);
-        return new self($keys, new ApiKeyApiController($keys, new ApiResponse(self::SECRET_KEYS)));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION; $context overrides the session caller */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
-    }
-}

data/web/inc/lib/mailcow/ApiKey/Controller/ApiKeyApiController.php

@@ -1,115 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\ApiKey\Controller;
-
-use Mailcow\ApiKey\Model\ApiKey;
-use Mailcow\ApiKey\Service\ApiKeyService;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php and edit.php (routes `api-key`).
- *
- * Write methods return what json_api passes to process_*_return() and write their messages via ApiResponse.
- * Messages are logged as ['api', action, 'key', data].
- */
-class ApiKeyApiController
-{
-    public function __construct(private ApiKeyService $keys, private ApiResponse $response)
-    {
-    }
-
-    /** POST add/api-key */
-    public function addKey(array $attr): bool
-    {
-        return $this->handle('add', $attr,
-            fn(array $log): bool => $this->response->success(['api_key_added', self::label($this->keys->create($attr))], $log));
-    }
-
-    /** POST edit/api-key: one message per id */
-    public function editKeys(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
-            $this->keys->requireAdmin();
-            return $this->batch((array)($data['id'] ?? []), $log,
-                fn(int $id): array => ['api_key_modified', self::label($this->keys->update($id, $attr))]);
-        });
-    }
-
-    /** POST delete/api-key: one message per id */
-    public function deleteKeys(array $items): bool
-    {
-        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
-            $this->keys->requireAdmin();
-            return $this->batch($items, $log, fn(int $id): array => ['api_key_deleted', self::label($this->keys->delete($id))]);
-        });
-    }
-
-    /** GET api-key/all */
-    public function getKeys(): array
-    {
-        return array_map(fn(ApiKey $key): array => $key->toArray(), $this->keys->all());
-    }
-
-    /** GET api-key/<id>, also edit.php */
-    public function getKey(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $key = ($id === null) ? null : $this->keys->get($id);
-        return ($key === null) ? [] : $key->toArray();
-    }
-
-    /** Name of a key in messages: its description, else its id */
-    private static function label(ApiKey $key): string
-    {
-        return htmlspecialchars($key->description !== '' ? $key->description : (string)$key->id);
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, mixed $data, callable $work): bool
-    {
-        $log = ['api', $action, 'key', $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** One message per id: the success message returned by $work(id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id)), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/ApiKey/Model/ApiKey.php

@@ -1,64 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\ApiKey\Model;
-
-/**
- * An API key (table `api`), sent by clients in the X-API-Key header (see sessions.inc.php).
- *
- * Unlike other secrets the key is part of toArray(): admins copy it from the UI. ApiKeyService limits
- * read access to admins (not with a read-only API key).
- */
-class ApiKey
-{
-    public const ACCESS_RO = 'ro';
-    public const ACCESS_RW = 'rw';
-
-    public ?int $id = null;
-    public string $key = '';
-    public string $description = '';
-    public string $access = self::ACCESS_RO;
-    /** @var string[] IPs and CIDR networks the key may be used from */
-    public array $allowFrom = [];
-    public int $skipIpCheck = 0;
-    /** 1 = service key from mailcow.conf: shown, but neither editable nor deletable in UI and API */
-    public int $service = 0;
-    public int $active = 1;
-    public ?string $created = null;
-    public ?string $modified = null;
-
-    /** Builds a key from an `api` row (column names as in the table) */
-    public static function fromRow(array $row): self
-    {
-        $key = new self();
-        $key->id = isset($row['id']) ? (int)$row['id'] : null;
-        $key->key = (string)($row['api_key'] ?? '');
-        $key->description = (string)($row['description'] ?? '');
-        $key->access = (string)($row['access'] ?? self::ACCESS_RO);
-        $key->allowFrom = array_values(array_filter(array_map('trim', explode(',', (string)($row['allow_from'] ?? ''))), 'strlen'));
-        $key->skipIpCheck = (int)($row['skip_ip_check'] ?? 0);
-        $key->service = (int)($row['service'] ?? 0);
-        $key->active = (int)($row['active'] ?? 1);
-        $key->created = isset($row['created']) ? (string)$row['created'] : null;
-        $key->modified = isset($row['modified']) ? (string)$row['modified'] : null;
-        return $key;
-    }
-
-    /** API representation (GET api-key) */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'description' => $this->description,
-            'access' => $this->access,
-            'api_key' => $this->key,
-            'allow_from' => $this->allowFrom,
-            'skip_ip_check' => $this->skipIpCheck,
-            'service' => $this->service,
-            'active' => $this->active,
-            'created' => $this->created,
-            'modified' => $this->modified,
-        ];
-    }
-}

data/web/inc/lib/mailcow/ApiKey/Repository/ApiKeyRepository.php

@@ -1,84 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\ApiKey\Repository;
-
-use Mailcow\ApiKey\Model\ApiKey;
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * SQL for the API keys (table `api`).
- */
-class ApiKeyRepository extends Repository
-{
-    /** @return ApiKey[] service keys first, then by id */
-    public function all(): array
-    {
-        return array_map([ApiKey::class, 'fromRow'],
-            $this->query("SELECT * FROM `api` ORDER BY `service` DESC, `id`")->fetchAll(PDO::FETCH_ASSOC));
-    }
-
-    public function find(int $id): ?ApiKey
-    {
-        $row = $this->query("SELECT * FROM `api` WHERE `id` = :id", ['id' => $id])->fetch(PDO::FETCH_ASSOC);
-        return $row ? ApiKey::fromRow($row) : null;
-    }
-
-    /** Inserts the key and returns the new id */
-    public function insert(ApiKey $key): int
-    {
-        $key->id = $this->insertRow('api', $this->values($key) + ['service' => $key->service]);
-        return $key->id;
-    }
-
-    public function update(ApiKey $key): void
-    {
-        $this->updateRow('api', (int)$key->id, $this->values($key));
-    }
-
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `api` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /**
-     * Makes the service keys match $keys in one transaction: other service keys are deleted,
-     * an existing row with the same key (also a UI key) becomes the service key.
-     *
-     * @param ApiKey[] $keys
-     */
-    public function syncService(array $keys): void
-    {
-        $this->transaction(function () use ($keys): void {
-            $params = [];
-            foreach (array_values($keys) as $i => $key) {
-                $params['k' . $i] = $key->key;
-            }
-            $keep = empty($params) ? '' : " AND `api_key` NOT IN (:" . implode(', :', array_keys($params)) . ")";
-            $this->query("DELETE FROM `api` WHERE `service` = '1'" . $keep, $params);
-            // No INSERT ... ON DUPLICATE KEY UPDATE: it uses up an id on every php-fpm start
-            foreach ($keys as $key) {
-                $id = $this->query("SELECT `id` FROM `api` WHERE `api_key` = :api_key", ['api_key' => $key->key])->fetchColumn();
-                if ($id === false) {
-                    $this->insertRow('api', $this->values($key) + ['service' => 1]);
-                } else {
-                    $this->updateRow('api', (int)$id, $this->values($key) + ['service' => 1]);
-                }
-            }
-        });
-    }
-
-    private function values(ApiKey $key): array
-    {
-        return [
-            'api_key' => $key->key,
-            'description' => $key->description,
-            'allow_from' => implode(',', $key->allowFrom),
-            'skip_ip_check' => $key->skipIpCheck,
-            'access' => $key->access,
-            'active' => $key->active,
-        ];
-    }
-}

data/web/inc/lib/mailcow/ApiKey/Service/ApiKeyService.php

@@ -1,247 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\ApiKey\Service;
-
-use Mailcow\ApiKey\Model\ApiKey;
-use Mailcow\ApiKey\Repository\ApiKeyRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Use cases and rules for API keys.
- *
- * Admins only. A read-only API key may not even read the keys, since they include read-write keys.
- * Service keys are defined in mailcow.conf (API_KEY, API_KEY_READ_ONLY, API_ALLOW_FROM) and mirrored
- * into the table on every php-fpm start (syncServiceKeys()). UI and API show them, but cannot change them.
- */
-class ApiKeyService
-{
-    /** mailcow.conf variable => access of the service key it defines */
-    public const SERVICE_KEYS = ['API_KEY' => ApiKey::ACCESS_RW, 'API_KEY_READ_ONLY' => ApiKey::ACCESS_RO];
-
-    public function __construct(private ApiKeyRepository $repo, private Context $context)
-    {
-    }
-
-    /** @throws AccessDenied unless an admin (login or read-write API key) */
-    public function requireAdmin(): void
-    {
-        $this->context->requireAdmin();
-    }
-
-    /** @return ApiKey[] all keys, service keys first; [] for other callers */
-    public function all(): array
-    {
-        return $this->canRead() ? $this->repo->all() : [];
-    }
-
-    public function get(int $id): ?ApiKey
-    {
-        return $this->canRead() ? $this->repo->find($id) : null;
-    }
-
-    /**
-     * Creates a key with a random value; `access` is required.
-     *
-     * @throws UserError
-     */
-    public function create(array $input): ApiKey
-    {
-        $this->requireAdmin();
-        $key = new ApiKey();
-        $key->key = self::generate();
-        $this->apply($key, $input);
-        $this->repo->insert($key);
-        return $key;
-    }
-
-    /**
-     * Updates a key; attributes missing in $input keep their value, `regen_key` issues a new key value.
-     *
-     * @throws UserError access_denied, api_key_service_locked, validation errors
-     */
-    public function update(int $id, array $input): ApiKey
-    {
-        $this->requireAdmin();
-        $key = $this->editable($id);
-        $this->apply($key, $input);
-        if (self::flag($input['regen_key'] ?? 0) === 1) {
-            $key->key = self::generate();
-        }
-        $this->repo->update($key);
-        return $key;
-    }
-
-    /**
-     * Deletes a key and returns it.
-     *
-     * @throws UserError access_denied, api_key_service_locked
-     */
-    public function delete(int $id): ApiKey
-    {
-        $this->requireAdmin();
-        $key = $this->editable($id);
-        $this->repo->delete($id);
-        return $key;
-    }
-
-    /**
-     * Mirrors the service keys of mailcow.conf into the table. Runs on php-fpm start (init_db CLI, master only),
-     * so there is no permission check. $env holds the container environment (getenv()).
-     * Without a valid API_ALLOW_FROM no service key is active, like before. Service keys that were removed from
-     * mailcow.conf are deleted; a UI key with the same value becomes the service key.
-     *
-     * @return string[] warnings for the container log
-     */
-    public function syncServiceKeys(array $env): array
-    {
-        $warnings = [];
-        $allowFrom = [];
-        foreach (self::split($env['API_ALLOW_FROM'] ?? '') as $network) {
-            // 'invalid' is the docker-compose default for an unset variable
-            if ($network === 'invalid') {
-                continue;
-            }
-            if (self::isNetwork($network)) {
-                $allowFrom[] = $network;
-            } else {
-                $warnings[] = 'API_ALLOW_FROM: skipped invalid IP/network ' . $network;
-            }
-        }
-        $keys = [];
-        foreach (self::SERVICE_KEYS as $var => $access) {
-            $value = trim((string)($env[$var] ?? ''));
-            if ($value === '' || $value === 'invalid') {
-                continue;
-            }
-            if (!preg_match('/^[a-zA-Z0-9-]+$/', $value)) {
-                $warnings[] = $var . ': invalid characters (allowed: a-z, A-Z, 0-9, -), key ignored';
-                continue;
-            }
-            if (empty($allowFrom)) {
-                $warnings[] = $var . ': API_ALLOW_FROM contains no valid IP/network, key ignored';
-                continue;
-            }
-            if (isset($keys[$value])) {
-                $warnings[] = $var . ': same value as ' . $keys[$value]->description . ', key ignored';
-                continue;
-            }
-            $key = new ApiKey();
-            $key->key = $value;
-            $key->description = $var;
-            $key->access = $access;
-            $key->allowFrom = $allowFrom;
-            $key->service = 1;
-            $keys[$value] = $key;
-        }
-        $this->repo->syncService(array_values($keys));
-        return $warnings;
-    }
-
-    private function canRead(): bool
-    {
-        return $this->context->hasAdminRights() && !$this->context->isReadOnlyApi();
-    }
-
-    /** @throws AccessDenied when missing, UserError api_key_service_locked for service keys */
-    private function editable(int $id): ApiKey
-    {
-        $key = $this->repo->find($id) ?? throw new AccessDenied();
-        if ($key->service === 1) {
-            throw new UserError('api_key_service_locked', [$key->description]);
-        }
-        return $key;
-    }
-
-    /**
-     * Add/edit rules: attributes missing in $input keep the current value, a new key starts with the
-     * model defaults. Without skip_ip_check the key needs at least one IP/network.
-     *
-     * @throws UserError
-     */
-    private function apply(ApiKey $key, array $input): void
-    {
-        if (isset($input['description'])) {
-            $description = Input::text($input['description']);
-            if ($description === null || mb_strlen($description) > 255) {
-                throw new UserError('api_key_description_invalid');
-            }
-            $key->description = $description;
-        }
-        if (isset($input['access']) || $key->id === null) {
-            $access = $input['access'] ?? null;
-            if (!in_array($access, [ApiKey::ACCESS_RO, ApiKey::ACCESS_RW], true)) {
-                throw new UserError('api_key_access_invalid');
-            }
-            $key->access = $access;
-        }
-        if (isset($input['allow_from'])) {
-            $key->allowFrom = self::networks($input['allow_from']);
-        }
-        if (isset($input['skip_ip_check'])) {
-            $key->skipIpCheck = self::flag($input['skip_ip_check']);
-        }
-        if (isset($input['active'])) {
-            $key->active = self::flag($input['active']);
-        }
-        if ($key->skipIpCheck === 0 && empty($key->allowFrom)) {
-            throw new UserError('ip_list_empty');
-        }
-    }
-
-    /** New random key in the format of the legacy UI: five groups of six hex digits */
-    private static function generate(): string
-    {
-        return implode('-', array_map(fn(): string => strtoupper(bin2hex(random_bytes(3))), range(1, 5)));
-    }
-
-    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
-    private static function flag(mixed $value): int
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN) ? 1 : 0;
-    }
-
-    /**
-     * @return string[] validated IPs/networks
-     * @throws UserError api_key_network_invalid for the first invalid entry
-     */
-    private static function networks(mixed $value): array
-    {
-        $networks = self::split($value);
-        foreach ($networks as $network) {
-            if (!self::isNetwork($network)) {
-                throw new UserError('api_key_network_invalid', [htmlspecialchars($network)]);
-            }
-        }
-        return $networks;
-    }
-
-    /** @return string[] unique entries of a list: array or string separated by spaces, commas, semicolons or newlines */
-    private static function split(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', is_scalar($value) ? (string)$value : '');
-        $values = array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values);
-        return array_values(array_unique(array_filter($values, 'strlen')));
-    }
-
-    /** IPv4/IPv6 address or CIDR network */
-    private static function isNetwork(string $value): bool
-    {
-        [$ip, $prefix] = array_pad(explode('/', $value, 2), 2, null);
-        if (filter_var($ip, FILTER_VALIDATE_IP) === false) {
-            return false;
-        }
-        if ($prefix === null) {
-            return true;
-        }
-        $max = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4) ? 32 : 128;
-        return ctype_digit($prefix) && (int)$prefix <= $max;
-    }
-}

data/web/inc/lib/mailcow/Auth/AuthModule.php

@@ -1,36 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Auth;
-
-use Mailcow\Auth\Service\IdpLoginService;
-use Mailcow\Core\Context;
-use Mailcow\Identity\IdentityModule;
-use Mailcow\MailboxGroup\MailboxGroupModule;
-use PDO;
-
-/**
- * Logins. For now the IdP login steps of the legacy flows (keycloak_mbox_login_rest(), ldap_mbox_login(),
- * identity_provider('verify-sso')); the auth logic of the function files moves here step by step.
- */
-class AuthModule
-{
-    public function __construct(public readonly IdpLoginService $idp)
-    {
-    }
-
-    /** @param array $iamSettings identity provider settings as identity_provider('get') returns them */
-    public static function create(PDO $pdo, Context $context, array $iamSettings): self
-    {
-        $identity = IdentityModule::create($pdo, $context, $iamSettings);
-        return new self(new IdpLoginService($identity->settings, $identity->links, MailboxGroupModule::create($pdo, $context)->templates));
-    }
-
-    /** Logins run before a session exists, so the default context is the system */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        $settings = $GLOBALS['iam_settings'] ?? null;
-        return self::create($GLOBALS['pdo'], $context ?? Context::system(), is_array($settings) ? $settings : identity_provider('get'));
-    }
-}

data/web/inc/lib/mailcow/Auth/IdpUnavailable.php

@@ -1,15 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Auth;
-
-use RuntimeException;
-
-/**
- * The identity provider could not answer a login (unreachable, timeout, server error). No failed login:
- * callers report a temporary failure and must not count it as an attempt for netfilter.
- */
-class IdpUnavailable extends RuntimeException
-{
-}

data/web/inc/lib/mailcow/Auth/Service/IdpLoginService.php

@@ -1,105 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Auth\Service;
-
-use LdapRecord\Auth\BindException;
-use LdapRecord\Connection;
-use LdapRecord\LdapRecordException;
-use Mailcow\Auth\IdpUnavailable;
-use Mailcow\Identity\Model\Settings;
-use Mailcow\Identity\Service\LinkService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use Throwable;
-
-/**
- * Steps of an IdP login (Keycloak REST, LDAP bind, OIDC) the legacy login flows call. The link checks fail
- * closed: an error denies the login.
- */
-class IdpLoginService
-{
-    public function __construct(private Settings $settings, private LinkService $links, private TemplateService $templates)
-    {
-    }
-
-    /**
-     * Binds as the user to check the password, then again as the configured user. False for wrong credentials,
-     * IdpUnavailable when the server cannot answer (client side codes are negative: server down, timeout, ...).
-     */
-    public function verifyLdapPassword(Connection $ldap, string $dn, string $password): bool
-    {
-        // an empty password would be an unauthenticated bind, which succeeds
-        if ($dn === '' || $password === '') {
-            return false;
-        }
-        $guard = $ldap->auth();
-        try {
-            $guard->bind($dn, $password);
-            $bound = true;
-        } catch (BindException $e) {
-            $code = $e->getDetailedError()?->getErrorCode();
-            if ($code !== null && ($code < 0 || in_array($code, [51, 52], true))) {
-                throw new IdpUnavailable('LDAP bind failed: ' . $e->getMessage(), 0, $e);
-            }
-            $bound = false;
-        } catch (LdapRecordException $e) {
-            throw new IdpUnavailable('LDAP bind failed: ' . $e->getMessage(), 0, $e);
-        }
-        try {
-            $guard->bindAsConfiguredUser();
-        } catch (LdapRecordException $e) {
-            // the login needs no further LDAP access
-        }
-        return $bound;
-    }
-
-    /** Template of the first attribute mapping matching one of the IdP attribute values, null if none */
-    public function mappedTemplate(mixed $values): ?string
-    {
-        return $this->settings->mappedTemplate((array)$values);
-    }
-
-    /** Template for the login of an existing mailbox: its effective group template, else the mapped one */
-    public function templateFor(string $username, ?string $mapped): ?string
-    {
-        try {
-            return $this->templates->templateForLogin($username, $mapped);
-        } catch (Throwable $e) {
-            error_log('mailcow: group template lookup failed for ' . $username . ': ' . $e->getMessage());
-            return $mapped;
-        }
-    }
-
-    /** The IdP user may open the mailbox, an unlinked mailbox gets linked */
-    public function verifyLink(string $provider, string $username, string $externalId): bool
-    {
-        try {
-            return $this->links->verify($provider, $username, $externalId);
-        } catch (Throwable $e) {
-            error_log('mailcow: IdP link check failed for ' . $username . ': ' . $e->getMessage());
-            return false;
-        }
-    }
-
-    /** Before a mailbox is created on login: the IdP user must not own another mailbox */
-    public function mayCreate(string $provider, string $username, string $externalId): bool
-    {
-        try {
-            return $this->links->mayCreate($provider, $username, $externalId);
-        } catch (Throwable $e) {
-            error_log('mailcow: IdP link check failed for ' . $username . ': ' . $e->getMessage());
-            return false;
-        }
-    }
-
-    /** Links a mailbox created on login */
-    public function storeLink(string $provider, string $username, string $externalId): void
-    {
-        try {
-            $this->links->link($provider, $username, $externalId);
-        } catch (Throwable $e) {
-            error_log('mailcow: IdP link could not be stored for ' . $username . ': ' . $e->getMessage());
-        }
-    }
-}

data/web/inc/lib/mailcow/Core/AccessDenied.php

@@ -1,16 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-/**
- * The caller may not do this (or may not see the object). Always shown as `access_denied`.
- */
-class AccessDenied extends UserError
-{
-    public function __construct()
-    {
-        parent::__construct('access_denied');
-    }
-}

data/web/inc/lib/mailcow/Core/ApiResponse.php

@@ -1,76 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-/**
- * Collects the messages of one API call and writes them to $_SESSION['return'] in the mailcow format
- * (`type`, `log`, `msg`), which json_api.php and the UI read. This is the only class that touches the session.
- *
- * Values of the configured secret keys are replaced by '*' inside every array of the `log` part,
- * so passwords or client secrets never reach the mailcow log or an API response.
- */
-class ApiResponse
-{
-    private array $messages = [];
-
-    public function __construct(private array $secretKeys = [])
-    {
-    }
-
-    /** Returns true, so callers can `return $response->success(...)` */
-    public function success(string|array $msg, array $log): bool
-    {
-        $this->add('success', $msg, $log);
-        return true;
-    }
-
-    /** Returns false, so callers can `return $response->danger(...)` */
-    public function danger(string|array $msg, array $log): bool
-    {
-        $this->add('danger', $msg, $log);
-        return false;
-    }
-
-    public function error(UserError $error, array $log): bool
-    {
-        return $this->danger($error->msg(), $log);
-    }
-
-    /** Something went partly wrong, but the call itself succeeded */
-    public function warning(string|array $msg, array $log): void
-    {
-        $this->add('warning', $msg, $log);
-    }
-
-    /** Messages collected since the last flush() */
-    public function messages(): array
-    {
-        return $this->messages;
-    }
-
-    /** Appends the collected messages to $_SESSION['return'] and starts over */
-    public function flush(): void
-    {
-        foreach ($this->messages as $message) {
-            $_SESSION['return'][] = $message;
-        }
-        $this->messages = [];
-    }
-
-    private function add(string $type, string|array $msg, array $log): void
-    {
-        foreach ($log as $i => $part) {
-            if (!is_array($part)) {
-                continue;
-            }
-            foreach ($this->secretKeys as $key) {
-                if (isset($part[$key]) && $part[$key] !== '') {
-                    $log[$i][$key] = '*';
-                }
-            }
-        }
-        $this->messages[] = ['type' => $type, 'log' => $log, 'msg' => $msg];
-    }
-}

data/web/inc/lib/mailcow/Core/Context.php

@@ -1,107 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-/**
- * Who is calling: role, username, ACL flags and the access-all exception of the session.
- *
- * Services get a Context instead of reading $_SESSION. Build it once per request in the entry point:
- * `Context::fromSession($_SESSION)` for web requests, `Context::system()` for cron/CLI code paths
- * that do no permission checks.
- */
-class Context
-{
-    /**
-     * @param array $acl          ACL flags of the session (user_acl / da_acl, always set for admins)
-     * @param bool $accessAll     the session may ignore object ownership (`access_all_exception`)
-     * @param ?string $apiAccess  'ro' / 'rw' when the request authenticated with an X-API-Key, else null
-     */
-    public function __construct(
-        public readonly string $role,
-        public readonly ?string $username,
-        private readonly array $acl = [],
-        public readonly bool $accessAll = false,
-        public readonly ?string $apiAccess = null
-    ) {
-    }
-
-    public static function fromSession(array $session): self
-    {
-        $username = $session['mailcow_cc_username'] ?? null;
-        $apiAccess = null;
-        if (($session['mailcow_cc_api'] ?? false) === true) {
-            $apiAccess = (($session['mailcow_cc_api_access'] ?? null) === 'rw') ? 'rw' : 'ro';
-        }
-        return new self(
-            (string)($session['mailcow_cc_role'] ?? ''),
-            is_scalar($username) ? (string)$username : null,
-            is_array($session['acl'] ?? null) ? $session['acl'] : [],
-            ($session['access_all_exception'] ?? null) == '1',
-            $apiAccess
-        );
-    }
-
-    /** Caller for cron and CLI code: not a logged-in account, so every permission check fails */
-    public static function system(): self
-    {
-        return new self('system', null);
-    }
-
-    public function isAdmin(): bool
-    {
-        return $this->role === 'admin';
-    }
-
-    public function isDomainAdmin(): bool
-    {
-        return $this->role === 'domainadmin';
-    }
-
-    public function isUser(): bool
-    {
-        return $this->role === 'user';
-    }
-
-    /** Request with a read-only API key (role admin, but no write access) */
-    public function isReadOnlyApi(): bool
-    {
-        return $this->apiAccess === 'ro';
-    }
-
-    /** An admin, or a legacy call that may ignore ownership (access_all_exception) */
-    public function hasAdminRights(): bool
-    {
-        return $this->isAdmin() || $this->accessAll;
-    }
-
-    /**
-     * Admin rights; a write also needs more than a read-only API key.
-     *
-     * @throws AccessDenied
-     */
-    public function requireAdmin(bool $write = true): void
-    {
-        if (!$this->hasAdminRights() || ($write && $this->isReadOnlyApi())) {
-            throw new AccessDenied();
-        }
-    }
-
-    public function hasAcl(string $name): bool
-    {
-        return isset($this->acl[$name]) && $this->acl[$name] == '1';
-    }
-
-    /** Bridge to the legacy hasMailboxObjectAccess() until functions.inc.php is converted */
-    public function canAccessMailbox(string $mailbox): bool
-    {
-        return (bool)hasMailboxObjectAccess($this->username, $this->role, $mailbox);
-    }
-
-    /** Bridge to the legacy hasDomainAccess() until functions.inc.php is converted */
-    public function canAccessDomain(string $domain): bool
-    {
-        return (bool)hasDomainAccess($this->username, $this->role, $domain);
-    }
-}

data/web/inc/lib/mailcow/Core/Input.php

@@ -1,35 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-/**
- * Helpers for loosely typed request values. Form values arrive as strings, JSON bodies as any type,
- * and 011-api.js turns JSON-looking strings into arrays, so services must not trust the PHP type.
- */
-class Input
-{
-    /**
-     * Scalar value as trimmed string; '' for null; null for arrays/objects (= invalid).
-     */
-    public static function text(mixed $value): ?string
-    {
-        if ($value === null) {
-            return '';
-        }
-        if (!is_scalar($value)) {
-            return null;
-        }
-        return trim((string)$value);
-    }
-
-    /**
-     * Integer within [$min, $max] (filter_var rules: "12" and 12 pass, "12abc" and "1.5" do not), else null.
-     */
-    public static function intInRange(mixed $value, int $min, int $max): ?int
-    {
-        $int = filter_var($value, FILTER_VALIDATE_INT, ['options' => ['min_range' => $min, 'max_range' => $max]]);
-        return ($int === false) ? null : $int;
-    }
-}

data/web/inc/lib/mailcow/Core/Repository.php

@@ -1,65 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-use PDO;
-use PDOStatement;
-use Throwable;
-
-/**
- * Base class of the repositories: the PDO plus the helpers every repository needs.
- * Named parameters are passed without the colon (['id' => 5]).
- */
-class Repository
-{
-    public function __construct(protected PDO $pdo)
-    {
-    }
-
-    /** Prepares and executes one statement */
-    protected function query(string $sql, array $params = []): PDOStatement
-    {
-        $stmt = $this->pdo->prepare($sql);
-        $stmt->execute($params);
-        return $stmt;
-    }
-
-    /** Inserts one row (column => value) and returns the new id */
-    protected function insertRow(string $table, array $values): int
-    {
-        $columns = array_keys($values);
-        $this->query("INSERT INTO `$table` (`" . implode('`, `', $columns) . "`) VALUES (:" . implode(', :', $columns) . ")", $values);
-        return (int)$this->pdo->lastInsertId();
-    }
-
-    /** Updates the columns (column => value) of the row with the given `id` */
-    protected function updateRow(string $table, int $id, array $values): void
-    {
-        $set = implode(', ', array_map(fn(string $column): string => "`$column` = :$column", array_keys($values)));
-        $this->query("UPDATE `$table` SET $set WHERE `id` = :id", $values + ['id' => $id]);
-    }
-
-    /**
-     * Runs $work in a transaction and returns its result. Inside a transaction of the caller it just runs,
-     * so the caller keeps control over commit and rollback.
-     */
-    protected function transaction(callable $work): mixed
-    {
-        if ($this->pdo->inTransaction()) {
-            return $work();
-        }
-        $this->pdo->beginTransaction();
-        try {
-            $result = $work();
-            $this->pdo->commit();
-            return $result;
-        } catch (Throwable $e) {
-            if ($this->pdo->inTransaction()) {
-                $this->pdo->rollBack();
-            }
-            throw $e;
-        }
-    }
-}

data/web/inc/lib/mailcow/Core/UserError.php

@@ -1,28 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Core;
-
-use RuntimeException;
-
-/**
- * An expected error that is shown to the user (validation failed, not found, not allowed).
- *
- * Carries a lang key of the `danger` section plus optional message arguments, e.g.
- * `throw new UserError('object_exists', [$name]);`
- * Controllers turn it into an ApiResponse message; anything else that is thrown is a real failure.
- */
-class UserError extends RuntimeException
-{
-    public function __construct(public readonly string $key, public readonly array $args = [])
-    {
-        parent::__construct($key);
-    }
-
-    /** The `msg` value of a mailcow message: the lang key alone, or [key, arg1, ...] when there are arguments */
-    public function msg(): string|array
-    {
-        return empty($this->args) ? $this->key : array_merge([$this->key], array_values($this->args));
-    }
-}

data/web/inc/lib/mailcow/Dovecot/DovecotModule.php

@@ -1,22 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Dovecot;
-
-use Mailcow\Dovecot\Service\DovecotService;
-
-/**
- * Dovecot through dockerapi (ACLs, subscriptions, auth cache). Modules use it instead of calling docker().
- */
-class DovecotModule
-{
-    public function __construct(public readonly DovecotService $dovecot)
-    {
-    }
-
-    public static function create(): self
-    {
-        return new self(new DovecotService());
-    }
-}

data/web/inc/lib/mailcow/Dovecot/Service/DovecotService.php

@@ -1,54 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Dovecot\Service;
-
-/**
- * doveadm in the dovecot container through dockerapi. The docker() call sits in a protected method so tests can
- * replace it.
- */
-class DovecotService
-{
-    /**
-     * Sets the rights of an ACL identifier (user=<name>, group=<name>) on every folder of the owner; no rights
-     * remove it.
-     *
-     * @return ?string null on success, else the error of dockerapi
-     */
-    public function setAclAll(string $owner, string $identifier, array $rights): ?string
-    {
-        return self::error($this->exec(['cmd' => 'doveadm', 'task' => 'set_acl_all', 'user' => $owner, 'id' => $identifier,
-            'rights' => array_values($rights)]));
-    }
-
-    /**
-     * Subscribes the users to every folder of the owner they see under Shared/<owner>/, or unsubscribes them.
-     *
-     * @return ?string null on success, else the error of dockerapi
-     */
-    public function subscribeShared(string $owner, array $usernames, bool $unsubscribe = false): ?string
-    {
-        return self::error($this->exec(['cmd' => 'doveadm', 'task' => 'subscribe_shared', 'owner' => $owner,
-            'users' => array_values($usernames), 'unsubscribe' => $unsubscribe]));
-    }
-
-    /** Drops the cached userdb results, so changed userdb fields apply at the next login instead of after auth_cache_ttl */
-    public function flushAuthCache(array $usernames): void
-    {
-        if (!empty($usernames)) {
-            $this->exec(['cmd' => 'doveadm', 'task' => 'auth_cache_flush', 'users' => array_values($usernames)]);
-        }
-    }
-
-    protected function exec(array $fields): mixed
-    {
-        return docker('post', 'dovecot-mailcow', 'exec', $fields);
-    }
-
-    private static function error(mixed $response): ?string
-    {
-        $result = json_decode((string)$response, true);
-        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
-    }
-}

data/web/inc/lib/mailcow/Identity/Controller/IdentityApiController.php

@@ -1,72 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Controller;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\Identity\Service\StatusService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php (routes `identity-provider/sync-status`, `identity-provider-sync`,
- * `identity-link`). Messages are logged as ['identity_provider', action, object, data].
- */
-class IdentityApiController
-{
-    public function __construct(private StatusService $status, private ApiResponse $response)
-    {
-    }
-
-    /** GET identity-provider/sync-status */
-    public function getStatus(): array
-    {
-        try {
-            return $this->status->status();
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /** POST edit/identity-provider-sync: attr mode = preview|run|force */
-    public function requestRun(array $attr): bool
-    {
-        return $this->handle('edit', 'sync', $attr, function (array $log) use ($attr): bool {
-            $this->status->requestRun($attr['mode'] ?? 'run');
-            return $this->response->success('iam_sync_requested', $log);
-        });
-    }
-
-    /** GET identity-link/<username> */
-    public function getLink(string $username): array
-    {
-        return $this->status->link($username) ?? [];
-    }
-
-    /** POST delete/identity-link: usernames */
-    public function unlink(array $items): bool
-    {
-        return $this->handle('delete', 'link', $items, function (array $log) use ($items): bool {
-            foreach ($items as $username) {
-                $this->status->unlink((string)$username);
-                $this->response->success(['iam_link_removed', htmlspecialchars((string)$username)], $log);
-            }
-            return true;
-        });
-    }
-
-    private function handle(string $action, string $object, mixed $data, callable $work): bool
-    {
-        $log = ['identity_provider', $action, $object, $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-}

data/web/inc/lib/mailcow/Identity/Directory/Directory.php

@@ -1,21 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Directory;
-
-use Mailcow\Identity\Model\Listing;
-
-/**
- * Lists users (and groups) of an IdP for the sync. Implementations: KeycloakDirectory, LdapDirectory.
- */
-interface Directory
-{
-    /**
-     * The complete listing. Anything that could make it incomplete (HTTP or LDAP errors, size limits, ranged
-     * attributes) throws, because the sync treats users that are not listed as removed.
-     *
-     * @throws DirectoryException
-     */
-    public function listAll(bool $withGroups): Listing;
-}

data/web/inc/lib/mailcow/Identity/Directory/DirectoryException.php

@@ -1,14 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Directory;
-
-use RuntimeException;
-
-/**
- * The IdP could not be listed completely; the sync run stops without writing anything.
- */
-class DirectoryException extends RuntimeException
-{
-}

data/web/inc/lib/mailcow/Identity/Directory/KeycloakDirectory.php

@@ -1,105 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Directory;
-
-use Mailcow\Identity\Http\KeycloakClient;
-use Mailcow\Identity\Model\DirectoryGroup;
-use Mailcow\Identity\Model\DirectoryUser;
-use Mailcow\Identity\Model\Listing;
-use Mailcow\Identity\Model\Settings;
-
-/**
- * Keycloak realm users and the groups selected by their path. Only direct group members count.
- */
-class KeycloakDirectory implements Directory
-{
-    private const PAGE = 100;
-
-    public function __construct(private KeycloakClient $client, private Settings $settings)
-    {
-    }
-
-    public function listAll(bool $withGroups): Listing
-    {
-        $users = [];
-        foreach ($this->paged('/users', ['briefRepresentation' => 'false']) as $row) {
-            if (isset($row['id'])) {
-                $user = self::user($row);
-                $users[$user->id] = $user;
-            }
-        }
-        return new Listing($users, $withGroups ? $this->groups($users) : null);
-    }
-
-    /** A user of the admin REST API */
-    public static function user(array $row): DirectoryUser
-    {
-        $email = (isset($row['email']) && $row['email'] !== '') ? strtolower((string)$row['email']) : null;
-        return new DirectoryUser((string)$row['id'], $email, trim(($row['firstName'] ?? '') . ' ' . ($row['lastName'] ?? '')),
-            !empty($row['enabled']), array_map('strval', (array)($row['attributes']['mailcow_template'] ?? [])));
-    }
-
-    /**
-     * @param array<string, DirectoryUser> $users
-     * @return array<string, DirectoryGroup> selected groups, members limited to listed users
-     */
-    private function groups(array $users): array
-    {
-        $groups = [];
-        foreach ($this->paged('/groups', ['briefRepresentation' => 'false']) as $row) {
-            $this->collect($row, $groups);
-        }
-        foreach ($groups as $id => $group) {
-            foreach ($this->paged('/groups/' . rawurlencode((string)$id) . '/members', ['briefRepresentation' => 'true']) as $member) {
-                if (isset($member['id'], $users[$member['id']])) {
-                    $group->memberIds[] = (string)$member['id'];
-                }
-            }
-        }
-        return $groups;
-    }
-
-    /** Adds the group and its subgroups that match the configured paths (Keycloak >= 23 lists children separately) */
-    private function collect(array $row, array &$groups): void
-    {
-        $path = (string)($row['path'] ?? '/' . ($row['name'] ?? ''));
-        if ($this->selected($path)) {
-            $mail = $row['attributes']['mail'][0] ?? null;
-            $groups[(string)$row['id']] = new DirectoryGroup((string)$row['id'], (string)($row['name'] ?? $path),
-                is_string($mail) && $mail !== '' ? strtolower($mail) : null);
-        }
-        $children = $row['subGroups'] ?? [];
-        if (empty($children) && (int)($row['subGroupCount'] ?? 0) > 0) {
-            $children = $this->paged('/groups/' . rawurlencode((string)$row['id']) . '/children', ['briefRepresentation' => 'false']);
-        }
-        foreach ($children as $child) {
-            $this->collect($child, $groups);
-        }
-    }
-
-    /** '/a' exactly, '/a/*' every group below /a, '/*' every group */
-    private function selected(string $path): bool
-    {
-        foreach ($this->settings->groupPaths as $pattern) {
-            if ($pattern === $path || (str_ends_with($pattern, '/*') && str_starts_with($path, substr($pattern, 0, -1)))) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /** @return array[] all rows of a paged list endpoint */
-    private function paged(string $path, array $query): array
-    {
-        $rows = [];
-        for ($first = 0; ; $first += self::PAGE) {
-            $page = array_values($this->client->get($path, $query + ['first' => $first, 'max' => self::PAGE]));
-            $rows = array_merge($rows, $page);
-            if (count($page) < self::PAGE) {
-                return $rows;
-            }
-        }
-    }
-}

data/web/inc/lib/mailcow/Identity/Directory/LdapDirectory.php

@@ -1,240 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Directory;
-
-use LdapRecord\Connection;
-use Mailcow\Identity\Model\DirectoryGroup;
-use Mailcow\Identity\Model\DirectoryUser;
-use Mailcow\Identity\Model\Listing;
-use Mailcow\Identity\Model\Settings;
-
-/**
- * LDAP / Active Directory users and the groups matching the group filter.
- *
- * Searches run on the raw connection with paged results and check the result code of every page, because
- * LdapRecord's paginator stops silently on size limits. Group members come from memberOf (default), from the
- * member attribute of the groups, or from the AD in-chain rule (nested groups).
- */
-class LdapDirectory implements Directory
-{
-    private const PAGE = 500;
-    private const ID_ATTRIBUTES = ['objectguid', 'entryuuid', 'nsuniqueid', 'ipauniqueid'];
-    private const AD_IN_CHAIN = '1.2.840.113556.1.4.1941';
-    /** userAccountControl: ACCOUNTDISABLE */
-    private const UAC_DISABLED = 0x2;
-
-    public function __construct(private ?Connection $connection, private Settings $settings)
-    {
-    }
-
-    public function listAll(bool $withGroups): Listing
-    {
-        $s = $this->settings;
-        $userFilter = $this->userFilter();
-        $disabled = [];
-        if ($s->disabledFilter !== '') {
-            foreach ($this->search($s->basedn, '(&' . $userFilter . self::wrap($s->disabledFilter) . ')', ['dn']) as $entry) {
-                $disabled[strtolower((string)$entry['dn'])] = true;
-            }
-        }
-        $attributes = array_merge([$s->usernameField, 'displayname', 'cn', 'useraccountcontrol'],
-            ($s->attributeField !== '') ? [$s->attributeField] : [],
-            self::idAttributes($s->idAttribute),
-            ($withGroups && $s->groupMemberMode === 'memberof') ? ['memberof'] : []);
-        $users = [];
-        $memberOf = [];
-        foreach ($this->search($s->basedn, $userFilter, array_values(array_unique($attributes))) as $entry) {
-            $dn = strtolower((string)$entry['dn']);
-            $email = self::first($entry, $s->usernameField);
-            $uac = self::first($entry, 'useraccountcontrol');
-            $user = new DirectoryUser(self::entryId($entry, $s->idAttribute),
-                ($email === null) ? null : strtolower($email),
-                self::first($entry, 'displayname') ?? self::first($entry, 'cn') ?? '',
-                !isset($disabled[$dn]) && !($uac !== null && ((int)$uac & self::UAC_DISABLED)),
-                ($s->attributeField !== '') ? self::values($entry, $s->attributeField) : [],
-                $dn);
-            $users[$user->id] = $user;
-            $memberOf[$user->id] = array_map('strtolower', self::values($entry, 'memberof'));
-        }
-        return new Listing($users, $withGroups ? $this->groups($users, $memberOf, $userFilter) : null);
-    }
-
-    /** Attributes entryId() reads */
-    public static function idAttributes(string $configured): array
-    {
-        return ($configured !== '') ? [$configured] : self::ID_ATTRIBUTES;
-    }
-
-    /**
-     * Stable id of an entry: the configured attribute, else objectGUID (as GUID string), entryUUID, nsUniqueId,
-     * ipaUniqueID, else the DN. Also used at login, with an entry of an LdapRecord raw query.
-     */
-    public static function entryId(array $entry, string $configured = ''): string
-    {
-        foreach (self::idAttributes($configured) as $attribute) {
-            $value = self::first($entry, $attribute);
-            if ($value !== null && $value !== '') {
-                return ($attribute === 'objectguid' && strlen($value) === 16) ? self::guid($value) : strtolower($value);
-            }
-        }
-        return 'dn:' . strtolower((string)(self::first($entry, 'dn') ?? ''));
-    }
-
-    /**
-     * @param array<string, DirectoryUser> $users
-     * @param array<string, string[]> $memberOf user id => lowercase group DNs
-     * @return array<string, DirectoryGroup>
-     */
-    private function groups(array $users, array $memberOf, string $userFilter): array
-    {
-        $s = $this->settings;
-        if ($s->groupFilter === '') {
-            throw new DirectoryException('The LDAP group filter is empty');
-        }
-        $userIds = [];
-        foreach ($users as $user) {
-            $userIds[$user->dn] = $user->id;
-        }
-        $attributes = array_merge([$s->groupNameAttribute, 'mail'], self::ID_ATTRIBUTES,
-            ($s->groupMemberMode === 'member') ? [$s->groupMemberAttribute] : []);
-        $groups = [];
-        $groupIds = [];
-        foreach ($this->search(($s->groupBasedn !== '') ? $s->groupBasedn : $s->basedn, self::wrap($s->groupFilter),
-            array_values(array_unique($attributes))) as $entry) {
-            $dn = strtolower((string)$entry['dn']);
-            $mail = self::first($entry, 'mail');
-            $group = new DirectoryGroup(self::entryId($entry), self::first($entry, $s->groupNameAttribute) ?? $dn,
-                ($mail === null) ? null : strtolower($mail));
-            if ($s->groupMemberMode === 'member') {
-                foreach (array_keys($entry) as $key) {
-                    // AD returns large member lists in ranges; a partial list would remove members
-                    if (is_string($key) && str_starts_with($key, $s->groupMemberAttribute . ';range=')) {
-                        throw new DirectoryException('LDAP returned a ranged member list for ' . $dn . ', use the memberOf or AD mode');
-                    }
-                }
-                foreach (self::values($entry, $s->groupMemberAttribute) as $memberDn) {
-                    if (isset($userIds[strtolower($memberDn)])) {
-                        $group->memberIds[] = $userIds[strtolower($memberDn)];
-                    }
-                }
-            }
-            $groups[$group->id] = $group;
-            $groupIds[$dn] = $group->id;
-        }
-        if ($s->groupMemberMode === 'memberof') {
-            foreach ($memberOf as $userId => $dns) {
-                foreach ($dns as $dn) {
-                    if (isset($groupIds[$dn])) {
-                        $groups[$groupIds[$dn]]->memberIds[] = (string)$userId;
-                    }
-                }
-            }
-        } elseif ($s->groupMemberMode === 'ad_chain') {
-            foreach ($groupIds as $dn => $groupId) {
-                $filter = '(&' . $userFilter . '(memberOf:' . self::AD_IN_CHAIN . ':=' . self::escape($dn) . '))';
-                foreach ($this->search($s->basedn, $filter, ['dn']) as $entry) {
-                    $userDn = strtolower((string)$entry['dn']);
-                    if (isset($userIds[$userDn])) {
-                        $groups[$groupId]->memberIds[] = $userIds[$userDn];
-                    }
-                }
-            }
-        }
-        foreach ($groups as $group) {
-            $group->memberIds = array_values(array_unique($group->memberIds));
-        }
-        return $groups;
-    }
-
-    /** Users of the configured filter that have the username attribute */
-    private function userFilter(): string
-    {
-        $filter = ($this->settings->filter !== '') ? self::wrap($this->settings->filter) : '(objectClass=*)';
-        return '(&' . $filter . '(' . $this->settings->usernameField . '=*))';
-    }
-
-    /**
-     * Paged search that fails instead of returning a partial result.
-     *
-     * @return array[] entries as returned by ldap_get_entries()
-     * @throws DirectoryException
-     */
-    protected function search(string $base, string $filter, array $attributes): array
-    {
-        if ($this->connection === null) {
-            throw new DirectoryException('No connection to the LDAP server');
-        }
-        $link = $this->connection->getLdapConnection()->getConnection();
-        $entries = [];
-        $cookie = '';
-        $paged = true;
-        while (true) {
-            $controls = $paged ? [['oid' => LDAP_CONTROL_PAGEDRESULTS, 'iscritical' => true,
-                'value' => ['size' => self::PAGE, 'cookie' => $cookie]]] : [];
-            $result = @ldap_search($link, $base, $filter, $attributes, 0, 0, 0, LDAP_DEREF_NEVER, $controls);
-            if ($result === false) {
-                throw new DirectoryException('LDAP search failed: ' . ldap_error($link));
-            }
-            $responseControls = [];
-            if (!ldap_parse_result($link, $result, $code, $matched, $message, $referrals, $responseControls)) {
-                throw new DirectoryException('LDAP result could not be read: ' . ldap_error($link));
-            }
-            // 12 = unavailableCriticalExtension: no paging on this server, search once without it
-            if ($code === 12 && $paged && $cookie === '') {
-                $paged = false;
-                continue;
-            }
-            if ($code !== 0) {
-                throw new DirectoryException('LDAP search incomplete (code ' . $code . '): ' . (($message !== '') ? $message : ldap_err2str($code)));
-            }
-            $page = ldap_get_entries($link, $result);
-            for ($i = 0; $i < (int)($page['count'] ?? 0); $i++) {
-                $entries[] = $page[$i];
-            }
-            $cookie = $paged ? (string)($responseControls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'] ?? '') : '';
-            if ($cookie === '') {
-                return $entries;
-            }
-        }
-    }
-
-    /** First value of an attribute: ldap_get_entries() format or LdapRecord attributes */
-    private static function first(array $entry, string $attribute): ?string
-    {
-        $values = self::values($entry, $attribute);
-        return $values[0] ?? null;
-    }
-
-    /** @return string[] */
-    private static function values(array $entry, string $attribute): array
-    {
-        $raw = $entry[$attribute] ?? $entry[strtolower($attribute)] ?? null;
-        if (!is_array($raw)) {
-            return is_scalar($raw) ? [(string)$raw] : [];
-        }
-        unset($raw['count']);
-        return array_values(array_map('strval', array_filter($raw, 'is_scalar')));
-    }
-
-    private static function wrap(string $filter): string
-    {
-        return str_starts_with($filter, '(') ? $filter : '(' . $filter . ')';
-    }
-
-    /** RFC 4515 value escaping */
-    private static function escape(string $value): string
-    {
-        return str_replace(['\\', '*', '(', ')', "\0"], ['\\5c', '\\2a', '\\28', '\\29', '\\00'], $value);
-    }
-
-    /** Binary objectGUID as the usual GUID string */
-    private static function guid(string $binary): string
-    {
-        $hex = bin2hex($binary);
-        return substr($hex, 6, 2) . substr($hex, 4, 2) . substr($hex, 2, 2) . substr($hex, 0, 2) . '-'
-            . substr($hex, 10, 2) . substr($hex, 8, 2) . '-' . substr($hex, 14, 2) . substr($hex, 12, 2) . '-'
-            . substr($hex, 16, 4) . '-' . substr($hex, 20, 12);
-    }
-}

data/web/inc/lib/mailcow/Identity/Http/KeycloakClient.php

@@ -1,92 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Http;
-
-use Mailcow\Identity\Directory\DirectoryException;
-
-/**
- * Admin REST API of a Keycloak realm with the client_credentials token of the mailcow client
- * (service account roles: view-users, query-groups). The HTTP call sits in send() so tests can replace it.
- */
-class KeycloakClient
-{
-    private ?string $token = null;
-
-    public function __construct(private string $serverUrl, private string $realm, private string $clientId,
-        private string $clientSecret, private bool $ignoreSslError = false)
-    {
-    }
-
-    /**
-     * GET below /admin/realms/<realm>, decoded JSON. A rejected token is renewed once.
-     *
-     * @throws DirectoryException
-     */
-    public function get(string $path, array $query = []): array
-    {
-        $url = $this->serverUrl . '/admin/realms/' . rawurlencode($this->realm) . $path . (empty($query) ? '' : '?' . http_build_query($query));
-        for ($attempt = 0; $attempt < 2; $attempt++) {
-            $this->token ??= $this->fetchToken();
-            [$code, $body] = $this->send('GET', $url, ['Authorization: Bearer ' . $this->token, 'Accept: application/json'], null);
-            if ($code === 401 && $attempt === 0) {
-                $this->token = null;
-                continue;
-            }
-            if ($code !== 200) {
-                throw new DirectoryException('Keycloak returned HTTP ' . $code . ' for ' . $path);
-            }
-            $data = json_decode($body, true);
-            if (!is_array($data)) {
-                throw new DirectoryException('Keycloak returned invalid JSON for ' . $path);
-            }
-            return $data;
-        }
-        throw new DirectoryException('Keycloak rejected the access token of the mailcow client');
-    }
-
-    /** @throws DirectoryException */
-    private function fetchToken(): string
-    {
-        [$code, $body] = $this->send('POST', $this->serverUrl . '/realms/' . rawurlencode($this->realm) . '/protocol/openid-connect/token',
-            ['Content-Type: application/x-www-form-urlencoded'],
-            http_build_query(['grant_type' => 'client_credentials', 'client_id' => $this->clientId, 'client_secret' => $this->clientSecret]));
-        $data = json_decode($body, true);
-        if ($code !== 200 || !is_array($data) || empty($data['access_token'])) {
-            throw new DirectoryException('Keycloak token request failed (HTTP ' . $code . ')');
-        }
-        return (string)$data['access_token'];
-    }
-
-    /**
-     * @return array{0: int, 1: string} HTTP status and body
-     * @throws DirectoryException on transport errors
-     */
-    protected function send(string $method, string $url, array $headers, ?string $body): array
-    {
-        $curl = curl_init($url);
-        curl_setopt_array($curl, [
-            CURLOPT_RETURNTRANSFER => true,
-            CURLOPT_CONNECTTIMEOUT => 5,
-            CURLOPT_TIMEOUT => 30,
-            CURLOPT_HTTPHEADER => $headers,
-        ]);
-        if ($method === 'POST') {
-            curl_setopt($curl, CURLOPT_POST, true);
-            curl_setopt($curl, CURLOPT_POSTFIELDS, (string)$body);
-        }
-        if ($this->ignoreSslError) {
-            curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
-            curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
-        }
-        $response = curl_exec($curl);
-        $code = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE);
-        $error = curl_error($curl);
-        curl_close($curl);
-        if ($response === false) {
-            throw new DirectoryException('Keycloak request failed: ' . $error);
-        }
-        return [$code, (string)$response];
-    }
-}

data/web/inc/lib/mailcow/Identity/IdentityModule.php

@@ -1,85 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity;
-
-use Closure;
-use LdapRecord\Connection;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Identity\Controller\IdentityApiController;
-use Mailcow\Identity\Directory\Directory;
-use Mailcow\Identity\Directory\KeycloakDirectory;
-use Mailcow\Identity\Directory\LdapDirectory;
-use Mailcow\Identity\Http\KeycloakClient;
-use Mailcow\Identity\Legacy\LegacyAccounts;
-use Mailcow\Identity\Model\Settings;
-use Mailcow\Identity\Repository\IdentityRepository;
-use Mailcow\Identity\Repository\RunRepository;
-use Mailcow\Identity\Service\LinkService;
-use Mailcow\Identity\Service\StatusService;
-use Mailcow\Identity\Service\SyncPlanner;
-use Mailcow\Identity\Service\SyncService;
-use Mailcow\MailboxGroup\MailboxGroupModule;
-use Mailcow\Sogo\SogoModule;
-use PDO;
-
-/**
- * Composition root of the identity module (IdP links, sync, offboarding).
- *
- *   IdentityModule::create($pdo, Context::system(), identity_provider('get'), fn() => identity_provider('init'))->sync->runDue();  // cron
- *   IdentityModule::fromGlobals()->api->getLink($username);  // edit page
- *
- * Logins use the links through Mailcow\Auth (AuthModule::fromGlobals()->idp).
- */
-class IdentityModule
-{
-    public function __construct(
-        public readonly Settings $settings,
-        public readonly SyncService $sync,
-        public readonly LinkService $links,
-        public readonly StatusService $status,
-        public readonly IdentityApiController $api
-    ) {
-    }
-
-    /**
-     * @param array $settings identity_provider('get')
-     * @param ?Closure $ldap returns the bound LDAP connection (identity_provider('init')), called when a run starts
-     */
-    public static function create(PDO $pdo, Context $context, array $settings, ?Closure $ldap = null): self
-    {
-        $config = Settings::fromArray($settings);
-        $repo = new IdentityRepository($pdo);
-        $runs = new RunRepository($pdo);
-        $groups = MailboxGroupModule::create($pdo, Context::system());
-        $status = new StatusService($config, $repo, $runs, $context);
-        $directory = fn(): ?Directory => self::directory($config, $ldap);
-        return new self($config,
-            new SyncService($config, $directory, $repo, $runs, new SyncPlanner(), $groups->idp, $groups->templates, new LegacyAccounts(),
-                SogoModule::create()->sogo),
-            new LinkService($repo), $status, new IdentityApiController($status, new ApiResponse()));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo, $iam_settings and $_SESSION; $context overrides the session caller */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        $settings = $GLOBALS['iam_settings'] ?? null;
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []),
-            is_array($settings) ? $settings : identity_provider('get'), fn(): mixed => $GLOBALS['iam_provider'] ?? null);
-    }
-
-    private static function directory(Settings $settings, ?Closure $ldap): ?Directory
-    {
-        if ($settings->provider === Settings::KEYCLOAK) {
-            return new KeycloakDirectory(new KeycloakClient($settings->serverUrl, $settings->realm, $settings->clientId,
-                $settings->clientSecret, $settings->ignoreSslError), $settings);
-        }
-        if ($settings->provider === Settings::LDAP) {
-            $connection = ($ldap === null) ? null : $ldap();
-            return new LdapDirectory(($connection instanceof Connection) ? $connection : null, $settings);
-        }
-        return null;
-    }
-}

data/web/inc/lib/mailcow/Identity/Legacy/LegacyAccounts.php

@@ -1,30 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Legacy;
-
-use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
-
-/**
- * Bridge to the procedural mailbox code for the IdP sync: creating mailboxes, applying templates.
- */
-class LegacyAccounts extends LegacyMailbox
-{
-    /**
-     * Creates a mailbox of the IdP from a template, like the IdP login does.
-     *
-     * @return ?string null on success, else the message key of the legacy error
-     */
-    public function createMailbox(string $email, string $name, string $authsource, string $template): ?string
-    {
-        return $this->asSystem(fn(): mixed => $this->callCreate($email, $name, $authsource, $template));
-    }
-
-    protected function callCreate(string $email, string $name, string $authsource, string $template): mixed
-    {
-        [$local, $domain] = explode('@', $email, 2);
-        return mailbox('add', 'mailbox_from_template', ['domain' => $domain, 'local_part' => $local, 'name' => $name,
-            'authsource' => $authsource, 'template' => $template]);
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/Account.php

@@ -1,27 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * A mailbox with the authsource of the configured IdP and its link (table `identity_link`).
- */
-class Account
-{
-    /**
-     * @param int $active       mailbox.active: 1 active, 2 login disabled (still receives mail), 0 inactive
-     * @param bool $idpDisabled the sync set active = 2, so it may undo it
-     * @param ?string $missingSince first complete run that did not list the user
-     */
-    public function __construct(
-        public readonly string $username,
-        public readonly string $name,
-        public readonly int $active,
-        public readonly ?string $externalId = null,
-        public readonly bool $idpDisabled = false,
-        public readonly ?string $missingSince = null,
-        public readonly string $attributeHash = ''
-    ) {
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/DirectoryGroup.php

@@ -1,23 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * A selected IdP group with the ids of its member users.
- */
-class DirectoryGroup
-{
-    /**
-     * @param ?string $mail      group address in the IdP, decides the domain of an auto-created group
-     * @param string[] $memberIds ids of the member users (DirectoryUser::$id)
-     */
-    public function __construct(
-        public readonly string $id,
-        public readonly string $name,
-        public readonly ?string $mail = null,
-        public array $memberIds = []
-    ) {
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/DirectoryUser.php

@@ -1,26 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * A user as listed by the IdP. `id` is stable across renames (Keycloak user id, LDAP objectGUID/entryUUID/...).
- */
-class DirectoryUser
-{
-    /**
-     * @param ?string $email   lowercase, null if the user has none
-     * @param string[] $templateValues values of the template attribute (legacy mapping)
-     * @param string $dn       LDAP only, lowercase, used to resolve group members
-     */
-    public function __construct(
-        public readonly string $id,
-        public readonly ?string $email,
-        public readonly string $name,
-        public readonly bool $enabled,
-        public readonly array $templateValues = [],
-        public readonly string $dn = ''
-    ) {
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/Listing.php

@@ -1,20 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * A complete listing of the IdP. Directories never return partial listings, they throw DirectoryException,
- * so a user missing here is really gone (or out of the filter).
- */
-class Listing
-{
-    /**
-     * @param array<string, DirectoryUser> $users by id
-     * @param ?array<string, DirectoryGroup> $groups by id, null when groups were not listed
-     */
-    public function __construct(public readonly array $users, public readonly ?array $groups = null)
-    {
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/Settings.php

@@ -1,118 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * Identity provider settings (table `identity_provider`, key/value, see identity_provider('get')) with defaults.
- */
-class Settings
-{
-    public const KEYCLOAK = 'keycloak';
-    public const LDAP = 'ldap';
-    public const GENERIC_OIDC = 'generic-oidc';
-    public const MEMBER_MODES = ['memberof', 'member', 'ad_chain'];
-
-    public string $provider = '';
-    public bool $periodicSync = false;
-    public bool $importUsers = false;
-    /** Minutes between scheduled runs */
-    public int $syncInterval = 15;
-    /** Users disabled or removed in the IdP get login disabled (mailbox.active = 2) */
-    public bool $offboarding = false;
-    /** Max. share (percent) of active IdP mailboxes one run may deactivate, at least 5 mailboxes */
-    public int $offboardingLimit = 10;
-    public bool $groupSync = false;
-    public bool $groupAutocreate = true;
-    public string $groupDefaultDomain = '';
-    public string $defaultTemplate = '';
-    /** @var string[] attribute value per mapping, keys aligned with $templates */
-    public array $mappers = [];
-    /** @var string[] */
-    public array $templates = [];
-    public string $serverUrl = '';
-    public string $realm = '';
-    public string $clientId = '';
-    public string $clientSecret = '';
-    public bool $ignoreSslError = false;
-    /** @var string[] Keycloak group paths: '/path', '/path/*' (subtree) or '/*' (all) */
-    public array $groupPaths = [];
-    public string $basedn = '';
-    public string $filter = '';
-    public string $usernameField = 'mail';
-    public string $attributeField = '';
-    /** LDAP attribute with the stable user id, '' = objectGUID, entryUUID, nsUniqueId, ipaUniqueID, else the DN */
-    public string $idAttribute = '';
-    /** LDAP filter matching disabled users (besides AD userAccountControl) */
-    public string $disabledFilter = '';
-    public string $groupBasedn = '';
-    public string $groupFilter = '';
-    public string $groupNameAttribute = 'cn';
-    public string $groupMemberMode = 'memberof';
-    public string $groupMemberAttribute = 'member';
-
-    public static function fromArray(array $s): self
-    {
-        $settings = new self();
-        $settings->provider = (string)($s['authsource'] ?? '');
-        $settings->periodicSync = self::flag($s['periodic_sync'] ?? 0);
-        $settings->importUsers = self::flag($s['import_users'] ?? 0);
-        $settings->syncInterval = max(1, (int)($s['sync_interval'] ?? 15));
-        $settings->offboarding = self::flag($s['offboarding'] ?? 0);
-        $settings->offboardingLimit = min(100, max(0, (int)($s['offboarding_limit'] ?? 10)));
-        $settings->groupSync = self::flag($s['group_sync'] ?? 0);
-        $settings->groupAutocreate = self::flag($s['group_autocreate'] ?? 1);
-        $settings->groupDefaultDomain = strtolower(trim((string)($s['group_default_domain'] ?? '')));
-        $settings->defaultTemplate = (string)($s['default_template'] ?? '');
-        $settings->mappers = array_map('strval', (array)($s['mappers'] ?? []));
-        $settings->templates = array_map('strval', (array)($s['templates'] ?? []));
-        $settings->serverUrl = rtrim((string)($s['server_url'] ?? ''), '/');
-        $settings->realm = (string)($s['realm'] ?? '');
-        $settings->clientId = (string)($s['client_id'] ?? '');
-        $settings->clientSecret = (string)($s['client_secret'] ?? '');
-        $settings->ignoreSslError = self::flag($s['ignore_ssl_error'] ?? 0);
-        $settings->groupPaths = array_values(array_filter(array_map('trim', explode(',', (string)($s['group_paths'] ?? ''))), 'strlen'));
-        $settings->basedn = (string)($s['basedn'] ?? '');
-        $settings->filter = trim((string)($s['filter'] ?? ''));
-        $settings->usernameField = strtolower((string)(($s['username_field'] ?? '') ?: 'mail'));
-        $settings->attributeField = strtolower(trim((string)($s['attribute_field'] ?? '')));
-        $settings->idAttribute = strtolower(trim((string)($s['id_attribute'] ?? '')));
-        $settings->disabledFilter = trim((string)($s['disabled_filter'] ?? ''));
-        $settings->groupBasedn = trim((string)($s['group_basedn'] ?? ''));
-        $settings->groupFilter = trim((string)($s['group_filter'] ?? ''));
-        $settings->groupNameAttribute = strtolower((string)(($s['group_name_attribute'] ?? '') ?: 'cn'));
-        $mode = (string)($s['group_member_mode'] ?? 'memberof');
-        $settings->groupMemberMode = in_array($mode, self::MEMBER_MODES, true) ? $mode : 'memberof';
-        $settings->groupMemberAttribute = strtolower((string)(($s['group_member_attribute'] ?? '') ?: 'member'));
-        return $settings;
-    }
-
-    /** Only Keycloak and LDAP can be listed by the sync */
-    public function canSync(): bool
-    {
-        return in_array($this->provider, [self::KEYCLOAK, self::LDAP], true);
-    }
-
-    public function syncEnabled(): bool
-    {
-        return $this->canSync() && ($this->periodicSync || $this->importUsers || $this->offboarding || $this->groupSync);
-    }
-
-    /** Template of the first mapping (in configured order) matching one of the values, null if none */
-    public function mappedTemplate(array $values): ?string
-    {
-        $values = array_map('strval', array_filter($values, 'is_scalar'));
-        foreach ($this->mappers as $key => $mapper) {
-            if (in_array($mapper, $values, true) && ($this->templates[$key] ?? '') !== '') {
-                return $this->templates[$key];
-            }
-        }
-        return null;
-    }
-
-    private static function flag(mixed $value): bool
-    {
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/State.php

@@ -1,30 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * What mailcow knows before a sync run, read by SyncService and planned against by SyncPlanner.
- */
-class State
-{
-    /**
-     * @param array<string, Account> $accounts           mailboxes with the provider as authsource, by username
-     * @param array<string, string> $mailboxes           username => authsource of every mailbox
-     * @param string[] $domains                          mailcow domains
-     * @param array[] $groups                            mailbox groups {id, domain, name, idp_ref, template, priority,
-     *                                                   active, members: username => source}
-     * @param array<string, ?string> $effectiveTemplates username => effective group template
-     * @param array<string, string> $templateHashes      template name => attribute hash
-     */
-    public function __construct(
-        public readonly array $accounts,
-        public readonly array $mailboxes = [],
-        public readonly array $domains = [],
-        public readonly array $groups = [],
-        public readonly array $effectiveTemplates = [],
-        public readonly array $templateHashes = []
-    ) {
-    }
-}

data/web/inc/lib/mailcow/Identity/Model/SyncPlan.php

@@ -1,73 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Model;
-
-/**
- * What one sync run changes. Built by SyncPlanner from a listing and the current state, applied by SyncService.
- */
-class SyncPlan
-{
-    /** Details per kind kept in the stored summary */
-    private const DETAILS = 50;
-
-    /** @var array<string, string> username => external id (adopted by email or relinked) */
-    public array $links = [];
-    /** @var array[] {email, name, template, id}: mailboxes to create */
-    public array $creates = [];
-    /** @var string[] usernames set to login disabled (active = 2) */
-    public array $deactivate = [];
-    /** @var string[] usernames the sync had disabled and the IdP enabled again (active = 1) */
-    public array $restore = [];
-    /** @var string[] usernames whose idp_disabled marker is dropped, active stays (an admin changed it) */
-    public array $release = [];
-    /** @var string[] usernames not listed for the first time (missing_since is set) */
-    public array $missing = [];
-    /** @var string[] usernames listed again (missing_since is cleared) */
-    public array $found = [];
-    /** @var array[] {username, name, template}: existing mailboxes whose name/template is synced; template = legacy mapping */
-    public array $updates = [];
-    /** @var array<string, array> username => {name, template}: IdP name and legacy mapping of every listed mailbox */
-    public array $profiles = [];
-    /** @var array[] {ref, name, domain, mail}: groups to create for IdP groups */
-    public array $groupCreates = [];
-    /** @var array<int, string> group id => new name */
-    public array $groupRenames = [];
-    /** @var array<string, string[]> group ref => usernames of its IdP members */
-    public array $memberships = [];
-    /** @var int[] linked groups no longer listed; their IdP members are removed */
-    public array $orphanedGroups = [];
-    /** Number of IdP memberships the plan removes (for the guard) */
-    public int $membershipRemovals = 0;
-    /** @var string[] situations an admin has to resolve */
-    public array $conflicts = [];
-    /** Users without email, not in a mailcow domain or without template (not created) */
-    public int $skipped = 0;
-    /** The guard stopped deactivations and membership removals */
-    public bool $blocked = false;
-
-    /** Stored with the run and shown in the UI */
-    public function summary(): array
-    {
-        $summary = ['counts' => [], 'details' => []];
-        foreach (['links' => 'links', 'creates' => 'creates', 'deactivate' => 'deactivate', 'restore' => 'restore',
-            'release' => 'release', 'missing' => 'missing', 'found' => 'found', 'updates' => 'updates', 'group_creates' => 'groupCreates',
-            'group_renames' => 'groupRenames', 'memberships' => 'memberships', 'orphaned_groups' => 'orphanedGroups',
-            'conflicts' => 'conflicts'] as $key => $kind) {
-            $summary['counts'][$key] = count($this->$kind);
-        }
-        $summary['counts']['membership_removals'] = $this->membershipRemovals;
-        $summary['counts']['skipped'] = $this->skipped;
-        $summary['details'] = [
-            'creates' => array_slice(array_column($this->creates, 'email'), 0, self::DETAILS),
-            'deactivate' => array_slice($this->deactivate, 0, self::DETAILS),
-            'restore' => array_slice($this->restore, 0, self::DETAILS),
-            'missing' => array_slice($this->missing, 0, self::DETAILS),
-            'group_creates' => array_slice(array_column($this->groupCreates, 'name'), 0, self::DETAILS),
-            'conflicts' => array_slice($this->conflicts, 0, self::DETAILS),
-        ];
-        $summary['blocked'] = $this->blocked;
-        return $summary;
-    }
-}

data/web/inc/lib/mailcow/Identity/Repository/IdentityRepository.php

@@ -1,130 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\Identity\Model\Account;
-use PDO;
-
-/**
- * SQL for IdP mailboxes and their links (table `identity_link`).
- */
-class IdentityRepository extends Repository
-{
-    /** @return array<string, Account> mailboxes with the provider as authsource, by username */
-    public function accounts(string $provider): array
-    {
-        $rows = $this->query("SELECT m.`username`, m.`name`, m.`active`,
-            JSON_UNQUOTE(JSON_EXTRACT(m.`attributes`, '$.attribute_hash')) AS `attribute_hash`,
-            l.`external_id`, l.`idp_disabled`, l.`missing_since`
-          FROM `mailbox` m
-            LEFT JOIN `identity_link` l ON l.`username` = m.`username` AND l.`provider` = :p1
-          WHERE m.`authsource` = :p2 AND m.`kind` = ''", ['p1' => $provider, 'p2' => $provider])->fetchAll(PDO::FETCH_ASSOC);
-        $accounts = [];
-        foreach ($rows as $row) {
-            $accounts[(string)$row['username']] = new Account((string)$row['username'], (string)$row['name'], (int)$row['active'],
-                isset($row['external_id']) ? (string)$row['external_id'] : null, (int)($row['idp_disabled'] ?? 0) === 1,
-                isset($row['missing_since']) ? (string)$row['missing_since'] : null, (string)($row['attribute_hash'] ?? ''));
-        }
-        return $accounts;
-    }
-
-    /** @return array{accounts: int, linked: int, idp_disabled: int, missing: int} mailboxes of the provider */
-    public function stats(string $provider): array
-    {
-        $row = $this->query("SELECT COUNT(*) AS `accounts`, COUNT(l.`external_id`) AS `linked`,
-            COALESCE(SUM(l.`idp_disabled`), 0) AS `idp_disabled`, COUNT(l.`missing_since`) AS `missing`
-          FROM `mailbox` m LEFT JOIN `identity_link` l ON l.`username` = m.`username` AND l.`provider` = :p1
-          WHERE m.`authsource` = :p2 AND m.`kind` = ''", ['p1' => $provider, 'p2' => $provider])->fetch(PDO::FETCH_ASSOC);
-        return array_map('intval', $row);
-    }
-
-    /** @return array<string, string> username => authsource of every mailbox (incl. resources) */
-    public function mailboxes(): array
-    {
-        return $this->query("SELECT `username`, `authsource` FROM `mailbox`")->fetchAll(PDO::FETCH_KEY_PAIR);
-    }
-
-    /** @return string[] mailcow domains (no alias domains) */
-    public function domains(): array
-    {
-        return $this->query("SELECT `domain` FROM `domain`")->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /** @return array<string, string> template name => hash as stored by mailbox_from_template in attributes.attribute_hash */
-    public function templateHashes(): array
-    {
-        $hashes = [];
-        foreach ($this->query("SELECT `template`, `attributes` FROM `templates` WHERE `type` = 'mailbox'")->fetchAll(PDO::FETCH_ASSOC) as $row) {
-            $hashes[(string)$row['template']] = sha1(json_encode($row['attributes']));
-        }
-        return $hashes;
-    }
-
-    /** @return ?array{provider: string, external_id: string, idp_disabled: int, missing_since: ?string} */
-    public function findLink(string $username): ?array
-    {
-        $row = $this->query("SELECT `provider`, `external_id`, `idp_disabled`, `missing_since` FROM `identity_link` WHERE `username` = :username",
-            ['username' => $username])->fetch(PDO::FETCH_ASSOC);
-        return $row ?: null;
-    }
-
-    /** Mailbox linked to the IdP user; links of mailboxes that no longer use the provider do not count */
-    public function usernameOf(string $provider, string $externalId): ?string
-    {
-        $username = $this->query("SELECT l.`username` FROM `identity_link` l
-            JOIN `mailbox` m ON m.`username` = l.`username` AND m.`authsource` = l.`provider`
-          WHERE l.`provider` = :provider AND l.`external_id` = :id", ['provider' => $provider, 'id' => $externalId])->fetchColumn();
-        return ($username === false) ? null : (string)$username;
-    }
-
-    /**
-     * Links the mailbox to the IdP user; an older link of the mailbox is replaced. Callers make sure the IdP user
-     * is not linked to another mailbox of the provider (usernameOf()); stale links of the id are dropped.
-     */
-    public function link(string $username, string $provider, string $externalId): void
-    {
-        $this->transaction(function () use ($username, $provider, $externalId): void {
-            $this->query("DELETE FROM `identity_link` WHERE `provider` = :provider AND `external_id` = :id AND `username` != :username",
-                ['provider' => $provider, 'id' => $externalId, 'username' => $username]);
-            $this->query("INSERT INTO `identity_link` (`username`, `provider`, `external_id`) VALUES (:username, :provider, :id)
-              ON DUPLICATE KEY UPDATE `provider` = VALUES(`provider`), `external_id` = VALUES(`external_id`), `missing_since` = NULL",
-                ['username' => $username, 'provider' => $provider, 'id' => $externalId]);
-        });
-    }
-
-    public function unlink(string $username): void
-    {
-        $this->query("DELETE FROM `identity_link` WHERE `username` = :username", ['username' => $username]);
-    }
-
-    /** Marks the IdP mailbox as not listed (now) or listed again; unlinked mailboxes get a link row without id */
-    public function setMissing(string $username, string $provider, bool $missing): void
-    {
-        $this->query("INSERT INTO `identity_link` (`username`, `provider`, `missing_since`) VALUES (:username, :provider, "
-            . ($missing ? "NOW()" : "NULL") . ") ON DUPLICATE KEY UPDATE `missing_since` = VALUES(`missing_since`)",
-            ['username' => $username, 'provider' => $provider]);
-    }
-
-    /**
-     * Sets mailbox.active like the mailbox edit does (incl. the mailbox's own alias row) and records whether the
-     * IdP sync caused it.
-     */
-    public function setActive(string $username, string $provider, int $active, bool $idpDisabled): void
-    {
-        $this->transaction(function () use ($username, $provider, $active, $idpDisabled): void {
-            $this->query("UPDATE `mailbox` SET `active` = :active WHERE `username` = :username", ['active' => $active, 'username' => $username]);
-            $this->query("UPDATE `alias` SET `active` = :active WHERE `address` = :username", ['active' => $active, 'username' => $username]);
-            $this->setIdpDisabled($username, $provider, $idpDisabled);
-        });
-    }
-
-    public function setIdpDisabled(string $username, string $provider, bool $idpDisabled): void
-    {
-        $this->query("INSERT INTO `identity_link` (`username`, `provider`, `idp_disabled`) VALUES (:username, :provider, :flag)
-          ON DUPLICATE KEY UPDATE `idp_disabled` = VALUES(`idp_disabled`)",
-            ['username' => $username, 'provider' => $provider, 'flag' => $idpDisabled ? 1 : 0]);
-    }
-}

data/web/inc/lib/mailcow/Identity/Repository/RunRepository.php

@@ -1,81 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * Sync runs (table `identity_sync_run`): requests from the UI (status pending), the run itself and its summary.
- */
-class RunRepository extends Repository
-{
-    private const KEEP = 30;
-
-    /** Queues a run for the cron; an identical pending request is reused */
-    public function request(string $provider, string $mode, string $requestedBy): int
-    {
-        $id = $this->query("SELECT `id` FROM `identity_sync_run` WHERE `status` = 'pending' AND `mode` = :mode",
-            ['mode' => $mode])->fetchColumn();
-        if ($id !== false) {
-            return (int)$id;
-        }
-        return $this->insertRow('identity_sync_run', ['provider' => $provider, 'mode' => $mode, 'status' => 'pending',
-            'requested_by' => $requestedBy]);
-    }
-
-    /** @return ?array oldest pending request */
-    public function nextPending(): ?array
-    {
-        $row = $this->query("SELECT * FROM `identity_sync_run` WHERE `status` = 'pending' ORDER BY `id` LIMIT 1")->fetch(PDO::FETCH_ASSOC);
-        return $row ?: null;
-    }
-
-    /** A scheduled run (not requested by anyone) */
-    public function create(string $provider, string $mode): int
-    {
-        return $this->insertRow('identity_sync_run', ['provider' => $provider, 'mode' => $mode, 'status' => 'pending', 'requested_by' => '']);
-    }
-
-    public function start(int $id): void
-    {
-        $this->query("UPDATE `identity_sync_run` SET `status` = 'running', `started` = NOW() WHERE `id` = :id", ['id' => $id]);
-    }
-
-    public function finish(int $id, string $status, array $summary): void
-    {
-        $this->query("UPDATE `identity_sync_run` SET `status` = :status, `finished` = NOW(), `summary` = :summary WHERE `id` = :id",
-            ['status' => $status, 'summary' => json_encode($summary), 'id' => $id]);
-        $this->query("DELETE FROM `identity_sync_run` WHERE `id` NOT IN (SELECT `id` FROM (SELECT `id` FROM `identity_sync_run`
-          ORDER BY `id` DESC LIMIT " . self::KEEP . ") AS `keep`)");
-    }
-
-    /** Seconds since the last scheduled run started, null if there was none */
-    public function secondsSinceScheduled(): ?int
-    {
-        $seconds = $this->query("SELECT TIMESTAMPDIFF(SECOND, MAX(`started`), NOW()) FROM `identity_sync_run`
-          WHERE `requested_by` = '' AND `mode` = 'run'")->fetchColumn();
-        return ($seconds === false || $seconds === null) ? null : (int)$seconds;
-    }
-
-    /** Runs left in status running by a crashed cron are closed, so they do not hide new runs */
-    public function failStale(int $minutes): void
-    {
-        $this->query("UPDATE `identity_sync_run` SET `status` = 'error', `finished` = NOW(),
-            `summary` = JSON_OBJECT('error', 'The sync run did not finish')
-          WHERE `status` = 'running' AND `started` < NOW() - INTERVAL :minutes MINUTE", ['minutes' => $minutes]);
-    }
-
-    /** @return array[] newest first, summary decoded */
-    public function recent(int $limit = 10): array
-    {
-        $rows = $this->query("SELECT * FROM `identity_sync_run` ORDER BY `id` DESC LIMIT " . max(1, $limit))->fetchAll(PDO::FETCH_ASSOC);
-        foreach ($rows as &$row) {
-            $row['id'] = (int)$row['id'];
-            $row['summary'] = ($row['summary'] === null) ? null : json_decode((string)$row['summary'], true);
-        }
-        return $rows;
-    }
-}

data/web/inc/lib/mailcow/Identity/Service/LinkService.php

@@ -1,51 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Service;
-
-use Mailcow\Identity\Repository\IdentityRepository;
-
-/**
- * Links of mailboxes to IdP users, checked at IdP logins (REST, LDAP bind, OIDC). Mailboxes are found by email,
- * so the IdP user id has to match the link: an address taken over by another IdP user must not open the old
- * mailbox. Unlinked mailboxes are linked on their first login.
- */
-class LinkService
-{
-    public function __construct(private IdentityRepository $repo)
-    {
-    }
-
-    /** The IdP user may log in to the mailbox; links the mailbox when it has no link yet */
-    public function verify(string $provider, string $username, string $externalId): bool
-    {
-        if ($externalId === '') {
-            return false;
-        }
-        $link = $this->repo->findLink($username);
-        if ($link !== null && $link['provider'] === $provider && $link['external_id'] !== null) {
-            return hash_equals((string)$link['external_id'], $externalId);
-        }
-        if (!$this->mayCreate($provider, $username, $externalId)) {
-            return false;
-        }
-        $this->repo->link($username, $provider, $externalId);
-        return true;
-    }
-
-    /** The IdP user owns no other mailbox (checked before a mailbox is created on login) */
-    public function mayCreate(string $provider, string $username, string $externalId): bool
-    {
-        $owner = ($externalId === '') ? null : $this->repo->usernameOf($provider, $externalId);
-        return $externalId !== '' && ($owner === null || $owner === $username);
-    }
-
-    /** Links a mailbox created on login */
-    public function link(string $provider, string $username, string $externalId): void
-    {
-        if ($externalId !== '') {
-            $this->repo->link($username, $provider, $externalId);
-        }
-    }
-}

data/web/inc/lib/mailcow/Identity/Service/StatusService.php

@@ -1,80 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\Identity\Model\Settings;
-use Mailcow\Identity\Repository\IdentityRepository;
-use Mailcow\Identity\Repository\RunRepository;
-
-/**
- * Admin use cases of the sync: status, run requests (executed by the cron) and links of mailboxes.
- * Admins only, read-only API keys may read. Domain admins may read the link of mailboxes they manage.
- */
-class StatusService
-{
-    public const MODES = ['preview', 'run', 'force'];
-
-    public function __construct(private Settings $settings, private IdentityRepository $repo, private RunRepository $runs,
-        private Context $context)
-    {
-    }
-
-    /** @throws AccessDenied */
-    public function status(): array
-    {
-        $this->context->requireAdmin(false);
-        return [
-            'provider' => $this->settings->provider,
-            'can_sync' => $this->settings->canSync(),
-            'sync_enabled' => $this->settings->syncEnabled(),
-            'counts' => $this->repo->stats($this->settings->provider),
-            'runs' => $this->runs->recent(),
-        ];
-    }
-
-    /**
-     * Queues a run for the cron: preview (dry run), run, or force (run without the change limit).
-     *
-     * @throws UserError
-     */
-    public function requestRun(mixed $mode): int
-    {
-        $this->context->requireAdmin();
-        $mode = Input::text($mode);
-        if (!in_array($mode, self::MODES, true)) {
-            throw new UserError('iam_sync_mode_invalid');
-        }
-        if (!$this->settings->canSync()) {
-            throw new UserError('iam_sync_unavailable');
-        }
-        return $this->runs->request($this->settings->provider, $mode, (string)$this->context->username);
-    }
-
-    /** Link of a mailbox for the mailbox edit page, null if none or not allowed */
-    public function link(string $username): ?array
-    {
-        $username = strtolower(trim($username));
-        if (!($this->context->hasAdminRights() || ($this->context->isDomainAdmin() && $this->context->canAccessMailbox($username)))) {
-            return null;
-        }
-        return $this->repo->findLink($username);
-    }
-
-    /**
-     * Drops the link, the mailbox is linked again by email (next run or login). Resolves conflicts, e.g. when an
-     * address was given to another IdP user.
-     *
-     * @throws AccessDenied
-     */
-    public function unlink(string $username): void
-    {
-        $this->context->requireAdmin();
-        $this->repo->unlink(strtolower(trim($username)));
-    }
-}

data/web/inc/lib/mailcow/Identity/Service/SyncPlanner.php

@@ -1,311 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Service;
-
-use Mailcow\Identity\Model\Account;
-use Mailcow\Identity\Model\DirectoryGroup;
-use Mailcow\Identity\Model\DirectoryUser;
-use Mailcow\Identity\Model\Listing;
-use Mailcow\Identity\Model\Settings;
-use Mailcow\Identity\Model\State;
-use Mailcow\Identity\Model\SyncPlan;
-
-/**
- * Decides what a sync run changes, without side effects.
- *
- * Users: linked mailboxes are found by IdP user id, unlinked ones are adopted by email. A mailbox linked to
- *        another IdP user, or an IdP user whose email moved away from its mailbox, is a conflict; nothing is
- *        relinked or created for it.
- * Offboarding: a user disabled in the IdP, or not listed by two consecutive complete runs, gets login disabled
- *        (active = 2), once per IdP disable. Only those mailboxes are activated again when the IdP enables
- *        the user; manual changes of active always win.
- * Groups: IdP groups map to mailbox groups by idp_ref; unknown ones are created in the domain of their mail,
- *        else in the default group domain. Linked users that are not listed keep their IdP memberships.
- * Guard: more deactivations or membership removals than max(5, limit % of active IdP mailboxes resp. IdP
- *        memberships) block the run unless forced.
- */
-class SyncPlanner
-{
-    private const GUARD_MIN = 5;
-
-    public function plan(Settings $settings, Listing $listing, State $state, bool $force = false): SyncPlan
-    {
-        $plan = new SyncPlan();
-        $owners = [];
-        foreach ($state->accounts as $username => $account) {
-            if ($account->externalId !== null) {
-                $owners[$account->externalId] = (string)$username;
-            }
-        }
-        /** @var array<string, DirectoryUser> $matched username => listed user */
-        $matched = [];
-        /** @var array<string, DirectoryUser> $candidates email => user to create, template still open */
-        $candidates = [];
-        $duplicates = [];
-        foreach ($listing->users as $user) {
-            if (isset($owners[$user->id])) {
-                $username = $owners[$user->id];
-                if ($user->email !== null && $user->email !== $username) {
-                    self::conflict($plan, $username . ' is linked to an IdP user whose email is now ' . $user->email);
-                }
-                $matched[$username] = $user;
-            } elseif ($user->email === null) {
-                $plan->skipped++;
-            } elseif (isset($state->accounts[$user->email])) {
-                $account = $state->accounts[$user->email];
-                if ($account->externalId !== null || isset($plan->links[$user->email])) {
-                    self::conflict($plan, $user->email . ' is linked to another IdP user');
-                    continue;
-                }
-                $plan->links[$user->email] = $user->id;
-                $matched[$user->email] = $user;
-            } elseif (isset($state->mailboxes[$user->email]) || !$settings->importUsers || !$user->enabled
-                || !in_array(substr((string)strrchr($user->email, '@'), 1), $state->domains, true)) {
-                // other authsource, import off, disabled or no mailcow domain
-                $plan->skipped++;
-            } elseif (isset($candidates[$user->email]) || isset($duplicates[$user->email])) {
-                self::conflict($plan, 'Several IdP users have the email ' . $user->email);
-                unset($candidates[$user->email]);
-                $duplicates[$user->email] = true;
-            } else {
-                $candidates[$user->email] = $user;
-            }
-        }
-        $groups = ($settings->groupSync && $listing->groups !== null) ? $listing->groups : null;
-        $this->planCreates($settings, $candidates, $groups ?? [], $state, $plan);
-        $usernames = [];
-        foreach ($matched as $username => $user) {
-            $usernames[$user->id] = (string)$username;
-        }
-        foreach ($plan->creates as $create) {
-            $usernames[$create['id']] = $create['email'];
-        }
-        if ($groups !== null) {
-            $this->planGroups($settings, $groups, $state, $usernames, $matched, $plan);
-        }
-        $this->planStates($settings, $state, $matched, $plan);
-        if ($settings->periodicSync) {
-            $this->planUpdates($settings, $state, $matched, $plan);
-        }
-        $active = count(array_filter($state->accounts, fn(Account $a): bool => $a->active === 1));
-        $memberships = array_sum(array_map(fn(array $g): int => count(self::idpMembers($g)), $state->groups));
-        if (!$force && (count($plan->deactivate) > self::limit($active, $settings->offboardingLimit)
-            || $plan->membershipRemovals > self::limit($memberships, $settings->offboardingLimit))) {
-            $plan->blocked = true;
-        }
-        return $plan;
-    }
-
-    /**
-     * @param array<string, DirectoryGroup> $groups
-     * @param array<string, string> $usernames user id => username of matched and new users
-     */
-    private function planGroups(Settings $settings, array $groups, State $state, array $usernames, array $matched, SyncPlan $plan): void
-    {
-        $linked = [];
-        $names = [];
-        foreach ($state->groups as $group) {
-            if ($group['idp_ref'] !== null) {
-                $linked[(string)$group['idp_ref']] = $group;
-            }
-            $names[$group['domain']][mb_strtolower($group['name'])] = (int)$group['id'];
-        }
-        foreach ($groups as $ref => $directoryGroup) {
-            $ref = (string)$ref;
-            $members = [];
-            foreach ($directoryGroup->memberIds as $id) {
-                if (isset($usernames[$id])) {
-                    $members[] = $usernames[$id];
-                }
-            }
-            $current = $linked[$ref] ?? null;
-            if ($current === null) {
-                if (!$settings->groupAutocreate) {
-                    continue;
-                }
-                $domain = self::groupDomain($settings, $directoryGroup, $state->domains);
-                if ($domain === null) {
-                    self::conflict($plan, 'IdP group ' . $directoryGroup->name . ' has no mailcow domain');
-                    continue;
-                }
-                if (isset($names[$domain][mb_strtolower($directoryGroup->name)])) {
-                    self::conflict($plan, 'A group ' . $directoryGroup->name . ' exists in ' . $domain . ' and is not linked to the IdP group');
-                    continue;
-                }
-                $names[$domain][mb_strtolower($directoryGroup->name)] = 0;
-                $plan->groupCreates[] = ['ref' => $ref, 'name' => $directoryGroup->name, 'domain' => $domain, 'mail' => $directoryGroup->mail];
-                $plan->memberships[$ref] = self::sorted($members);
-                continue;
-            }
-            $idpMembers = self::idpMembers($current);
-            // Linked users missing in the listing keep their memberships until they are listed again
-            foreach ($idpMembers as $username) {
-                if (isset($state->accounts[$username]) && !isset($matched[$username])) {
-                    $members[] = $username;
-                }
-            }
-            $members = self::sorted($members);
-            $removed = array_diff($idpMembers, $members);
-            if (!empty($removed) || !empty(array_diff($members, array_map('strval', array_keys($current['members']))))) {
-                $plan->memberships[$ref] = $members;
-                $plan->membershipRemovals += count($removed);
-            }
-            if ($directoryGroup->name !== $current['name']) {
-                $taken = $names[$current['domain']][mb_strtolower($directoryGroup->name)] ?? null;
-                if ($taken !== null && $taken !== (int)$current['id']) {
-                    self::conflict($plan, 'IdP group ' . $current['name'] . ' was renamed to ' . $directoryGroup->name . ', which exists in ' . $current['domain']);
-                } else {
-                    $plan->groupRenames[(int)$current['id']] = $directoryGroup->name;
-                    $names[$current['domain']][mb_strtolower($directoryGroup->name)] = (int)$current['id'];
-                }
-            }
-        }
-        foreach ($linked as $ref => $group) {
-            if (!isset($groups[$ref])) {
-                $plan->orphanedGroups[] = (int)$group['id'];
-                $plan->membershipRemovals += count(self::idpMembers($group));
-            }
-        }
-    }
-
-    /**
-     * Template of a new mailbox: the template of its highest-priority linked group (as in TemplateService), else
-     * the attribute mapping, else the default template.
-     *
-     * @param array<string, DirectoryUser> $candidates
-     * @param array<string, DirectoryGroup> $groups listed groups, [] without group sync
-     */
-    private function planCreates(Settings $settings, array $candidates, array $groups, State $state, SyncPlan $plan): void
-    {
-        $linked = [];
-        foreach ($state->groups as $group) {
-            if ($group['idp_ref'] !== null && $group['template'] !== null && $group['active'] === 1) {
-                $linked[(string)$group['idp_ref']] = $group;
-            }
-        }
-        $groupsOf = [];
-        foreach ($groups as $ref => $directoryGroup) {
-            if (isset($linked[(string)$ref])) {
-                foreach ($directoryGroup->memberIds as $id) {
-                    $groupsOf[$id][] = $linked[(string)$ref];
-                }
-            }
-        }
-        foreach ($candidates as $email => $user) {
-            $best = null;
-            foreach ($groupsOf[$user->id] ?? [] as $group) {
-                if ($best === null || $group['priority'] > $best['priority'] || ($group['priority'] === $best['priority'] && $group['id'] < $best['id'])) {
-                    $best = $group;
-                }
-            }
-            $template = $best['template'] ?? $settings->mappedTemplate($user->templateValues)
-                ?? (($settings->defaultTemplate !== '') ? $settings->defaultTemplate : null);
-            if ($template === null) {
-                self::conflict($plan, 'No template matches ' . $email);
-                continue;
-            }
-            $plan->creates[] = ['email' => (string)$email, 'name' => self::name($user->name, ''), 'template' => $template, 'id' => $user->id];
-        }
-    }
-
-    /** @param array<string, DirectoryUser> $matched */
-    private function planStates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
-    {
-        foreach ($state->accounts as $username => $account) {
-            $username = (string)$username;
-            $user = $matched[$username] ?? null;
-            if ($user === null && $account->missingSince === null) {
-                $plan->missing[] = $username;
-                continue;
-            }
-            if ($user !== null && $account->missingSince !== null) {
-                $plan->found[] = $username;
-            }
-            if (!$settings->offboarding) {
-                continue;
-            }
-            // Second complete run without the user counts as removed from the IdP
-            $disabled = ($user === null) || !$user->enabled;
-            if ($disabled && !$account->idpDisabled && $account->active === 1) {
-                $plan->deactivate[] = $username;
-            } elseif (!$disabled && $account->idpDisabled) {
-                if ($account->active === 2) {
-                    $plan->restore[] = $username;
-                } else {
-                    $plan->release[] = $username;
-                }
-            }
-        }
-    }
-
-    /**
-     * Name and template of linked mailboxes, like the IdP login does. The planned template is the legacy mapping;
-     * SyncService applies the effective template again after the group changes of the run.
-     *
-     * @param array<string, DirectoryUser> $matched
-     */
-    private function planUpdates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
-    {
-        foreach ($matched as $username => $user) {
-            $account = $state->accounts[$username];
-            $legacy = $settings->mappedTemplate($user->templateValues);
-            $plan->profiles[$username] = ['name' => self::name($user->name, $account->name), 'template' => $legacy];
-            $template = $state->effectiveTemplates[$username] ?? $legacy;
-            if ($template === null) {
-                continue;
-            }
-            if (!isset($state->templateHashes[$template])) {
-                self::conflict($plan, 'Template ' . $template . ' does not exist');
-                continue;
-            }
-            $name = self::name($user->name, $account->name);
-            if ($name !== $account->name || $state->templateHashes[$template] !== $account->attributeHash) {
-                $plan->updates[] = ['username' => (string)$username, 'name' => $name, 'template' => $legacy];
-            }
-        }
-    }
-
-    /** Domain for a new group: the domain of its mail if it is a mailcow domain, else the default group domain */
-    private static function groupDomain(Settings $settings, DirectoryGroup $group, array $domains): ?string
-    {
-        $domain = ($group->mail !== null) ? substr((string)strrchr($group->mail, '@'), 1) : '';
-        if (in_array($domain, $domains, true)) {
-            return $domain;
-        }
-        return in_array($settings->groupDefaultDomain, $domains, true) ? $settings->groupDefaultDomain : null;
-    }
-
-    /** Mailbox name as mailbox_from_template stores it; an empty IdP name keeps the current one */
-    private static function name(string $name, string $current): string
-    {
-        $name = ltrim(rtrim(trim($name), '>'), '<');
-        return ($name === '') ? $current : $name;
-    }
-
-    private static function limit(int $base, int $percent): int
-    {
-        return max(self::GUARD_MIN, (int)ceil($base * $percent / 100));
-    }
-
-    /** @return string[] members of a state group with source idp */
-    private static function idpMembers(array $group): array
-    {
-        return array_map('strval', array_keys(array_filter($group['members'], fn(string $source): bool => $source === 'idp')));
-    }
-
-    private static function sorted(array $usernames): array
-    {
-        $usernames = array_values(array_unique(array_map('strval', $usernames)));
-        sort($usernames);
-        return $usernames;
-    }
-
-    private static function conflict(SyncPlan $plan, string $message): void
-    {
-        if (!in_array($message, $plan->conflicts, true)) {
-            $plan->conflicts[] = $message;
-        }
-    }
-}

data/web/inc/lib/mailcow/Identity/Service/SyncService.php

@@ -1,231 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Identity\Service;
-
-use Closure;
-use Mailcow\Core\UserError;
-use Mailcow\Identity\Directory\Directory;
-use Mailcow\Identity\Directory\DirectoryException;
-use Mailcow\Identity\Legacy\LegacyAccounts;
-use Mailcow\Identity\Model\Settings;
-use Mailcow\Identity\Model\State;
-use Mailcow\Identity\Model\SyncPlan;
-use Mailcow\Identity\Repository\IdentityRepository;
-use Mailcow\Identity\Repository\RunRepository;
-use Mailcow\MailboxGroup\Service\IdpSyncService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use Mailcow\Sogo\Service\SogoService;
-use Throwable;
-
-/**
- * Sync runs of the IdP (cron only): list the directory, plan, apply. A listing error aborts the run before
- * anything is written; a blocked or preview run writes nothing but the run summary.
- * The directory is created only when a run is due, so idle cron calls never connect to the IdP.
- */
-class SyncService
-{
-    /** A run still `running` after this many minutes crashed */
-    private const STALE_MINUTES = 60;
-    /** Errors kept in the run summary */
-    private const DETAILS = 50;
-
-    public function __construct(
-        private Settings $settings,
-        /** @var Closure(): ?Directory */
-        private Closure $directory,
-        private IdentityRepository $repo,
-        private RunRepository $runs,
-        private SyncPlanner $planner,
-        private IdpSyncService $groups,
-        private TemplateService $templates,
-        private LegacyAccounts $legacy,
-        private SogoService $sogo
-    ) {
-    }
-
-    /**
-     * Runs the oldest requested run, else a scheduled run when the sync interval is over.
-     *
-     * @return ?array the finished run (id, mode, status, summary), null if nothing was due
-     */
-    public function runDue(): ?array
-    {
-        $this->runs->failStale(self::STALE_MINUTES);
-        $run = $this->runs->nextPending();
-        if ($run === null) {
-            if (!$this->settings->syncEnabled()) {
-                return null;
-            }
-            $since = $this->runs->secondsSinceScheduled();
-            if ($since !== null && $since < $this->settings->syncInterval * 60) {
-                return null;
-            }
-            $run = ['id' => $this->runs->create($this->settings->provider, 'run'), 'mode' => 'run'];
-        }
-        return $this->execute((int)$run['id'], (string)$run['mode']);
-    }
-
-    /** @return array{id: int, mode: string, status: string, summary: array} */
-    public function execute(int $id, string $mode): array
-    {
-        $this->runs->start($id);
-        try {
-            $directory = $this->settings->canSync() ? ($this->directory)() : null;
-            if ($directory === null) {
-                throw new DirectoryException('The identity provider cannot be synced');
-            }
-            $listing = $directory->listAll($this->settings->groupSync);
-            $plan = $this->planner->plan($this->settings, $listing, $this->state(), $mode === 'force');
-            $summary = $plan->summary();
-            if ($mode === 'preview') {
-                $status = 'ok';
-            } elseif ($plan->blocked) {
-                $status = 'blocked';
-            } else {
-                [$errors, $updated] = $this->apply($plan);
-                $summary['counts']['updates'] = $updated;
-                $summary['counts']['errors'] = count($errors);
-                $summary['errors'] = array_slice($errors, 0, self::DETAILS);
-                $status = 'ok';
-            }
-        } catch (DirectoryException $e) {
-            $status = 'error';
-            $summary = ['error' => $e->getMessage()];
-        } catch (Throwable $e) {
-            $this->runs->finish($id, 'error', ['error' => 'The sync run failed: ' . $e->getMessage(), 'mode' => $mode]);
-            throw $e;
-        }
-        $summary['mode'] = $mode;
-        $this->runs->finish($id, $status, $summary);
-        return ['id' => $id, 'mode' => $mode, 'status' => $status, 'summary' => $summary];
-    }
-
-    private function state(): State
-    {
-        $accounts = $this->repo->accounts($this->settings->provider);
-        return new State($accounts, $this->repo->mailboxes(), $this->repo->domains(),
-            $this->settings->groupSync ? $this->groups->groups() : [],
-            $this->settings->periodicSync ? $this->templates->effective(array_keys($accounts)) : [],
-            $this->repo->templateHashes());
-    }
-
-    /** @return array{0: string[], 1: int} errors of single changes (the rest is still applied), updated mailboxes */
-    private function apply(SyncPlan $plan): array
-    {
-        $provider = $this->settings->provider;
-        $errors = [];
-        foreach ($plan->links as $username => $externalId) {
-            $this->repo->link((string)$username, $provider, $externalId);
-        }
-        foreach ($plan->found as $username) {
-            $this->repo->setMissing($username, $provider, false);
-        }
-        foreach ($plan->missing as $username) {
-            $this->repo->setMissing($username, $provider, true);
-        }
-        foreach ($plan->creates as $create) {
-            $error = $this->legacy->createMailbox($create['email'], $create['name'], $provider, $create['template']);
-            if ($error !== null) {
-                $errors[] = $create['email'] . ': ' . $error;
-                continue;
-            }
-            $this->repo->link($create['email'], $provider, $create['id']);
-        }
-        foreach ($plan->deactivate as $username) {
-            $this->repo->setActive($username, $provider, 2, true);
-        }
-        foreach ($plan->restore as $username) {
-            $this->repo->setActive($username, $provider, 1, false);
-        }
-        foreach ($plan->release as $username) {
-            $this->repo->setIdpDisabled($username, $provider, false);
-        }
-        $this->sogo->refreshStaticView(array_merge($plan->deactivate, $plan->restore));
-        $changed = [];
-        $errors = array_merge($errors, $this->applyGroups($plan, $changed));
-        [$updateErrors, $updated] = $this->applyUpdates($plan, $changed);
-        return [array_merge($errors, $updateErrors), $updated];
-    }
-
-    /**
-     * @param string[] $changed collects the members that joined or left a group
-     * @return string[]
-     */
-    private function applyGroups(SyncPlan $plan, array &$changed): array
-    {
-        $errors = [];
-        foreach ($plan->groupCreates as $create) {
-            try {
-                foreach ($this->groups->createFromIdp($create['ref'], $create['name'], $create['domain'], $create['mail']) as $warning) {
-                    $errors[] = $create['name'] . ': ' . $warning[0];
-                }
-            } catch (UserError $e) {
-                $errors[] = $create['name'] . ': ' . $e->key;
-            }
-        }
-        foreach ($plan->groupRenames as $id => $name) {
-            try {
-                $this->groups->renameFromIdp((int)$id, $name);
-            } catch (UserError $e) {
-                $errors[] = $name . ': ' . $e->key;
-            }
-        }
-        foreach ($plan->memberships as $ref => $usernames) {
-            $group = $this->groups->findByIdpRef((string)$ref);
-            if ($group !== null) {
-                $this->groupResult($group->name, $this->groups->syncGroupMembers((int)$group->id, $usernames), $errors, $changed);
-            }
-        }
-        foreach ($plan->orphanedGroups as $id) {
-            $this->groupResult((string)$id, $this->groups->syncGroupMembers($id, []), $errors, $changed);
-        }
-        return $errors;
-    }
-
-    /**
-     * Name and template updates after the group changes: planned updates and members that joined or left a group
-     * in this run (e.g. the legacy mapping applies again after leaving a template group). The effective template
-     * is resolved and compared again.
-     *
-     * @return array{0: string[], 1: int} errors, updated mailboxes
-     */
-    private function applyUpdates(SyncPlan $plan, array $changed): array
-    {
-        $candidates = array_values(array_unique(array_merge(array_column($plan->updates, 'username'),
-            array_intersect($changed, array_keys($plan->profiles)))));
-        if (empty($candidates)) {
-            return [[], 0];
-        }
-        $errors = [];
-        $updated = 0;
-        $accounts = $this->repo->accounts($this->settings->provider);
-        $hashes = $this->repo->templateHashes();
-        $effective = $this->templates->effective($candidates);
-        foreach ($candidates as $username) {
-            $profile = $plan->profiles[$username] ?? null;
-            $account = $accounts[$username] ?? null;
-            $template = $effective[$username] ?? $profile['template'] ?? null;
-            if ($profile === null || $account === null || $template === null
-                || ($profile['name'] === $account->name && ($hashes[$template] ?? null) === $account->attributeHash)) {
-                continue;
-            }
-            $error = $this->legacy->applyTemplate($username, $profile['name'], $template);
-            if ($error !== null) {
-                $errors[] = $username . ': ' . $error;
-            } else {
-                $updated++;
-            }
-        }
-        return [$errors, $updated];
-    }
-
-    private function groupResult(string $group, array $result, array &$errors, array &$changed): void
-    {
-        foreach ($result['warnings'] as $warning) {
-            $errors[] = $group . ': ' . $warning[0];
-        }
-        array_push($changed, ...$result['added'], ...$result['removed']);
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Controller/ImapsyncApiController.php

@@ -1,267 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Controller;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\Imapsync\Model\Source;
-use Mailcow\Imapsync\Service\JobService;
-use Mailcow\Imapsync\Service\SourceService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php (and the pages that show the same data).
- *
- * Write methods take the decoded request ($items = ids, $attr = attributes), loop over batch ids, turn
- * UserError into danger messages and write all messages to $_SESSION['return'] via ApiResponse.
- * They return what json_api passes to process_*_return(): false when the whole call was rejected.
- * A database error stops the call with one `mysql_error` message, like the rest of mailcow.
- * Read methods return the API data (an empty array when there is nothing the caller may see).
- * Messages are logged as ['syncjob', action, type, data] like before the conversion.
- */
-class ImapsyncApiController
-{
-    public function __construct(private SourceService $sources, private JobService $jobs, private ApiResponse $response)
-    {
-    }
-
-    /** POST add/syncjob */
-    public function addJob(array $attr): bool
-    {
-        return $this->handle('add', 'job', $attr,
-            fn(array $log): bool => $this->response->success(['mailbox_modified', $this->jobs->create($attr)->user2], $log));
-    }
-
-    /** POST add/syncjob_source */
-    public function addSource(array $attr): bool
-    {
-        return $this->handle('add', 'source', $attr, function (array $log) use ($attr): bool {
-            $source = $this->sources->create($attr);
-            $this->tokenFailure($source, $log);
-            return $this->response->success(['imapsync_source_added', $source->name], $log);
-        });
-    }
-
-    /** POST edit/syncjob: one message per id; `id` in $attr wins over $items (array_merge like before) */
-    public function editJobs(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', 'job', $data, function (array $log) use ($data): bool {
-            $this->jobs->requireAcl();
-            return $this->batch((array)($data['id'] ?? []), $log,
-                fn(int $id): array => ['mailbox_modified', $this->jobs->update($id, $data)->user2]);
-        });
-    }
-
-    /** POST edit/syncjob/run_next (admin): false when no job was moved */
-    public function runNext(?array $items): bool
-    {
-        return $this->handle('edit', 'run_next', ['id' => $items], function (array $log) use ($items): bool {
-            $outcome = $this->jobs->runNext(array_map('intval', array_values(array_filter((array)$items, 'is_numeric'))));
-            foreach ($outcome['running'] as $id) {
-                $this->response->danger(['imapsync_job_running', $id], $log);
-            }
-            return ($outcome['moved'] > 0) ? $this->response->success(['imapsync_run_next_done', $outcome['moved']], $log) : false;
-        });
-    }
-
-    /** POST edit/syncjob_source: one message per id, non-numeric ids are skipped */
-    public function editSources(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', 'source', $data, function (array $log) use ($data): bool {
-            $this->sources->requireEditor();
-            return $this->batch(array_filter((array)($data['id'] ?? []), 'is_numeric'), $log, function (int $id) use ($data, $log): array {
-                $source = $this->sources->update($id, $data);
-                $this->tokenFailure($source, $log);
-                return ['imapsync_source_modified', $source->name];
-            });
-        });
-    }
-
-    /** POST edit/syncjob_source/refresh_token: the first id only */
-    public function refreshSourceToken(?array $items): bool
-    {
-        return $this->handle('edit', 'refresh_token', ['id' => $items], function (array $log) use ($items): bool {
-            $source = $this->sources->refreshToken($this->requireId(is_array($items) ? reset($items) : null));
-            return $this->response->success(['imapsync_source_token_refreshed', $source->name], $log);
-        });
-    }
-
-    /** POST edit/imapsync_settings (admin) */
-    public function editSettings(array $attr): bool
-    {
-        return $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
-            $this->jobs->saveSettings($attr);
-            return $this->response->success('max_parallel_saved', $log);
-        });
-    }
-
-    /** POST delete/syncjob: one message per id (with the id as requested) */
-    public function deleteJobs(array $items): bool
-    {
-        return $this->handle('delete', 'job', ['id' => $items], function (array $log) use ($items): bool {
-            $this->jobs->requireAcl();
-            return $this->batch($items, $log, function (int $id, mixed $requested): array {
-                $this->jobs->delete($id);
-                return ['deleted_syncjob', $requested];
-            });
-        });
-    }
-
-    /** POST delete/syncjob_source: one message per id, non-numeric ids are skipped */
-    public function deleteSources(array $items): bool
-    {
-        return $this->handle('delete', 'source', ['id' => $items], function (array $log) use ($items): bool {
-            $this->sources->requireAcl();
-            return $this->batch(array_filter($items, 'is_numeric'), $log,
-                fn(int $id): array => ['imapsync_source_deleted', $this->sources->delete($id)->name]);
-        });
-    }
-
-    /** GET syncjob_options */
-    public function getOptions(): array
-    {
-        return $this->jobs->customParamOptions();
-    }
-
-    /** GET imapsync_settings (admin), also admin/system.php */
-    public function getSettings(): array
-    {
-        try {
-            return $this->jobs->getSettings()->toArray();
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /** GET syncjob_source/all */
-    public function getSources(): array
-    {
-        return array_values(array_filter(array_map(fn(int $id): array => $this->getSource($id), $this->sources->visibleIds())));
-    }
-
-    /** GET syncjob_source/<id>, also edit.php */
-    public function getSource(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $source = ($id === null) ? null : $this->sources->get($id);
-        return ($source === null) ? [] : $this->sources->toArray($source);
-    }
-
-    /**
-     * GET syncjobs/all. Admins get every job (incl. orphaned ones). Others get the jobs of $mailboxes,
-     * the mailboxes the caller manages (json_api collects them with the legacy mailbox() functions
-     * until functions.mailbox.inc.php is converted), in queue order.
-     */
-    public function getAllJobs(?string $extra, array $mailboxes): array
-    {
-        if ($this->jobs->canListAll()) {
-            return $this->jobRows($this->jobs->allIds(), $extra);
-        }
-        $ids = [];
-        foreach ($mailboxes as $mailbox) {
-            $ids = array_merge($ids, $this->jobs->idsForMailbox((string)$mailbox));
-        }
-        $rows = $this->jobRows($ids, $extra);
-        // Queue order (C4): prio desc, never-run first, least recently run, id
-        usort($rows, fn(array $a, array $b): int => [$b['prio'], !empty($a['last_run']), (string)$a['last_run'], (int)$a['id']]
-            <=> [$a['prio'], !empty($b['last_run']), (string)$b['last_run'], (int)$b['id']]);
-        return $rows;
-    }
-
-    /** GET syncjobs/<mailbox>; without a valid mailbox the caller's own jobs */
-    public function getJobs(?string $mailbox, ?string $extra): array
-    {
-        return $this->jobRows($this->jobs->idsForMailbox($mailbox), $extra);
-    }
-
-    /** One job (edit.php, inc/ajax/syncjob_logs.php); [] when missing or not accessible. `no_log` in $extra skips the log. */
-    public function getJob(mixed $id, ?string $extra = null): array
-    {
-        $id = $this->toId($id);
-        $withLog = ($extra === null || !in_array('no_log', explode(',', $extra), true));
-        $job = ($id === null) ? null : $this->jobs->get($id, $withLog);
-        return ($job === null) ? [] : $job->toArray($withLog);
-    }
-
-    /**
-     * Account rename ($new) or delete ($new = null) from legacy account code that runs outside a transaction
-     * (admin, domain admin, mailbox delete): a database error becomes a `mysql_error` message.
-     * Inside a transaction (mailbox rename) call $module->sources->onAccountRenamed() directly instead,
-     * so the caller's catch can roll back.
-     */
-    public function accountChanged(string $old, ?string $new = null): void
-    {
-        try {
-            if ($new === null) {
-                $this->sources->onAccountDeleted($old);
-            } else {
-                $this->sources->onAccountRenamed($old, $new);
-            }
-        } catch (PDOException $e) {
-            $this->response->danger(['mysql_error', $e->getMessage()], ['imapsync_account_changed', $old, $new]);
-        }
-        $this->response->flush();
-    }
-
-    /** The token fetched when saving a client_credentials source failed: the source is saved, report the error */
-    private function tokenFailure(Source $source, array $log): void
-    {
-        if ($source->usesClientCredentials() && $source->active === 1 && $source->lastRefreshError !== null) {
-            $this->response->danger(['imapsync_source_token_refresh_failed', $source->name . ' (' . $source->lastRefreshError . ')'], $log);
-        }
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, string $type, mixed $data, callable $work): bool
-    {
-        $log = ['syncjob', $action, $type, $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** One message per id: the success message returned by $work(id, requested id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id), $id), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** @return array[] API rows of the jobs the caller may access */
-    private function jobRows(array $ids, ?string $extra): array
-    {
-        return array_values(array_filter(array_map(fn(int $id): array => $this->getJob($id, $extra), $ids)));
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else, also "1.5" (matches no row) */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Http/OAuthClient.php

@@ -1,172 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Http;
-
-/**
- * HTTPS requests to OAuth provider endpoints with an address policy against SSRF (C5).
- *
- * Every endpoint must be https://, without userinfo, with a valid hostname. In strict mode (endpoints of
- * sources not created by an admin) every resolved address must be public; curl is then pinned to the checked
- * addresses (CURLOPT_RESOLVE), so a DNS change between check and request does not help an attacker.
- *
- * resolve() and send() are the only I/O and protected, so tests can replace them.
- */
-class OAuthClient
-{
-    private const MAX_URL_LENGTH = 500;
-    private const MAX_ERROR_LENGTH = 200;
-
-    public function isAllowed(string $url, bool $strict): bool
-    {
-        return $this->pin($url, $strict) !== null;
-    }
-
-    /**
-     * Sends one request (POST form when $post is given, otherwise GET) and returns the decoded JSON object.
-     *
-     * @throws OAuthException with a sanitised message
-     */
-    public function request(string $url, bool $strict, ?array $post = null, array $headers = []): array
-    {
-        $pin = $this->pin($url, $strict);
-        if ($pin === null) {
-            throw new OAuthException('endpoint not allowed');
-        }
-        $options = [
-            CURLOPT_URL => $url,
-            CURLOPT_RETURNTRANSFER => true,
-            CURLOPT_FOLLOWLOCATION => false,
-            CURLOPT_PROXY => '',
-            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS,
-            CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS,
-            CURLOPT_CONNECTTIMEOUT => 10,
-            CURLOPT_TIMEOUT => 20,
-            CURLOPT_MAXFILESIZE => 1048576,
-            CURLOPT_SSL_VERIFYPEER => true,
-            CURLOPT_SSL_VERIFYHOST => 2,
-            CURLOPT_HTTPHEADER => array_merge(['Accept: application/json'], $headers),
-        ];
-        if (!empty($pin)) {
-            $options[CURLOPT_RESOLVE] = $pin;
-        }
-        if ($post !== null) {
-            $options[CURLOPT_POST] = true;
-            $options[CURLOPT_POSTFIELDS] = http_build_query($post);
-        }
-        [$body, $code] = $this->send($options);
-        if ($body === false) {
-            throw new OAuthException('connection failed');
-        }
-        $data = json_decode($body, true);
-        $ok = ($code >= 200 && $code < 300);
-        if ($ok && is_array($data)) {
-            return $data;
-        }
-        throw new OAuthException($this->errorText($code, $ok, $data));
-    }
-
-    /**
-     * IPv4 and IPv6 addresses of a host.
-     *
-     * @return string[]
-     */
-    protected function resolve(string $host): array
-    {
-        $ips = @gethostbynamel($host) ?: [];
-        foreach (@dns_get_record($host, DNS_AAAA) ?: [] as $record) {
-            $ips[] = $record['ipv6'] ?? '';
-        }
-        return $ips;
-    }
-
-    /**
-     * Runs curl with the given options.
-     *
-     * @return array{0: string|false, 1: int} response body (false on connection errors) and HTTP status
-     */
-    protected function send(array $options): array
-    {
-        $ch = curl_init();
-        curl_setopt_array($ch, $options);
-        $body = curl_exec($ch);
-        $code = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
-        curl_close($ch);
-        return [is_string($body) ? $body : false, $code];
-    }
-
-    /**
-     * Applies the endpoint policy. Returns null when the URL is not allowed, otherwise the CURLOPT_RESOLVE
-     * pin (empty when not strict).
-     */
-    private function pin(string $url, bool $strict): ?array
-    {
-        if (strlen($url) > self::MAX_URL_LENGTH || preg_match('/[\x00-\x20\x7f]/', $url)) {
-            return null;
-        }
-        $parts = parse_url($url);
-        if (!$parts || strtolower($parts['scheme'] ?? '') !== 'https' || isset($parts['user']) || isset($parts['pass'])) {
-            return null;
-        }
-        $host = strtolower($parts['host'] ?? '');
-        if (!filter_var($host, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME) || !filter_var($url, FILTER_VALIDATE_URL)) {
-            return null;
-        }
-        if (!$strict) {
-            return [];
-        }
-        $ips = array_values(array_unique($this->resolve($host)));
-        if (empty($ips)) {
-            return null;
-        }
-        $pinned = [];
-        foreach ($ips as $ip) {
-            if (!$this->isPublic($ip)) {
-                return null;
-            }
-            $pinned[] = filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? '[' . $ip . ']' : $ip;
-        }
-        return [$host . ':' . (int)($parts['port'] ?? 443) . ':' . implode(',', $pinned)];
-    }
-
-    /**
-     * Public unicast address only: no private, loopback, link-local or reserved ranges, no CGNAT (100.64/10),
-     * benchmarking (198.18/15) or multicast/reserved (224/3); IPv6 only global unicast 2000::/3 without
-     * 6to4 (2002::/16), Teredo (2001::/32) and documentation (2001:db8::/32).
-     */
-    private function isPublic(string $ip): bool
-    {
-        if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE)) {
-            return false;
-        }
-        if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
-            $long = ip2long($ip);
-            return ($long & 0xFFC00000) != 0x64400000
-                && ($long & 0xFFFE0000) != 0xC6120000
-                && ($long & 0xE0000000) != 0xE0000000;
-        }
-        $hex = bin2hex((string)inet_pton($ip));
-        return ($hex[0] === '2' || $hex[0] === '3')
-            && substr($hex, 0, 4) !== '2002'
-            && !in_array(substr($hex, 0, 8), ['20010000', '20010db8'], true);
-    }
-
-    /**
-     * Error text that is safe to store: "HTTP <code>" plus the provider's error / error_description.
-     */
-    private function errorText(int $code, bool $ok, mixed $data): string
-    {
-        $text = 'HTTP ' . $code . ($ok ? ': malformed response' : '');
-        $detail = [];
-        foreach (['error', 'error_description'] as $key) {
-            if (is_array($data) && isset($data[$key]) && is_scalar($data[$key]) && (string)$data[$key] !== '') {
-                $detail[] = (string)$data[$key];
-            }
-        }
-        if (!empty($detail)) {
-            $text .= ': ' . implode(' - ', $detail);
-        }
-        return mb_substr(trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', strip_tags($text))), 0, self::MAX_ERROR_LENGTH);
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Http/OAuthException.php

@@ -1,15 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Http;
-
-use RuntimeException;
-
-/**
- * A failed OAuth request. The message is safe to store and show: "HTTP <code>" plus the provider's
- * error fields, or a fixed text; never a raw response body or curl details.
- */
-class OAuthException extends RuntimeException
-{
-}

data/web/inc/lib/mailcow/Imapsync/ImapsyncModule.php

@@ -1,73 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Imapsync\Controller\ImapsyncApiController;
-use Mailcow\Imapsync\Http\OAuthClient;
-use Mailcow\Imapsync\Model\CustomParams;
-use Mailcow\Imapsync\Repository\JobRepository;
-use Mailcow\Imapsync\Repository\SettingsRepository;
-use Mailcow\Imapsync\Repository\SourceRepository;
-use Mailcow\Imapsync\Repository\TokenRepository;
-use Mailcow\Imapsync\Service\JobService;
-use Mailcow\Imapsync\Service\OAuthService;
-use Mailcow\Imapsync\Service\SourceService;
-use PDO;
-
-/**
- * Composition root of the sync job module: wires repositories, services and the API controller.
- *
- *   $imapsync = ImapsyncModule::create($pdo, Context::fromSession($_SESSION), [
- *     'allowlist' => CustomParams::allowlist($IMAPSYNC_OPTIONS, $IMAPSYNC_OPTIONS_DEFAULT),
- *     'hostname' => (string)getenv('MAILCOW_HOSTNAME'),
- *   ]);
- *   process_add_return($imapsync->api->addJob($attr));
- *
- * Procedural legacy code (json_api.php, pages, functions.*.inc.php) uses fromGlobals() instead.
- */
-class ImapsyncModule
-{
-    /** Request keys whose values never reach a log */
-    private const SECRET_KEYS = ['password1', 'password2', 'oauth_client_secret', 'oauth_extra_params'];
-
-    public function __construct(
-        public readonly SourceService $sources,
-        public readonly JobService $jobs,
-        public readonly OAuthService $oauth,
-        public readonly ImapsyncApiController $api
-    ) {
-    }
-
-    /**
-     * @param array $config 'allowlist' => effective custom param allowlist (CustomParams::allowlist()),
-     *                      'hostname'  => MAILCOW_HOSTNAME (OAuth redirect URI); both optional
-     */
-    public static function create(PDO $pdo, Context $context, array $config = []): self
-    {
-        $sourceRepository = new SourceRepository($pdo);
-        $tokenRepository = new TokenRepository($pdo);
-        $oauth = new OAuthService($sourceRepository, $tokenRepository, new OAuthClient(), (string)($config['hostname'] ?? ''));
-        $sources = new SourceService($sourceRepository, $tokenRepository, $oauth, $context);
-        $jobs = new JobService(new JobRepository($pdo), $sources, $tokenRepository, new SettingsRepository($pdo),
-            $context, (array)($config['allowlist'] ?? []));
-        return new self($sources, $jobs, $oauth, new ImapsyncApiController($sources, $jobs, new ApiResponse(self::SECRET_KEYS)));
-    }
-
-    /**
-     * Legacy entry point for procedural callers: the one place that reads the globals ($pdo, $_SESSION,
-     * $IMAPSYNC_OPTIONS / $IMAPSYNC_OPTIONS_DEFAULT from vars(.local).inc.php, MAILCOW_HOSTNAME)
-     * and passes them to create(). $context overrides the session caller (e.g. Context::system()).
-     */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), [
-            'allowlist' => CustomParams::allowlist((array)($GLOBALS['IMAPSYNC_OPTIONS'] ?? []),
-                (array)($GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? [])),
-            'hostname' => (string)getenv('MAILCOW_HOSTNAME'),
-        ]);
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Model/CustomParams.php

@@ -1,97 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Model;
-
-use Mailcow\Core\UserError;
-
-/**
- * Rules for the extra imapsync options of a sync job: a list of {o: option, v: value} pairs, stored as JSON
- * (`imapsync`.`custom_params`, read by the runner). Stored values are passed through unchanged.
- *
- * Allowlist rules (option name lowercase, without dashes):
- *   true   option requires a non-empty value
- *   false  flag, must not have a value
- *   string PCRE (no delimiters) the whole non-empty value must match
- */
-class CustomParams
-{
-    private const MAX_PAIRS = 100;
-    private const MAX_VALUE_LENGTH = 1000;
-
-    private static bool $warnedOldFormat = false;
-
-    /**
-     * Validates request input (list of {o, v}) against the allowlist and returns the JSON to store.
-     * Empty input means no options.
-     *
-     * @throws UserError imapsync_custom_params_format, imapsync_custom_param_invalid, imapsync_custom_param_value
-     */
-    public static function toJson(mixed $input, array $allowlist): string
-    {
-        $input = ($input === null || $input === '') ? [] : $input;
-        if (!is_array($input)) {
-            throw new UserError('imapsync_custom_params_format');
-        }
-        $pairs = [];
-        foreach ($input as $pair) {
-            $option = is_array($pair) ? ($pair['o'] ?? '') : null;
-            $value = is_array($pair) ? ($pair['v'] ?? '') : null;
-            if (!is_scalar($option) || !is_scalar($value)) {
-                throw new UserError('imapsync_custom_params_format');
-            }
-            $option = ltrim(strtolower(trim((string)$option)), '-');
-            $value = str_replace("\0", '', (string)$value);
-            if ($option === '' && trim($value) === '') {
-                continue; // empty UI row
-            }
-            $value = self::checkValue($option, $value, $allowlist);
-            if (strlen($value) > self::MAX_VALUE_LENGTH || count($pairs) >= self::MAX_PAIRS) {
-                throw new UserError('imapsync_custom_params_format');
-            }
-            $pairs[] = ['o' => $option, 'v' => $value];
-        }
-        return json_encode($pairs, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-    }
-
-    /**
-     * The effective allowlist: $IMAPSYNC_OPTIONS (after vars.local.inc.php) without the old
-     * 'whitelist' key and malformed entries; the defaults when nothing valid is left.
-     */
-    public static function allowlist(array $configured, array $defaults): array
-    {
-        $allowlist = [];
-        foreach ($configured as $option => $rule) {
-            $option = strtolower(ltrim(trim((string)$option), '-'));
-            if ($option !== 'whitelist' && preg_match('/^[a-z][a-z0-9]*$/', $option)
-                && (is_bool($rule) || (is_string($rule) && $rule !== ''))) {
-                $allowlist[$option] = $rule;
-            }
-        }
-        if (!self::$warnedOldFormat && isset($configured['whitelist'])) {
-            self::$warnedOldFormat = true;
-            error_log("mailcow: \$IMAPSYNC_OPTIONS['whitelist'] (old format) is ignored, use 'option' => true|false|'regex' (see data/web/inc/vars.inc.php)");
-        }
-        return empty($allowlist) ? $defaults : $allowlist;
-    }
-
-    /** The value to store for one option; throws when the allowlist rule is violated */
-    private static function checkValue(string $option, string $value, array $allowlist): string
-    {
-        $rule = $allowlist[$option] ?? null;
-        if ($option === '' || !(is_bool($rule) || is_string($rule))) {
-            throw new UserError('imapsync_custom_param_invalid', [htmlspecialchars($option)]);
-        }
-        $empty = (trim($value) === '');
-        $valid = match (true) {
-            $rule === false => $empty,
-            $rule === true => !$empty,
-            default => !$empty && preg_match('~^(?:' . str_replace('~', '\~', $rule) . ')$~D', $value) === 1,
-        };
-        if (!$valid) {
-            throw new UserError('imapsync_custom_param_value', [$option]);
-        }
-        return ($rule === false) ? '' : $value;
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Model/Job.php

@@ -1,154 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Model;
-
-/**
- * A sync job (table `imapsync`): copies the remote account user1 of a source into the local mailbox user2.
- *
- * Loaded jobs also carry display data of their source and of the per-user token ($source / $token,
- * filled from the JOIN; no secrets). Switches default to off, because unchecked form checkboxes are not sent.
- */
-class Job
-{
-    public ?int $id = null;
-    public string $user2 = '';
-    public ?string $regextrans2 = '';
-    public int $authmd51 = 0;
-    public string $domain2 = '';
-    public string $subfolder2 = '';
-    public string $user1 = '';
-    /** Remote password, never returned by toArray() */
-    public string $password1 = '';
-    public string $exclude = '';
-    public int $maxage = 0;
-    public int $minsInterval = 0;
-    /** Bytes/s limit, stored as text */
-    public string $maxBytesPerSecond = '0';
-    public int $delete2duplicates = 0;
-    public int $delete1 = 0;
-    public int $delete2 = 0;
-    public int $automap = 0;
-    public int $skipcrossduplicates = 0;
-    /** JSON list of {o, v} pairs (see CustomParams), old rows may hold other text */
-    public ?string $customParams = '[]';
-    public ?string $customParamsLegacy = null;
-    public int $timeout1 = 600;
-    public int $timeout2 = 600;
-    public int $subscribeall = 0;
-    public int $dry = 0;
-    public int $isRunning = 0;
-    public ?string $returnedText = null;
-    public ?string $lastRun = null;
-    public ?int $success = null;
-    public ?string $exitStatus = null;
-    public ?string $created = null;
-    public ?string $modified = null;
-    public int $active = 0;
-    public int $sourceId = 0;
-    public int $prio = 0;
-    public ?Source $source = null;
-    public ?OAuthToken $token = null;
-
-    /**
-     * Builds a job from an `imapsync` row, optionally joined with `source_<column>` (columns of the source)
-     * and `user_token_*` columns.
-     */
-    public static function fromRow(array $row): self
-    {
-        $job = new self();
-        $job->id = isset($row['id']) ? (int)$row['id'] : null;
-        foreach (['authmd51', 'maxage', 'delete2duplicates', 'delete1', 'delete2', 'automap', 'skipcrossduplicates', 'timeout1',
-            'timeout2', 'subscribeall', 'dry', 'active', 'prio'] as $column) {
-            $job->$column = (int)($row[$column] ?? 0);
-        }
-        foreach (['user2', 'domain2', 'subfolder2', 'user1', 'password1', 'exclude'] as $column) {
-            $job->$column = (string)($row[$column] ?? '');
-        }
-        $job->minsInterval = (int)($row['mins_interval'] ?? 0);
-        $job->maxBytesPerSecond = (string)($row['maxbytespersecond'] ?? '0');
-        $job->isRunning = (int)($row['is_running'] ?? 0);
-        $job->success = isset($row['success']) ? (int)$row['success'] : null;
-        // Nullable text columns (column => property)
-        foreach (['regextrans2' => 'regextrans2', 'custom_params' => 'customParams', 'custom_params_legacy' => 'customParamsLegacy',
-            'returned_text' => 'returnedText', 'last_run' => 'lastRun', 'exit_status' => 'exitStatus', 'created' => 'created',
-            'modified' => 'modified'] as $column => $property) {
-            $job->$property = isset($row[$column]) ? (string)$row[$column] : null;
-        }
-        $job->sourceId = (int)($row['source_id'] ?? 0);
-        if (isset($row['source_name'])) {
-            // source_id is the job's foreign key, i.e. the id of the source
-            $source = [];
-            foreach ($row as $key => $value) {
-                if (str_starts_with($key, 'source_')) {
-                    $source[substr($key, 7)] = $value;
-                }
-            }
-            $job->source = Source::fromRow($source);
-        }
-        if (!empty($row['user_token_exists'])) {
-            $job->token = OAuthToken::fromRow(['source_id' => $job->sourceId, 'owner' => $job->user2, 'username' => $job->user1,
-                'token_expires' => $row['user_token_expires'] ?? null, 'last_refresh_error' => $row['user_token_error'] ?? null]);
-        }
-        return $job;
-    }
-
-    /**
-     * API representation (GET syncjobs, edit page, log view). Never contains password1.
-     * Without $withLog the `log` field is empty (UI tables request `no_log`).
-     */
-    public function toArray(bool $withLog = true): array
-    {
-        $source = $this->source;
-        return [
-            'id' => $this->id,
-            'user2' => $this->user2,
-            'regextrans2' => $this->regextrans2,
-            'authmd51' => $this->authmd51,
-            'domain2' => $this->domain2,
-            'subfolder2' => $this->subfolder2,
-            'user1' => $this->user1,
-            'exclude' => $this->exclude,
-            'maxage' => $this->maxage,
-            'mins_interval' => $this->minsInterval,
-            'maxbytespersecond' => $this->maxBytesPerSecond,
-            'delete2duplicates' => $this->delete2duplicates,
-            'delete1' => $this->delete1,
-            'delete2' => $this->delete2,
-            'automap' => $this->automap,
-            'skipcrossduplicates' => $this->skipcrossduplicates,
-            'custom_params' => $this->customParams,
-            'timeout1' => $this->timeout1,
-            'timeout2' => $this->timeout2,
-            'subscribeall' => $this->subscribeall,
-            'dry' => $this->dry,
-            'is_running' => $this->isRunning,
-            'last_run' => $this->lastRun,
-            'success' => $this->success,
-            'exit_status' => $this->exitStatus,
-            'created' => $this->created,
-            'modified' => $this->modified,
-            'active' => $this->active,
-            'source_id' => $this->sourceId,
-            'custom_params_legacy' => $this->customParamsLegacy,
-            'prio' => $this->prio,
-            'source_name' => $source?->name,
-            'source_description' => $source?->description,
-            'source_host' => $source?->host,
-            'source_port' => $source?->port,
-            'source_enc' => $source?->enc,
-            'source_auth_type' => $source?->authType,
-            'source_created_by' => $source?->createdBy,
-            'source_scope' => $source?->scope,
-            'source_oauth_flow' => $source?->oauthFlow,
-            'source_active' => $source?->active,
-            'source_oauth_token_expires' => $source?->tokenExpires,
-            'source_oauth_last_refresh_error' => $source?->lastRefreshError,
-            'user_token_expires' => $this->token?->tokenExpires,
-            'user_token_error' => $this->token?->lastRefreshError,
-            'user_token_exists' => $this->token === null ? 0 : 1,
-            'log' => ($withLog && !empty($this->returnedText)) ? $this->returnedText : '',
-        ];
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Model/OAuthToken.php

@@ -1,38 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Model;
-
-/**
- * Per-user OAuth token of an authorization_code source (table `imapsync_source_oauth_token`).
- *
- * Bound to (source, owner, username): owner = local mailcow mailbox (job user2),
- * username = authenticated remote identity (job user1).
- */
-class OAuthToken
-{
-    public int $sourceId = 0;
-    public string $owner = '';
-    public string $username = '';
-    public ?string $accessToken = null;
-    public ?string $refreshToken = null;
-    public ?int $tokenExpires = null;
-    public ?string $lastRefreshError = null;
-    /** Seconds since the last refresh attempt by the DB clock; only filled by the refresh queries */
-    public ?int $refreshAttemptAge = null;
-
-    public static function fromRow(array $row): self
-    {
-        $token = new self();
-        $token->sourceId = (int)($row['source_id'] ?? 0);
-        $token->owner = (string)($row['owner'] ?? '');
-        $token->username = (string)($row['username'] ?? '');
-        $token->accessToken = isset($row['access_token']) ? (string)$row['access_token'] : null;
-        $token->refreshToken = isset($row['refresh_token']) ? (string)$row['refresh_token'] : null;
-        $token->tokenExpires = isset($row['token_expires']) ? (int)$row['token_expires'] : null;
-        $token->lastRefreshError = isset($row['last_refresh_error']) ? (string)$row['last_refresh_error'] : null;
-        $token->refreshAttemptAge = isset($row['refresh_attempt_age']) ? (int)$row['refresh_attempt_age'] : null;
-        return $token;
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Model/Settings.php

@@ -1,27 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Model;
-
-/**
- * Global sync job settings (table `imapsync_settings`), read by the runner.
- */
-class Settings
-{
-    public int $maxParallel = 1;
-    /** Global bandwidth limit in bytes/s, 0 = unlimited */
-    public int $maxBytesPerSecond = 0;
-
-    /**
-     * API representation (GET imapsync_settings); the UI edits the limit in KB/s.
-     */
-    public function toArray(): array
-    {
-        return [
-            'max_parallel' => $this->maxParallel,
-            'max_bytes_per_second' => $this->maxBytesPerSecond,
-            'max_kb_per_second' => intdiv($this->maxBytesPerSecond, 1024),
-        ];
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Model/Source.php

@@ -1,143 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Model;
-
-/**
- * A reusable IMAP source (table `imapsync_source` plus its scope targets).
- *
- * Visibility is driven by `scope` and the target lists:
- *   'all'    visible to everyone (admin-created only)
- *   'domain' visible to the listed domains (their domain admins and users)
- *   'user'   visible to the listed users (and their domain admins); no targets = private
- * `createdBy` records the creator ('' = handed over to the admins).
- *
- * The object holds secrets (client secret, access token); toArray() never returns them.
- */
-class Source
-{
-    public const AUTH_XOAUTH2 = 'XOAUTH2';
-    public const FLOW_CLIENT_CREDENTIALS = 'client_credentials';
-    public const FLOW_AUTHORIZATION_CODE = 'authorization_code';
-
-    public ?int $id = null;
-    public string $name = '';
-    public string $description = '';
-    public string $createdBy = '';
-    public string $scope = 'user';
-    public string $host = '';
-    public int $port = 0;
-    public ?string $enc = 'TLS';
-    public ?string $authType = 'PLAIN';
-    public string $oauthFlow = self::FLOW_CLIENT_CREDENTIALS;
-    public ?string $tokenEndpoint = null;
-    public ?string $authorizeEndpoint = null;
-    public ?string $userinfoEndpoint = null;
-    public ?string $clientId = null;
-    public ?string $clientSecret = null;
-    public ?string $oauthScope = null;
-    /** Extra OAuth parameters (sign-in or token request, see OAuthService), JSON object text */
-    public ?string $extraParams = null;
-    /** Shared token of a client_credentials source */
-    public ?string $accessToken = null;
-    public ?int $tokenExpires = null;
-    public ?string $lastRefreshError = null;
-    public ?string $lastRefreshAttempt = null;
-    /** Seconds since lastRefreshAttempt by the DB clock; only filled by the refresh queries */
-    public ?int $refreshAttemptAge = null;
-    public ?string $created = null;
-    public ?string $modified = null;
-    public int $active = 1;
-    /** @var string[] */
-    public array $domains = [];
-    /** @var string[] */
-    public array $users = [];
-
-    /** Builds a source from an `imapsync_source` row (column names as in the table) */
-    public static function fromRow(array $row): self
-    {
-        $source = new self();
-        $source->id = isset($row['id']) ? (int)$row['id'] : null;
-        $source->name = (string)($row['name'] ?? '');
-        $source->description = (string)($row['description'] ?? '');
-        $source->createdBy = (string)($row['created_by'] ?? '');
-        $source->scope = (string)($row['scope'] ?? 'user');
-        $source->host = (string)($row['host1'] ?? '');
-        $source->port = (int)($row['port1'] ?? 0);
-        $source->oauthFlow = (string)($row['oauth_flow'] ?? self::FLOW_CLIENT_CREDENTIALS);
-        $source->tokenExpires = isset($row['oauth_token_expires']) ? (int)$row['oauth_token_expires'] : null;
-        $source->refreshAttemptAge = isset($row['refresh_attempt_age']) ? (int)$row['refresh_attempt_age'] : null;
-        // Nullable text columns (column => property)
-        foreach (['enc1' => 'enc', 'auth_type' => 'authType', 'oauth_token_endpoint' => 'tokenEndpoint',
-            'oauth_authorize_endpoint' => 'authorizeEndpoint', 'oauth_userinfo_endpoint' => 'userinfoEndpoint',
-            'oauth_client_id' => 'clientId', 'oauth_client_secret' => 'clientSecret', 'oauth_scope' => 'oauthScope',
-            'oauth_extra_params' => 'extraParams', 'oauth_access_token' => 'accessToken', 'oauth_last_refresh_error' => 'lastRefreshError',
-            'oauth_last_refresh_attempt' => 'lastRefreshAttempt', 'created' => 'created', 'modified' => 'modified'] as $column => $property) {
-            $source->$property = isset($row[$column]) ? (string)$row[$column] : null;
-        }
-        $source->active = (int)($row['active'] ?? 1);
-        return $source;
-    }
-
-    public function isXOAuth2(): bool
-    {
-        return $this->authType === self::AUTH_XOAUTH2;
-    }
-
-    /** XOAUTH2 with one app-wide token that reaches every mailbox of the tenant */
-    public function usesClientCredentials(): bool
-    {
-        return $this->isXOAuth2() && $this->oauthFlow === self::FLOW_CLIENT_CREDENTIALS;
-    }
-
-    /** XOAUTH2 with a per-user token from the browser sign-in */
-    public function usesAuthorizationCode(): bool
-    {
-        return $this->isXOAuth2() && $this->oauthFlow === self::FLOW_AUTHORIZATION_CODE;
-    }
-
-    /**
-     * API representation (GET syncjob_source). Secrets and tokens are replaced by has_* flags;
-     * extra params are only shown to callers who may edit the source.
-     */
-    public function toArray(bool $canEdit, string $redirectUri): array
-    {
-        $data = [
-            'id' => $this->id,
-            'name' => $this->name,
-            'description' => $this->description,
-            'created_by' => $this->createdBy,
-            'scope' => $this->scope,
-            'host1' => $this->host,
-            'port1' => $this->port,
-            'enc1' => $this->enc,
-            'auth_type' => $this->authType,
-            'oauth_flow' => $this->oauthFlow,
-            'oauth_token_endpoint' => $this->tokenEndpoint,
-            'oauth_authorize_endpoint' => $this->authorizeEndpoint,
-            'oauth_userinfo_endpoint' => $this->userinfoEndpoint,
-            'oauth_client_id' => $this->clientId,
-            'oauth_scope' => $this->oauthScope,
-            'oauth_extra_params' => $canEdit ? $this->extraParams : null,
-            'oauth_token_expires' => $this->tokenExpires,
-            'oauth_last_refresh_error' => $this->lastRefreshError,
-            'oauth_last_refresh_attempt' => $this->lastRefreshAttempt,
-            'created' => $this->created,
-            'modified' => $this->modified,
-            'active' => $this->active,
-            'domains' => $this->domains,
-            'users' => $this->users,
-            'can_edit' => $canEdit,
-            'host' => $this->host,
-            'port' => $this->port,
-            'enc' => $this->enc,
-            'has_client_secret' => !empty($this->clientSecret),
-            'has_access_token' => !empty($this->accessToken),
-        ];
-        if ($this->usesAuthorizationCode()) {
-            $data['oauth_redirect_uri'] = $redirectUri;
-        }
-        return $data;
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Repository/JobRepository.php

@@ -1,128 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\Imapsync\Model\Job;
-use PDO;
-
-/**
- * SQL for sync jobs (table `imapsync`).
- */
-class JobRepository extends Repository
-{
-    /** Job queue order (C4): prio desc, never-run first, then least recently run */
-    private const QUEUE_ORDER = "ORDER BY `prio` DESC, `last_run` IS NOT NULL, `last_run` ASC, `id` ASC";
-
-    /** Every column except returned_text (the log, loaded on demand) */
-    private const COLUMNS = ['id', 'user2', 'regextrans2', 'authmd51', 'domain2', 'subfolder2', 'user1', 'password1',
-        'exclude', 'maxage', 'mins_interval', 'maxbytespersecond', 'delete2duplicates', 'delete1', 'delete2', 'automap',
-        'skipcrossduplicates', 'custom_params', 'timeout1', 'timeout2', 'subscribeall', 'dry', 'is_running',
-        'last_run', 'success', 'exit_status', 'created', 'modified', 'active', 'source_id', 'custom_params_legacy', 'prio'];
-
-    /**
-     * The job with display data of its source and per-user token (C1: source, owner = user2, username = user1).
-     * $withLog also loads returned_text.
-     */
-    public function find(int $id, bool $withLog = true): ?Job
-    {
-        $columns = $withLog ? [...self::COLUMNS, 'returned_text'] : self::COLUMNS;
-        $row = $this->query("SELECT i.`" . implode("`, i.`", $columns) . "`,
-              s.`name` AS source_name, s.`description` AS source_description, s.`host1` AS source_host1,
-              s.`port1` AS source_port1, s.`enc1` AS source_enc1, s.`auth_type` AS source_auth_type,
-              s.`created_by` AS source_created_by, s.`scope` AS source_scope, s.`oauth_flow` AS source_oauth_flow,
-              s.`active` AS source_active, s.`oauth_token_expires` AS source_oauth_token_expires,
-              s.`oauth_last_refresh_error` AS source_oauth_last_refresh_error,
-              t.`token_expires` AS user_token_expires, t.`last_refresh_error` AS user_token_error,
-              CASE WHEN t.`username` IS NULL THEN 0 ELSE 1 END AS user_token_exists
-            FROM `imapsync` i
-            LEFT JOIN `imapsync_source` s ON i.`source_id` = s.`id`
-            LEFT JOIN `imapsync_source_oauth_token` t
-              ON t.`source_id` = i.`source_id` AND t.`owner` = i.`user2` AND t.`username` = i.`user1`
-            WHERE i.`id` = :id", ['id' => $id])->fetch(PDO::FETCH_ASSOC);
-        return $row ? Job::fromRow($row) : null;
-    }
-
-    /** @return int[] job ids of a local mailbox in queue order */
-    public function idsForMailbox(string $user2): array
-    {
-        return array_map('intval', $this->query("SELECT `id` FROM `imapsync` WHERE `user2` = :user2 " . self::QUEUE_ORDER,
-            ['user2' => $user2])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** @return int[] all job ids (incl. orphaned ones) in queue order */
-    public function allIds(): array
-    {
-        return array_map('intval', $this->query("SELECT `id` FROM `imapsync` " . self::QUEUE_ORDER)->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** True when another job (not $exceptId) already syncs (source, user1) into user2 */
-    public function tupleExists(int $sourceId, string $user2, string $user1, ?int $exceptId = null): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `imapsync`
-            WHERE `user2` = :user2 AND `user1` = :user1 AND `source_id` = :source_id AND `id` != :id",
-            ['user2' => $user2, 'user1' => $user1, 'source_id' => $sourceId, 'id' => $exceptId ?? 0])->fetchColumn();
-    }
-
-    public function insert(Job $job): int
-    {
-        return $job->id = $this->insertRow('imapsync', $this->values($job) + ['user2' => $job->user2]);
-    }
-
-    public function update(Job $job): void
-    {
-        $this->updateRow('imapsync', (int)$job->id,
-            $this->values($job) + ['last_run' => $job->lastRun, 'success' => $job->success, 'prio' => $job->prio]);
-    }
-
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `imapsync` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /**
-     * Moves idle jobs to the front of the queue: each gets a prio above the current global max
-     * (lowest current prio first) and last_run/success are cleared. The rows are locked first, so the
-     * runner cannot claim a job in between (its claim UPDATE waits for this transaction).
-     *
-     * @param int[] $ids
-     * @return array{running: int[], moved: int} ids that are running (left alone) and the number of moved jobs
-     */
-    public function moveToFront(array $ids): array
-    {
-        return $this->transaction(function () use ($ids): array {
-            $rows = $this->query("SELECT `id`, `is_running` FROM `imapsync` WHERE `id` IN (" . self::placeholders($ids) . ") FOR UPDATE",
-                $ids)->fetchAll(PDO::FETCH_KEY_PAIR);
-            $running = array_keys(array_filter($rows, fn($isRunning): bool => (int)$isRunning === 1));
-            $idle = array_values(array_diff(array_keys($rows), $running));
-            $moved = 0;
-            if (!empty($idle)) {
-                $this->pdo->exec("SET @m := (SELECT COALESCE(MAX(`prio`), 0) FROM `imapsync`)");
-                $this->pdo->exec("SET @r := 0");
-                $moved = $this->query("UPDATE `imapsync` SET `prio` = @m + (@r := @r + 1), `last_run` = NULL, `success` = NULL
-                  WHERE `id` IN (" . self::placeholders($idle) . ") AND `is_running` = 0 ORDER BY `prio` ASC, `id` ASC", $idle)->rowCount();
-            }
-            return ['running' => $running, 'moved' => $moved];
-        });
-    }
-
-    /** Values of the columns a user can edit (shared by insert and update) */
-    private function values(Job $job): array
-    {
-        $values = ['mins_interval' => $job->minsInterval, 'maxbytespersecond' => $job->maxBytesPerSecond,
-            'custom_params' => $job->customParams, 'source_id' => $job->sourceId];
-        // Columns with a property of the same name
-        foreach (['user1', 'password1', 'subfolder2', 'exclude', 'maxage', 'timeout1', 'timeout2', 'active', 'delete1', 'delete2',
-            'delete2duplicates', 'automap', 'skipcrossduplicates', 'subscribeall', 'dry'] as $column) {
-            $values[$column] = $job->$column;
-        }
-        return $values;
-    }
-
-    private static function placeholders(array $values): string
-    {
-        return implode(',', array_fill(0, count($values), '?'));
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Repository/SettingsRepository.php

@@ -1,32 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\Imapsync\Model\Settings;
-use PDO;
-
-/**
- * SQL for the global sync job settings (table `imapsync_settings`, name => value).
- */
-class SettingsRepository extends Repository
-{
-    /** Stored settings; keys that are not stored yet keep the defaults of the model */
-    public function load(): Settings
-    {
-        $rows = $this->query("SELECT `name`, `value` FROM `imapsync_settings`")->fetchAll(PDO::FETCH_KEY_PAIR);
-        $settings = new Settings();
-        $settings->maxParallel = (int)($rows['max_parallel'] ?? $settings->maxParallel);
-        $settings->maxBytesPerSecond = (int)($rows['max_bytes_per_second'] ?? $settings->maxBytesPerSecond);
-        return $settings;
-    }
-
-    public function save(Settings $settings): void
-    {
-        $sql = "INSERT INTO `imapsync_settings` (`name`, `value`) VALUES (:name, :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)";
-        $this->query($sql, ['name' => 'max_parallel', 'value' => (string)$settings->maxParallel]);
-        $this->query($sql, ['name' => 'max_bytes_per_second', 'value' => (string)$settings->maxBytesPerSecond]);
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Repository/SourceRepository.php

@@ -1,215 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Repository;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\Imapsync\Model\Source;
-use PDO;
-
-/**
- * SQL for `imapsync_source` and its target tables `imapsync_source_domain` / `imapsync_source_user`.
- * No permission decisions here except the visibility filter, which is pure SQL (see visibility()).
- */
-class SourceRepository extends Repository
-{
-    /** Full source incl. secrets and targets, without visibility filter */
-    public function find(int $id): ?Source
-    {
-        return $this->fetch('1=1', ['id' => $id]);
-    }
-
-    /** Like find(), but only when the source is visible to the caller */
-    public function findVisible(int $id, Context $context): ?Source
-    {
-        [$visible, $params] = $this->visibility($context);
-        return $this->fetch($visible, $params + ['id' => $id]);
-    }
-
-    /** @return int[] ids of all sources visible to the caller, ordered by name */
-    public function visibleIds(Context $context): array
-    {
-        [$visible, $params] = $this->visibility($context);
-        return array_map('intval', $this->query("SELECT s.`id` FROM `imapsync_source` s WHERE $visible ORDER BY s.`name`, s.`id`",
-            $params)->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    public function nameExists(string $name, string $createdBy, ?int $exceptId = null): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `imapsync_source` WHERE `name` = :name AND `created_by` = :created_by AND `id` != :id",
-            ['name' => $name, 'created_by' => $createdBy, 'id' => $exceptId ?? 0])->fetchColumn();
-    }
-
-    /** True while a sync job references the source */
-    public function isInUse(int $id): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `imapsync` WHERE `source_id` = :id LIMIT 1", ['id' => $id])->fetchColumn();
-    }
-
-    public function isSuperAdmin(string $username): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `admin` WHERE `username` = :username AND `superadmin` = '1'",
-            ['username' => $username])->fetchColumn();
-    }
-
-    /** Inserts the source with its targets and returns the new id */
-    public function insert(Source $source): int
-    {
-        return $this->transaction(function () use ($source): int {
-            $source->id = $this->insertRow('imapsync_source',
-                $this->values($source) + ['created_by' => $source->createdBy, 'scope' => $source->scope]);
-            $this->writeTargets($source);
-            return $source->id;
-        });
-    }
-
-    /**
-     * Updates the source. $writeTargets also rewrites scope + target lists, $resetTokens drops the shared
-     * token and all per-user tokens of the source. All in one transaction.
-     */
-    public function update(Source $source, bool $writeTargets, bool $resetTokens): void
-    {
-        $this->transaction(function () use ($source, $writeTargets, $resetTokens): void {
-            $id = ['id' => $source->id];
-            $this->updateRow('imapsync_source', (int)$source->id, $this->values($source));
-            if ($writeTargets) {
-                $this->query("DELETE FROM `imapsync_source_domain` WHERE `source_id` = :id", $id);
-                $this->query("DELETE FROM `imapsync_source_user` WHERE `source_id` = :id", $id);
-                $this->writeTargets($source);
-            }
-            if ($resetTokens) {
-                $this->query("UPDATE `imapsync_source` SET `oauth_access_token` = NULL, `oauth_token_expires` = NULL,
-                  `oauth_last_refresh_error` = NULL, `oauth_last_refresh_attempt` = NULL WHERE `id` = :id", $id);
-                $this->query("DELETE FROM `imapsync_source_oauth_token` WHERE `source_id` = :id", $id);
-            }
-        });
-    }
-
-    /** Deletes the source; targets and per-user tokens go via FK ON DELETE CASCADE */
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `imapsync_source` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /** Deletes the source and its child rows explicitly, for callers running with FOREIGN_KEY_CHECKS = 0 (mailbox rename) */
-    public function deleteWithChildren(int $id): void
-    {
-        foreach (['imapsync_source_oauth_token', 'imapsync_source_user', 'imapsync_source_domain'] as $table) {
-            $this->query("DELETE FROM `$table` WHERE `source_id` = :id", ['id' => $id]);
-        }
-        $this->delete($id);
-    }
-
-    /** @return Source[] sources created by an account (id and name only) */
-    public function listCreatedBy(string $username): array
-    {
-        return array_map([Source::class, 'fromRow'], $this->query("SELECT `id`, `name` FROM `imapsync_source` WHERE `created_by` = :username",
-            ['username' => $username])->fetchAll(PDO::FETCH_ASSOC));
-    }
-
-    /** Hands a source over to the admins (created_by '') under the given name */
-    public function handOver(int $id, string $name): void
-    {
-        $this->query("UPDATE `imapsync_source` SET `created_by` = '', `name` = :name WHERE `id` = :id", ['name' => $name, 'id' => $id]);
-    }
-
-    /** Follows an account rename: creator and user targets */
-    public function renameAccount(string $old, string $new): void
-    {
-        $this->query("UPDATE `imapsync_source` SET `created_by` = :new WHERE `created_by` = :old", ['new' => $new, 'old' => $old]);
-        $this->query("UPDATE `imapsync_source_user` SET `username` = :new WHERE `username` = :old", ['new' => $new, 'old' => $old]);
-    }
-
-    /** Removes a deleted account from all user target lists */
-    public function deleteUserTarget(string $username): void
-    {
-        $this->query("DELETE FROM `imapsync_source_user` WHERE `username` = :username", ['username' => $username]);
-    }
-
-    private function fetch(string $where, array $params): ?Source
-    {
-        $row = $this->query("SELECT s.* FROM `imapsync_source` s WHERE s.`id` = :id AND $where", $params)->fetch(PDO::FETCH_ASSOC);
-        if (!$row) {
-            return null;
-        }
-        $source = Source::fromRow($row);
-        $id = ['id' => $source->id];
-        $source->domains = $this->query("SELECT `domain` FROM `imapsync_source_domain` WHERE `source_id` = :id ORDER BY `domain`",
-            $id)->fetchAll(PDO::FETCH_COLUMN);
-        $source->users = $this->query("SELECT `username` FROM `imapsync_source_user` WHERE `source_id` = :id ORDER BY `username`",
-            $id)->fetchAll(PDO::FETCH_COLUMN);
-        return $source;
-    }
-
-    /**
-     * WHERE fragment over `imapsync_source s` that limits rows to what the caller may see, and its parameters.
-     *   admin       everything
-     *   domainadmin own + 'all' + sources targeting his domains or their mailboxes + sources created by them
-     *   user        own + 'all' + sources targeting him or his domain
-     *
-     * @return array{0: string, 1: array}
-     */
-    private function visibility(Context $context): array
-    {
-        $username = $context->username;
-        if ($context->isAdmin()) {
-            return ['1=1', []];
-        }
-        // Native prepares: a placeholder may appear only once, so the username is bound under one name per use
-        $managed = "SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = ";
-        if ($context->isDomainAdmin()) {
-            return ["(s.`created_by` = :vis_self OR s.`scope` = 'all'
-                OR (s.`scope` = 'domain' AND EXISTS (SELECT 1 FROM `imapsync_source_domain` t
-                  WHERE t.`source_id` = s.`id` AND t.`domain` IN ($managed :vis_d1)))
-                OR (s.`scope` = 'user' AND EXISTS (SELECT 1 FROM `imapsync_source_user` t JOIN `mailbox` mb ON mb.`username` = t.`username`
-                  WHERE t.`source_id` = s.`id` AND mb.`domain` IN ($managed :vis_d2)))
-                OR s.`created_by` IN (SELECT `username` FROM `mailbox` WHERE `domain` IN ($managed :vis_d3)))",
-                ['vis_self' => $username, 'vis_d1' => $username, 'vis_d2' => $username, 'vis_d3' => $username]];
-        }
-        if ($context->isUser()) {
-            return ["(s.`created_by` = :vis_self OR s.`scope` = 'all'
-                OR (s.`scope` = 'user' AND EXISTS (SELECT 1 FROM `imapsync_source_user` t
-                  WHERE t.`source_id` = s.`id` AND t.`username` = :vis_u1))
-                OR (s.`scope` = 'domain' AND EXISTS (SELECT 1 FROM `imapsync_source_domain` t JOIN `mailbox` mb ON mb.`domain` = t.`domain`
-                  WHERE t.`source_id` = s.`id` AND mb.`username` = :vis_u2)))",
-                ['vis_self' => $username, 'vis_u1' => $username, 'vis_u2' => $username]];
-        }
-        return ['0=1', []];
-    }
-
-    private function writeTargets(Source $source): void
-    {
-        foreach ($source->domains as $domain) {
-            $this->query("INSERT INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)",
-                ['id' => $source->id, 'domain' => $domain]);
-        }
-        foreach ($source->users as $username) {
-            $this->query("INSERT INTO `imapsync_source_user` (`source_id`, `username`) VALUES (:id, :username)",
-                ['id' => $source->id, 'username' => $username]);
-        }
-        $this->query("UPDATE `imapsync_source` SET `scope` = :scope WHERE `id` = :id", ['scope' => $source->scope, 'id' => $source->id]);
-    }
-
-    private function values(Source $source): array
-    {
-        return [
-            'name' => $source->name,
-            'description' => $source->description,
-            'host1' => $source->host,
-            'port1' => $source->port,
-            'enc1' => $source->enc,
-            'auth_type' => $source->authType,
-            'oauth_flow' => $source->oauthFlow,
-            'oauth_token_endpoint' => $source->tokenEndpoint,
-            'oauth_authorize_endpoint' => $source->authorizeEndpoint,
-            'oauth_userinfo_endpoint' => $source->userinfoEndpoint,
-            'oauth_client_id' => $source->clientId,
-            'oauth_client_secret' => $source->clientSecret,
-            'oauth_scope' => $source->oauthScope,
-            'oauth_extra_params' => $source->extraParams,
-            'active' => $source->active,
-        ];
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Repository/TokenRepository.php

@@ -1,123 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\Imapsync\Model\OAuthToken;
-use Mailcow\Imapsync\Model\Source;
-use PDO;
-
-/**
- * SQL for OAuth tokens: the shared token of a client_credentials source (columns of `imapsync_source`)
- * and the per-user tokens of authorization_code sources (`imapsync_source_oauth_token`).
- */
-class TokenRepository extends Repository
-{
-    private const USER_TOKEN = "`source_id` = :sid AND `owner` = :owner AND `username` = :user";
-
-    public function exists(int $sourceId, string $owner, string $username): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `imapsync_source_oauth_token` WHERE " . self::USER_TOKEN,
-            ['sid' => $sourceId, 'owner' => $owner, 'user' => $username])->fetchColumn();
-    }
-
-    /**
-     * Stores the token of a finished browser sign-in. An existing refresh token is kept when the
-     * provider sent none (Microsoft only sends one with offline_access).
-     */
-    public function saveUserToken(OAuthToken $token): void
-    {
-        $this->query("INSERT INTO `imapsync_source_oauth_token`
-            (`source_id`, `owner`, `username`, `access_token`, `refresh_token`, `token_expires`, `last_refresh_error`, `oauth_last_refresh_attempt`)
-          VALUES (:sid, :owner, :user, :acc, :ref, :exp, NULL, NOW())
-          ON DUPLICATE KEY UPDATE
-            `access_token` = VALUES(`access_token`),
-            `refresh_token` = IF(VALUES(`refresh_token`) = '', `refresh_token`, VALUES(`refresh_token`)),
-            `token_expires` = VALUES(`token_expires`),
-            `last_refresh_error` = NULL,
-            `oauth_last_refresh_attempt` = NOW()",
-            $this->key($token) + ['acc' => $token->accessToken, 'ref' => (string)$token->refreshToken, 'exp' => $token->tokenExpires]);
-    }
-
-    /** Stores a refreshed per-user token and clears its last error */
-    public function storeUserRefresh(OAuthToken $token): void
-    {
-        $this->query("UPDATE `imapsync_source_oauth_token` SET `access_token` = :acc, `refresh_token` = :ref,
-            `token_expires` = :exp, `last_refresh_error` = NULL, `oauth_last_refresh_attempt` = NOW() WHERE " . self::USER_TOKEN,
-            $this->key($token) + ['acc' => $token->accessToken, 'ref' => $token->refreshToken, 'exp' => $token->tokenExpires]);
-    }
-
-    public function storeUserError(OAuthToken $token, string $error): void
-    {
-        $this->query("UPDATE `imapsync_source_oauth_token` SET `last_refresh_error` = :err, `oauth_last_refresh_attempt` = NOW()
-          WHERE " . self::USER_TOKEN, $this->key($token) + ['err' => $error]);
-    }
-
-    public function storeSourceToken(int $sourceId, string $accessToken, int $expires): void
-    {
-        $this->query("UPDATE `imapsync_source` SET `oauth_access_token` = :tok, `oauth_token_expires` = :exp,
-            `oauth_last_refresh_error` = NULL, `oauth_last_refresh_attempt` = NOW() WHERE `id` = :id",
-            ['tok' => $accessToken, 'exp' => $expires, 'id' => $sourceId]);
-    }
-
-    public function storeSourceError(int $sourceId, string $error): void
-    {
-        $this->query("UPDATE `imapsync_source` SET `oauth_last_refresh_error` = :err, `oauth_last_refresh_attempt` = NOW()
-          WHERE `id` = :id", ['err' => $error, 'id' => $sourceId]);
-    }
-
-    /** @return Source[] active client_credentials sources (incl. secrets), each with refreshAttemptAge */
-    public function refreshableSources(): array
-    {
-        return array_map([Source::class, 'fromRow'], $this->query("SELECT *,
-            TIMESTAMPDIFF(SECOND, `oauth_last_refresh_attempt`, NOW()) AS `refresh_attempt_age`
-          FROM `imapsync_source`
-          WHERE `active` = 1 AND `auth_type` = 'XOAUTH2' AND `oauth_flow` = 'client_credentials'
-          ORDER BY `id`")->fetchAll(PDO::FETCH_ASSOC));
-    }
-
-    /** @return OAuthToken[] per-user tokens with a refresh token on active authorization_code sources, with refreshAttemptAge */
-    public function refreshableUserTokens(): array
-    {
-        return array_map([OAuthToken::class, 'fromRow'], $this->query("SELECT t.*,
-            TIMESTAMPDIFF(SECOND, t.`oauth_last_refresh_attempt`, NOW()) AS `refresh_attempt_age`
-          FROM `imapsync_source_oauth_token` t
-          JOIN `imapsync_source` s ON s.`id` = t.`source_id`
-          WHERE s.`active` = 1 AND s.`auth_type` = 'XOAUTH2' AND s.`oauth_flow` = 'authorization_code'
-            AND t.`refresh_token` <> ''
-          ORDER BY t.`source_id`, t.`owner`, t.`username`")->fetchAll(PDO::FETCH_ASSOC));
-    }
-
-    /** Drops a per-user token once no job uses (source, owner, username) anymore */
-    public function deleteIfUnused(int $sourceId, string $owner, string $username): void
-    {
-        $this->query("DELETE FROM `imapsync_source_oauth_token` WHERE " . self::USER_TOKEN . "
-            AND NOT EXISTS (SELECT 1 FROM `imapsync` WHERE `source_id` = :sid2 AND `user2` = :owner2 AND `user1` = :user2)",
-            ['sid' => $sourceId, 'owner' => $owner, 'user' => $username, 'sid2' => $sourceId, 'owner2' => $owner, 'user2' => $username]);
-    }
-
-    /** Drops per-user tokens that no job uses and that are older than $seconds. Returns the number of rows. */
-    public function deleteUnusedOlderThan(int $seconds): int
-    {
-        return $this->query("DELETE t FROM `imapsync_source_oauth_token` t
-          LEFT JOIN `imapsync` i ON i.`source_id` = t.`source_id` AND i.`user2` = t.`owner` AND i.`user1` = t.`username`
-          WHERE i.`id` IS NULL AND t.`created` < NOW() - INTERVAL :seconds SECOND", ['seconds' => $seconds])->rowCount();
-    }
-
-    public function deleteByOwner(string $owner): void
-    {
-        $this->query("DELETE FROM `imapsync_source_oauth_token` WHERE `owner` = :owner", ['owner' => $owner]);
-    }
-
-    public function renameOwner(string $old, string $new): void
-    {
-        $this->query("UPDATE `imapsync_source_oauth_token` SET `owner` = :new WHERE `owner` = :old", ['new' => $new, 'old' => $old]);
-    }
-
-    private function key(OAuthToken $token): array
-    {
-        return ['sid' => $token->sourceId, 'owner' => $token->owner, 'user' => $token->username];
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Service/JobService.php

@@ -1,286 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\Imapsync\Model\CustomParams;
-use Mailcow\Imapsync\Model\Job;
-use Mailcow\Imapsync\Model\Settings;
-use Mailcow\Imapsync\Model\Source;
-use Mailcow\Imapsync\Repository\JobRepository;
-use Mailcow\Imapsync\Repository\SettingsRepository;
-use Mailcow\Imapsync\Repository\TokenRepository;
-
-/**
- * Use cases and rules for sync jobs and the global sync settings.
- *
- * A job belongs to its local mailbox (user2): whoever may access that mailbox may see and manage it
- * (admins also orphaned jobs). Writing needs the `syncjobs` ACL. Jobs on an authorization_code source
- * need the per-user token of (source, owner = user2, username = user1) first (C1).
- */
-class JobService
-{
-    private const FLAGS = ['active', 'delete2duplicates', 'subscribeall', 'dry', 'delete1', 'delete2', 'automap', 'skipcrossduplicates'];
-
-    public function __construct(private JobRepository $jobs, private SourceService $sources, private TokenRepository $tokens,
-        private SettingsRepository $settings, private Context $context, private array $allowlist)
-    {
-    }
-
-    /** @throws AccessDenied without the syncjobs ACL */
-    public function requireAcl(): void
-    {
-        if (!$this->context->hasAcl('syncjobs')) {
-            throw new AccessDenied();
-        }
-    }
-
-    /** The allowed custom params (option => rule), for the job form */
-    public function customParamOptions(): array
-    {
-        return $this->allowlist;
-    }
-
-    /**
-     * The job when the caller may access its mailbox (incl. password1: never pass the object to the UI, use toArray()).
-     * Admins may also manage orphaned jobs whose local mailbox no longer exists.
-     */
-    public function get(int $id, bool $withLog = true): ?Job
-    {
-        $job = $this->jobs->find($id, $withLog);
-        return ($job !== null && ($this->context->isAdmin() || $this->context->canAccessMailbox($job->user2))) ? $job : null;
-    }
-
-    /** @return int[] job ids of a mailbox (default: the caller's own) in queue order; [] without access */
-    public function idsForMailbox(mixed $mailbox): array
-    {
-        if ($mailbox !== null && filter_var($mailbox, FILTER_VALIDATE_EMAIL)) {
-            if (!$this->context->canAccessMailbox((string)$mailbox)) {
-                return [];
-            }
-        } else {
-            $mailbox = $this->context->username;
-        }
-        return ($mailbox === null) ? [] : $this->jobs->idsForMailbox((string)$mailbox);
-    }
-
-    /** Only admins list all jobs at once (incl. orphaned ones) */
-    public function canListAll(): bool
-    {
-        return $this->context->isAdmin();
-    }
-
-    /** @return int[] all job ids in queue order; [] for non-admins */
-    public function allIds(): array
-    {
-        return $this->canListAll() ? $this->jobs->allIds() : [];
-    }
-
-    /** @throws UserError */
-    public function create(array $input): Job
-    {
-        $this->requireAcl();
-        $job = new Job();
-        $job->user2 = $this->ownerMailbox($input);
-        // Add keeps its legacy parsing of these attributes (e.g. maxage '1.5' means none, on edit 1); then the edit rules apply
-        $input['source_id'] = (int)($input['source_id'] ?? 0);
-        $input['maxage'] = Input::intInRange($input['maxage'] ?? null, 1, 32000) ?? 0;
-        $input['maxbytespersecond'] = Input::intInRange($input['maxbytespersecond'] ?? null, 1, 125000000) ?? 0;
-        $this->save($job, null, $input + ['custom_params' => '']);
-        return $job;
-    }
-
-    /** @throws UserError */
-    public function update(int $id, array $input): Job
-    {
-        $this->requireAcl();
-        $current = $this->get($id, false) ?? throw new AccessDenied();
-        $job = clone $current;
-        $this->save($job, $current, $input);
-        return $job;
-    }
-
-    /** @throws UserError access_denied when the job does not exist or the caller may not access it */
-    public function delete(int $id): void
-    {
-        $this->requireAcl();
-        $job = $this->get($id, false) ?? throw new AccessDenied();
-        // prio is a sparse sort key (ties allowed), so no gap needs closing
-        $this->jobs->delete($id);
-        $this->tokens->deleteIfUnused($job->sourceId, $job->user2, $job->user1);
-    }
-
-    /**
-     * Admin only (C4): moves jobs to the front of the queue. Running jobs are left alone; the runner never resets prio.
-     *
-     * @param int[] $ids
-     * @return array{running: int[], moved: int}
-     */
-    public function runNext(array $ids): array
-    {
-        $ids = array_values(array_unique($ids));
-        if (!$this->context->isAdmin() || empty($ids)) {
-            throw new AccessDenied();
-        }
-        return $this->jobs->moveToFront($ids);
-    }
-
-    /** @throws AccessDenied for non-admins */
-    public function getSettings(): Settings
-    {
-        if (!$this->context->isAdmin()) {
-            throw new AccessDenied();
-        }
-        return $this->settings->load();
-    }
-
-    /**
-     * Saves the global settings (admin only). The UI sends the bandwidth limit in KB/s; stored as bytes/s
-     * like the per-job limit.
-     *
-     * @throws UserError
-     */
-    public function saveSettings(array $input): Settings
-    {
-        if (!$this->context->isAdmin()) {
-            throw new AccessDenied();
-        }
-        $maxParallel = Input::intInRange($input['max_parallel'] ?? null, 1, 32);
-        $kb = $input['max_kb_per_second'] ?? 0;
-        if ($maxParallel === null) {
-            throw new UserError('imapsync_max_parallel_invalid');
-        }
-        // 0 (unlimited) .. 10 GiB/s
-        $kb = Input::intInRange(($kb === '') ? 0 : $kb, 0, 10485760) ?? throw new UserError('imapsync_max_kb_per_second_invalid');
-        $settings = new Settings();
-        $settings->maxParallel = $maxParallel;
-        $settings->maxBytesPerSecond = $kb * 1024;
-        $this->settings->save($settings);
-        return $settings;
-    }
-
-    /**
-     * Applies an add ($current = null) or edit request to $job and stores it. Attributes missing in $input keep
-     * the value of $job, an empty password keeps the stored one. On edit the source is only checked when the job
-     * moves to another source, so a domain admin or user can still edit or deactivate a job bound to a source
-     * he cannot see.
-     *
-     * @throws UserError
-     */
-    private function save(Job $job, ?Job $current, array $input): void
-    {
-        foreach (self::FLAGS as $flag) {
-            if (isset($input[$flag])) {
-                $job->$flag = (int)$input[$flag];
-            }
-        }
-        foreach (['maxage', 'timeout1', 'timeout2'] as $number) {
-            if (isset($input[$number]) && $input[$number] != '') {
-                $job->$number = (int)$input[$number];
-            }
-        }
-        if (isset($input['maxbytespersecond']) && $input['maxbytespersecond'] != '') {
-            $job->maxBytesPerSecond = (string)(int)$input['maxbytespersecond'];
-        }
-        // last_run/success are reset when present; prio is a global-queue concern that only admins may change
-        $job->lastRun = isset($input['last_run']) ? null : $job->lastRun;
-        $job->success = isset($input['success']) ? null : $job->success;
-        if (isset($input['prio']) && is_numeric($input['prio']) && $this->context->isAdmin()) {
-            $job->prio = (int)$input['prio'];
-        }
-        // An empty user1 keeps the stored one on edit; a new job needs one
-        $user1 = (isset($input['user1']) && Input::text($input['user1']) !== '') ? Input::text($input['user1']) : $job->user1;
-        if ($user1 === null || mb_strlen($user1) > 255 || ($current === null && $user1 === '')) {
-            throw new UserError('imapsync_user1_invalid');
-        }
-        $password = (isset($input['password1']) && is_scalar($input['password1']) && (string)$input['password1'] !== '')
-            ? (string)$input['password1'] : $job->password1;
-        $sourceId = (!empty($input['source_id']) && is_scalar($input['source_id'])) ? (int)$input['source_id'] : $job->sourceId;
-        $sourceChanged = ($current === null || $sourceId !== $current->sourceId);
-        $tupleChanged = $sourceChanged || $user1 !== $current->user1;
-        $job->user1 = $user1;
-        $job->sourceId = $sourceId;
-
-        $source = $sourceChanged ? $this->sources->get($sourceId) : $current->source;
-        if ($source === null || ($sourceChanged && $source->active !== 1)) {
-            throw new UserError('imapsync_source_unavailable');
-        }
-        if ($source->usesClientCredentials() && $tupleChanged && !$sourceChanged && !$this->mayUse($source)) {
-            // The app token reaches every mailbox of the tenant, so only callers that may use the source
-            // may pick another remote mailbox (user1). Other fields stay editable.
-            throw new AccessDenied();
-        }
-        $job->password1 = $this->checkAuth($source, $job, $password, $tupleChanged);
-        if (array_key_exists('custom_params', $input)) {
-            $job->customParams = CustomParams::toJson($input['custom_params'], $this->allowlist);
-        }
-        $job->maxage = Input::intInRange($job->maxage, 1, 32000) ?? 0;
-        $job->timeout1 = Input::intInRange($job->timeout1, 1, 32000) ?? 600;
-        $job->timeout2 = Input::intInRange($job->timeout2, 1, 32000) ?? 600;
-        $job->maxBytesPerSecond = (string)(Input::intInRange($job->maxBytesPerSecond, 1, 125000000) ?? 0);
-        $exclude = isset($input['exclude']) ? Input::text($input['exclude']) : $job->exclude;
-        $subfolder2 = isset($input['subfolder2']) ? Input::text($input['subfolder2']) : $job->subfolder2;
-        $minsInterval = Input::intInRange(!empty($input['mins_interval']) ? $input['mins_interval'] : $job->minsInterval, 1, 43800);
-        // exclude must be a valid regex, the interval 1..43800 minutes
-        if ($exclude === null || $subfolder2 === null || $minsInterval === null || @preg_match("/" . $exclude . "/", '') === false) {
-            throw new AccessDenied();
-        }
-        $job->exclude = $exclude;
-        $job->subfolder2 = $subfolder2;
-        $job->minsInterval = $minsInterval;
-        if ($tupleChanged && $this->jobs->tupleExists($job->sourceId, $job->user2, $job->user1, $current?->id)) {
-            throw new UserError('object_exists', [htmlspecialchars($source->name . ' / ' . $job->user1)]);
-        }
-        if ($current === null) {
-            $this->jobs->insert($job);
-            return;
-        }
-        $this->jobs->update($job);
-        // The previous (source, owner, remote user) token is dropped once no job uses it (C1)
-        if ($tupleChanged) {
-            $this->tokens->deleteIfUnused($current->sourceId, $current->user2, $current->user1);
-        }
-    }
-
-    /** Whether the caller may use the source for new jobs (sees it or may edit it) */
-    private function mayUse(Source $source): bool
-    {
-        return $this->sources->get((int)$source->id) !== null || $this->sources->canEdit($source);
-    }
-
-    /**
-     * Local mailbox of a new job: `username` (needs access), else the caller's own mailbox (users only).
-     *
-     * @throws UserError access_denied, no_user_defined
-     */
-    private function ownerMailbox(array $input): string
-    {
-        $username = $input['username'] ?? null;
-        if (is_string($username) && filter_var($username, FILTER_VALIDATE_EMAIL)) {
-            return $this->context->canAccessMailbox($username) ? $username : throw new AccessDenied();
-        }
-        return $this->context->isUser() ? (string)$this->context->username : throw new UserError('no_user_defined');
-    }
-
-    /**
-     * Returns the password to store. XOAUTH2 sources carry no password (the runner uses an access token);
-     * an authorization_code source needs the per-user token of (source, owner, remote user) when it is new.
-     *
-     * @throws UserError imapsync_source_oauth_connect_required, password_empty
-     */
-    private function checkAuth(Source $source, Job $job, string $password, bool $tupleChanged): string
-    {
-        if (!$source->isXOAuth2()) {
-            return ($password !== '') ? $password : throw new UserError('password_empty');
-        }
-        if ($source->usesAuthorizationCode() && $tupleChanged && !$this->tokens->exists($job->sourceId, $job->user2, $job->user1)) {
-            throw new UserError('imapsync_source_oauth_connect_required');
-        }
-        return '';
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Service/OAuthService.php

@@ -1,332 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Service;
-
-use Exception;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\Imapsync\Http\OAuthClient;
-use Mailcow\Imapsync\Http\OAuthException;
-use Mailcow\Imapsync\Model\OAuthToken;
-use Mailcow\Imapsync\Model\Source;
-use Mailcow\Imapsync\Repository\SourceRepository;
-use Mailcow\Imapsync\Repository\TokenRepository;
-use PDOException;
-
-/**
- * OAuth2 for XOAUTH2 sources: their OAuth settings, token requests for every grant type, the browser sign-in
- * (authorization_code) and token refresh. Needs no Context, so the refresh cron can use it; callers check
- * permissions first.
- */
-class OAuthService
-{
-    /** Tokens expiring within this many seconds are refreshed (the runner starts jobs with >= 15 min left) */
-    private const REFRESH_AHEAD = 1200;
-    /** A token whose last refresh failed is retried after this many seconds */
-    private const RETRY_AFTER = 180;
-    /** Extra params that belong to the browser sign-in; all others go to the token request */
-    private const AUTHORIZE_PARAMS = ['prompt', 'access_type', 'login_hint', 'domain_hint', 'hd'];
-    /** Per-user tokens that no job uses are kept this long (time to save the job after signing in) */
-    private const UNUSED_TOKEN_GRACE = 86400;
-
-    private string $hostname;
-
-    public function __construct(private SourceRepository $sources, private TokenRepository $tokens, private OAuthClient $client,
-        string $hostname)
-    {
-        $this->hostname = strtolower(trim($hostname));
-    }
-
-    /**
-     * Redirect URI to register at the provider. Only MAILCOW_HOSTNAME is supported: the popup callback
-     * needs the session cookie and opener origin of the UI, so the UI must be used on that host.
-     */
-    public function redirectUri(): string
-    {
-        return ($this->hostname === '') ? '' : 'https://' . $this->hostname . '/syncjob-oauth';
-    }
-
-    /**
-     * Sets the OAuth settings of a source from request attributes: cleared for password sources, read and
-     * validated for XOAUTH2 (incl. the endpoint policy C5). Missing attributes keep the value of $source,
-     * an empty client secret keeps the stored one. Permission checks are the caller's job.
-     *
-     * @throws UserError
-     */
-    public function applySettings(Source $source, array $input): void
-    {
-        if (!$source->isXOAuth2()) {
-            $source->oauthFlow = Source::FLOW_CLIENT_CREDENTIALS;
-            $source->tokenEndpoint = $source->authorizeEndpoint = $source->userinfoEndpoint = null;
-            $source->clientId = $source->clientSecret = $source->oauthScope = $source->extraParams = null;
-            return;
-        }
-        $source->oauthFlow = (($input['oauth_flow'] ?? $source->oauthFlow) === Source::FLOW_AUTHORIZATION_CODE)
-            ? Source::FLOW_AUTHORIZATION_CODE : Source::FLOW_CLIENT_CREDENTIALS;
-        $source->tokenEndpoint = $this->oauthText($input, 'oauth_token_endpoint', $source->tokenEndpoint);
-        $authorizeEndpoint = $this->oauthText($input, 'oauth_authorize_endpoint', $source->authorizeEndpoint);
-        $userinfoEndpoint = $this->oauthText($input, 'oauth_userinfo_endpoint', $source->userinfoEndpoint);
-        $source->clientId = $this->oauthText($input, 'oauth_client_id', $source->clientId);
-        $source->oauthScope = $this->oauthText($input, 'oauth_scope', $source->oauthScope);
-        $secret = $input['oauth_client_secret'] ?? '';
-        if (!is_scalar($secret)) {
-            throw new UserError('imapsync_source_oauth_invalid');
-        }
-        $source->clientSecret = ((string)$secret !== '') ? (string)$secret : (string)$source->clientSecret;
-        $source->extraParams = $this->extraParams(array_key_exists('oauth_extra_params', $input) ? $input['oauth_extra_params'] : $source->extraParams);
-
-        if ($source->tokenEndpoint === '' || $source->clientId === '' || $source->clientSecret === '' || $source->oauthScope === '') {
-            throw new UserError('imapsync_source_oauth_required');
-        }
-        if (mb_strlen($source->clientId) > 500 || mb_strlen($source->clientSecret) > 500 || mb_strlen($source->oauthScope) > 500) {
-            throw new UserError('imapsync_source_oauth_invalid');
-        }
-        // C5, checked again before every request. The authorize endpoint is only opened in the browser,
-        // but must not point users to internal hosts either.
-        $endpoints = [$source->tokenEndpoint];
-        if ($source->usesAuthorizationCode()) {
-            $endpoints[] = $authorizeEndpoint;
-        }
-        if ($userinfoEndpoint !== '') {
-            $endpoints[] = $userinfoEndpoint;
-        }
-        foreach ($endpoints as $endpoint) {
-            if (!$this->endpointAllowed($endpoint, $source->createdBy)) {
-                throw new UserError('imapsync_source_oauth_endpoint_invalid');
-            }
-        }
-        $source->authorizeEndpoint = ($authorizeEndpoint === '') ? null : $authorizeEndpoint;
-        $source->userinfoEndpoint = ($userinfoEndpoint === '') ? null : $userinfoEndpoint;
-    }
-
-    /** Provider consent URL for the browser sign-in, or null when the authorize endpoint is not allowed */
-    public function authorizeUrl(Source $source, string $state): ?string
-    {
-        $endpoint = (string)$source->authorizeEndpoint;
-        if (!$this->endpointAllowed($endpoint, $source->createdBy)) {
-            return null;
-        }
-        // select_account: never sign in silently with the account the browser already uses (often the admin's)
-        $params = array_merge(['prompt' => 'select_account'], $this->extraParamsFor($source, true), ['client_id' => $source->clientId,
-            'response_type' => 'code', 'redirect_uri' => $this->redirectUri(), 'response_mode' => 'query',
-            'scope' => $source->oauthScope, 'state' => $state]);
-        return $endpoint . (str_contains($endpoint, '?') ? '&' : '?') . http_build_query($params);
-    }
-
-    /**
-     * Finishes the browser sign-in: exchanges the code, finds the remote identity and stores the token
-     * for the local mailbox $owner (C1). Returns the remote identity (the job's user1).
-     *
-     * @throws UserError token_failed (args: detail for the server log) or no_identity
-     */
-    public function connectUser(Source $source, string $owner, string $code): string
-    {
-        try {
-            $token = $this->requestToken($source, ['grant_type' => 'authorization_code', 'code' => $code, 'redirect_uri' => $this->redirectUri()]);
-        } catch (OAuthException $e) {
-            throw new UserError('token_failed', ['token exchange failed for source ' . $source->id . ': ' . $e->getMessage()]);
-        }
-        $identity = $this->identity($source, $token);
-        if ($identity === null) {
-            throw new UserError('no_identity');
-        }
-        try {
-            $this->tokens->saveUserToken(OAuthToken::fromRow(['source_id' => $source->id, 'owner' => $owner, 'username' => $identity,
-                'access_token' => $token['access_token'], 'refresh_token' => $token['refresh_token'] ?? '',
-                'token_expires' => time() + (int)$token['expires_in']]));
-        } catch (PDOException $e) {
-            throw new UserError('token_failed', ['storing token failed for source ' . $source->id . ': ' . $e->getMessage()]);
-        }
-        return $identity;
-    }
-
-    /** Fetches and stores a new shared token of a client_credentials source (or stores the error); returns the error or null */
-    public function refreshSourceToken(Source $source): ?string
-    {
-        if (!$source->usesClientCredentials()) {
-            return 'not refreshable';
-        }
-        try {
-            $token = $this->requestToken($source, ['grant_type' => 'client_credentials']);
-        } catch (OAuthException $e) {
-            $this->tokens->storeSourceError((int)$source->id, $e->getMessage());
-            return $e->getMessage();
-        }
-        $this->tokens->storeSourceToken((int)$source->id, $token['access_token'], time() + (int)$token['expires_in']);
-        return null;
-    }
-
-    /** Refreshes and stores a per-user token via its refresh token (or stores the error); returns the error or null */
-    public function refreshUserToken(Source $source, OAuthToken $token): ?string
-    {
-        if (!$source->usesAuthorizationCode() || (string)$token->refreshToken === '') {
-            return 'not refreshable';
-        }
-        try {
-            $new = $this->requestToken($source, ['grant_type' => 'refresh_token', 'refresh_token' => $token->refreshToken]);
-        } catch (OAuthException $e) {
-            $this->tokens->storeUserError($token, $e->getMessage());
-            return $e->getMessage();
-        }
-        $token->accessToken = $new['access_token'];
-        // Some providers omit a new refresh token on refresh: keep the existing one then
-        $token->refreshToken = empty($new['refresh_token']) ? $token->refreshToken : $new['refresh_token'];
-        $token->tokenExpires = time() + (int)$new['expires_in'];
-        $this->tokens->storeUserRefresh($token);
-        return null;
-    }
-
-    /**
-     * Cron run: refreshes due source tokens, drops unused per-user tokens, refreshes due per-user tokens.
-     *
-     * @return array<int, array{priority: string, message: string}> log entries for CRON_LOG
-     */
-    public function refreshDue(): array
-    {
-        $log = [];
-        foreach ($this->tokens->refreshableSources() as $source) {
-            if ($this->isDue($source->tokenExpires, $source->lastRefreshError, $source->refreshAttemptAge)) {
-                try {
-                    $error = $this->refreshSourceToken($source);
-                } catch (Exception $e) {
-                    $error = get_class($e);
-                }
-                $label = "source '" . $source->name . "' (id=" . $source->id . ")";
-                $log[] = $this->logEntry($error, "Refreshed token for " . $label, "Refresh failed for " . $label);
-            }
-        }
-        $removed = $this->tokens->deleteUnusedOlderThan(self::UNUSED_TOKEN_GRACE);
-        if ($removed > 0) {
-            $log[] = ['priority' => 'info', 'message' => "Removed " . $removed . " unused user token(s)"];
-        }
-        foreach ($this->tokens->refreshableUserTokens() as $token) {
-            if ($this->isDue($token->tokenExpires, $token->lastRefreshError, $token->refreshAttemptAge)) {
-                $source = null;
-                try {
-                    $source = $this->sources->find($token->sourceId);
-                    $error = ($source === null) ? 'not refreshable' : $this->refreshUserToken($source, $token);
-                } catch (Exception $e) {
-                    $error = get_class($e);
-                }
-                $label = "user token '" . $token->username . "' (mailbox " . $token->owner . ") on source '"
-                    . ($source->name ?? '') . "' (id=" . $token->sourceId . ")";
-                $log[] = $this->logEntry($error, "Refreshed " . $label, "Refresh failed for " . $label);
-            }
-        }
-        return $log;
-    }
-
-    /** Due when the token expires soon (or is missing) and the last failed attempt is old enough */
-    private function isDue(?int $expires, ?string $lastError, ?int $attemptAge): bool
-    {
-        return ($expires === null || $expires < time() + self::REFRESH_AHEAD)
-            && ($lastError === null || $attemptAge === null || $attemptAge > self::RETRY_AFTER);
-    }
-
-    private function logEntry(?string $error, string $success, string $failure): array
-    {
-        return ($error === null) ? ['priority' => 'info', 'message' => $success]
-            : ['priority' => 'warning', 'message' => $failure . ": " . $error];
-    }
-
-    /**
-     * Endpoint policy (C5). Endpoints of sources not created by an admin must resolve to public addresses;
-     * admin sources have no address restriction (API key sessions run as admin under the reserved name 'API').
-     */
-    private function endpointAllowed(string $url, string $createdBy): bool
-    {
-        return $this->client->isAllowed($url, $this->isStrict($createdBy));
-    }
-
-    private function isStrict(string $createdBy): bool
-    {
-        return !($createdBy === 'API' || $this->sources->isSuperAdmin($createdBy));
-    }
-
-    /** One OAuth text attribute from the request, or the stored value when the attribute is missing */
-    private function oauthText(array $input, string $key, ?string $stored): string
-    {
-        $value = array_key_exists($key, $input) ? Input::text($input[$key]) : (string)$stored;
-        return $value ?? throw new UserError('imapsync_source_oauth_invalid');
-    }
-
-    /** Stored extra params of the browser sign-in ($authorize) or of the token request */
-    private function extraParamsFor(Source $source, bool $authorize): array
-    {
-        $extra = json_decode((string)$source->extraParams, true);
-        return array_filter(is_array($extra) ? $extra : [],
-            fn($key): bool => in_array($key, self::AUTHORIZE_PARAMS, true) === $authorize, ARRAY_FILTER_USE_KEY);
-    }
-
-    /**
-     * Extra parameters: a JSON object with scalar values, as string or as array (after 011-api.js
-     * auto-parsing). Returns the JSON to store, null for none.
-     */
-    private function extraParams(mixed $extra): ?string
-    {
-        if ($extra === null || (is_string($extra) && trim($extra) === '')) {
-            $extra = [];
-        } elseif (is_string($extra)) {
-            $extra = json_decode($extra, true);
-        }
-        $bad = !is_array($extra);
-        foreach ((array)$extra as $key => $value) {
-            $bad = $bad || !is_string($key) || $key === '' || !(is_scalar($value) || $value === null);
-        }
-        $json = empty($extra) ? null : json_encode($extra, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-        if ($bad || strlen((string)$json) > 4000) {
-            throw new UserError('imapsync_source_oauth_extra_invalid');
-        }
-        return $json;
-    }
-
-    /**
-     * Token endpoint request for every grant type. Stored extra params never override the client and grant fields.
-     *
-     * @throws OAuthException
-     */
-    private function requestToken(Source $source, array $grant): array
-    {
-        $post = array_merge($this->extraParamsFor($source, false),
-            ['client_id' => $source->clientId, 'client_secret' => $source->clientSecret, 'scope' => $source->oauthScope], $grant);
-        $token = $this->client->request((string)$source->tokenEndpoint, $this->isStrict($source->createdBy), $post);
-        if (empty($token['access_token']) || !is_string($token['access_token'])) {
-            throw new OAuthException('malformed token response');
-        }
-        if (!isset($token['expires_in']) || !is_numeric($token['expires_in'])) {
-            $token['expires_in'] = 3600;
-        }
-        if (isset($token['refresh_token']) && !is_string($token['refresh_token'])) {
-            unset($token['refresh_token']);
-        }
-        return $token;
-    }
-
-    /** Remote identity: id_token claims (received directly from the token endpoint over TLS), else the userinfo endpoint */
-    private function identity(Source $source, array $token): ?string
-    {
-        $parts = explode('.', is_string($token['id_token'] ?? null) ? $token['id_token'] : '');
-        $email = (count($parts) === 3) ? $this->emailClaim(json_decode((string)base64_decode(strtr($parts[1], '-_', '+/')), true)) : null;
-        if ($email !== null || empty($source->userinfoEndpoint)) {
-            return $email;
-        }
-        try {
-            return $this->emailClaim($this->client->request($source->userinfoEndpoint, $this->isStrict($source->createdBy), null,
-                ['Authorization: Bearer ' . $token['access_token']]));
-        } catch (OAuthException $e) {
-            return null;
-        }
-    }
-
-    private function emailClaim(mixed $claims): ?string
-    {
-        foreach (['email', 'preferred_username', 'upn', 'unique_name'] as $key) {
-            if (is_array($claims) && !empty($claims[$key]) && is_string($claims[$key]) && filter_var($claims[$key], FILTER_VALIDATE_EMAIL)) {
-                return strtolower($claims[$key]);
-            }
-        }
-        return null;
-    }
-}

data/web/inc/lib/mailcow/Imapsync/Service/SourceService.php

@@ -1,357 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Imapsync\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\Imapsync\Model\Source;
-use Mailcow\Imapsync\Repository\SourceRepository;
-use Mailcow\Imapsync\Repository\TokenRepository;
-
-/**
- * Use cases and rules for sync sources.
- *
- * Read: admin all; domain admin own + 'all' + sources targeting his domains/mailboxes + sources created by
- *       his mailboxes; user own + 'all' + sources targeting him or his domain (see SourceRepository).
- * Edit: admin all; creator own; domain admin sources created by his mailboxes.
- * XOAUTH2 sources are created and edited by admins and domain admins only (users may use them).
- * Users and domain admins need the `syncjobs` ACL for everything.
- */
-class SourceService
-{
-    private const ENCRYPTIONS = ['TLS', 'SSL', 'PLAIN'];
-    private const AUTH_TYPES = ['PLAIN', 'LOGIN', 'CRAM-MD5', 'XOAUTH2'];
-    private const SCOPES = ['all', 'domain', 'user'];
-
-    public function __construct(private SourceRepository $repo, private TokenRepository $tokens, private OAuthService $oauth,
-        private Context $context)
-    {
-    }
-
-    /** @throws AccessDenied without the syncjobs ACL */
-    public function requireAcl(): void
-    {
-        if (!$this->context->hasAcl('syncjobs')) {
-            throw new AccessDenied();
-        }
-    }
-
-    /** Add/edit: a logged-in admin, domain admin or user with the syncjobs ACL */
-    public function requireEditor(): void
-    {
-        if (!($this->context->isAdmin() || $this->context->isDomainAdmin() || $this->context->isUser())) {
-            throw new AccessDenied();
-        }
-        $this->requireAcl();
-    }
-
-    /**
-     * The source when the caller may see it (incl. secrets: never pass the object to the UI, use toArray()).
-     * Users and domain admins without the syncjobs ACL see no sources.
-     */
-    public function get(int $id): ?Source
-    {
-        return $this->canRead() ? $this->repo->findVisible($id, $this->context) : null;
-    }
-
-    /** @return int[] ids of the sources the caller may see, ordered by name */
-    public function visibleIds(): array
-    {
-        return $this->canRead() ? $this->repo->visibleIds($this->context) : [];
-    }
-
-    public function canEdit(Source $source): bool
-    {
-        if ($this->context->hasAdminRights() || $source->createdBy === $this->context->username) {
-            return true;
-        }
-        // A domain admin may manage sources created by mailboxes of his domains
-        return $this->context->isDomainAdmin() && $source->createdBy !== '' && $this->context->canAccessMailbox($source->createdBy);
-    }
-
-    /** API representation for the caller (can_edit, extra params only for editors, redirect URI) */
-    public function toArray(Source $source): array
-    {
-        return $source->toArray($this->canEdit($source), $this->oauth->redirectUri());
-    }
-
-    /**
-     * Creates a source from request attributes (C9 names; host1/port1/enc1 are accepted as aliases).
-     *
-     * @throws UserError
-     */
-    public function create(array $input): Source
-    {
-        $this->requireEditor();
-        $source = new Source();
-        $source->createdBy = (string)$this->context->username;
-        $this->applyBasics($source, $input);
-        $this->applyOAuth($source, $input);
-        // Scope and targets are validated before anything is written
-        $this->applyScope($source, $input['scope'] ?? 'user', $input['domains'] ?? [], $input['users'] ?? []);
-        if ($this->repo->nameExists($source->name, $source->createdBy)) {
-            throw new UserError('object_exists', [htmlspecialchars($source->name)]);
-        }
-        $this->repo->insert($source);
-        $this->fetchToken($source);
-        return $source;
-    }
-
-    /**
-     * Updates a source; attributes missing in $input keep their value.
-     *
-     * @throws UserError
-     */
-    public function update(int $id, array $input): Source
-    {
-        $this->requireEditor();
-        $current = $this->editable($id);
-        $source = clone $current;
-        $this->applyBasics($source, $input);
-        if ($source->name !== $current->name && $this->repo->nameExists($source->name, $source->createdBy, $id)) {
-            throw new UserError('object_exists', [htmlspecialchars($source->name)]);
-        }
-        $this->applyOAuth($source, $input);
-        // Targets only change when the request carries `scope`; client_credentials sources are always private
-        $writeTargets = isset($input['scope']) || $source->usesClientCredentials();
-        if ($writeTargets) {
-            $this->applyScope($source, $input['scope'] ?? $current->scope, $input['domains'] ?? [], $input['users'] ?? []);
-        }
-        $this->repo->update($source, $writeTargets, self::connection($current) !== self::connection($source));
-        $this->fetchToken($source);
-        return $source;
-    }
-
-    /**
-     * Deletes an unused source and returns it.
-     *
-     * @throws UserError access_denied, imapsync_source_in_use
-     */
-    public function delete(int $id): Source
-    {
-        $this->requireAcl();
-        $source = $this->editable($id);
-        // FK ON DELETE RESTRICT prevents removal while sync jobs reference it
-        if ($this->repo->isInUse($id)) {
-            throw new UserError('imapsync_source_in_use', [$source->name]);
-        }
-        $this->repo->delete($id);
-        return $source;
-    }
-
-    /**
-     * Fetches a new shared token for a client_credentials source now.
-     *
-     * @throws UserError access_denied, imapsync_source_refresh_unsupported, imapsync_source_token_refresh_failed
-     */
-    public function refreshToken(int $id): Source
-    {
-        $this->requireAcl();
-        $source = $this->editable($id);
-        if (!$source->usesClientCredentials()) {
-            throw new UserError('imapsync_source_refresh_unsupported');
-        }
-        $error = $this->oauth->refreshSourceToken($source);
-        if ($error !== null) {
-            throw new UserError('imapsync_source_token_refresh_failed', [$source->name . ' (' . $error . ')']);
-        }
-        return $source;
-    }
-
-    /** A visible, active authorization_code source the caller may sign in to (OAuth popup), else null */
-    public function findConnectable(int $id): ?Source
-    {
-        $source = $this->get($id);
-        return ($source !== null && $source->active === 1 && $source->usesAuthorizationCode()) ? $source : null;
-    }
-
-    /**
-     * Account rename (mailbox, admin, domain admin): sources, user targets and token owners follow.
-     * Leftover sources of an earlier account named $new are released first (see releaseSourcesOf()).
-     * Runs on the caller's PDO; a PDOException is not caught, so a caller transaction can roll back.
-     */
-    public function onAccountRenamed(string $old, string $new): void
-    {
-        if ($old === '' || $old === $new) {
-            return;
-        }
-        $this->releaseSourcesOf($new);
-        $this->repo->renameAccount($old, $new);
-        $this->tokens->renameOwner($old, $new);
-    }
-
-    /** Account deleted (after its jobs were deleted): sources released, tokens and user targets removed */
-    public function onAccountDeleted(string $old): void
-    {
-        if ($old === '') {
-            return;
-        }
-        $this->releaseSourcesOf($old);
-        $this->tokens->deleteByOwner($old);
-        $this->repo->deleteUserTarget($old);
-    }
-
-    private function canRead(): bool
-    {
-        return $this->context->hasAcl('syncjobs') || $this->context->isAdmin();
-    }
-
-    /** @throws AccessDenied when the caller cannot see or edit the source */
-    private function editable(int $id): Source
-    {
-        $source = $this->get($id);
-        if ($source === null || !$this->canEdit($source)) {
-            throw new AccessDenied();
-        }
-        return $source;
-    }
-
-    /**
-     * Sources of an account are deleted when unused, otherwise handed over to the admins
-     * (created_by ''), so a re-created account inherits nothing.
-     */
-    private function releaseSourcesOf(string $username): void
-    {
-        foreach ($this->repo->listCreatedBy($username) as $source) {
-            $id = (int)$source->id;
-            if (!$this->repo->isInUse($id)) {
-                // Explicit child deletes: the mailbox rename runs with FOREIGN_KEY_CHECKS = 0 (no cascade)
-                $this->repo->deleteWithChildren($id);
-                continue;
-            }
-            // Keep (created_by, name) unique when handing over
-            $name = $this->repo->nameExists($source->name, '') ? mb_substr($source->name, 0, 80) . ' (#' . $id . ')' : $source->name;
-            $this->repo->handOver($id, $name);
-        }
-    }
-
-    /**
-     * Name, description, connection and active from the request; missing attributes keep the value of $source.
-     * On edit, stored host/port/enc that do not change are not validated again, so migrated legacy sources
-     * stay editable (e.g. host 'imap_server.lan'). A host that only differs in case keeps the stored spelling.
-     *
-     * @throws UserError
-     */
-    private function applyBasics(Source $source, array $input): void
-    {
-        $stored = ($source->id !== null);
-        $name = isset($input['name']) ? Input::text($input['name']) : $source->name;
-        $description = isset($input['description']) ? Input::text($input['description']) : $source->description;
-        $host = $input['host'] ?? $input['host1'] ?? null;
-        $host = ($host === null) ? $source->host : Input::text($host);
-        if ($host !== null) {
-            $host = (strtolower($host) === strtolower($source->host)) ? $source->host : strtolower($host);
-        }
-        $port = $input['port'] ?? $input['port1'] ?? $source->port;
-        $port = is_scalar($port) ? (int)$port : 0;
-        $enc = $input['enc'] ?? $input['enc1'] ?? $source->enc;
-        $authType = $input['auth_type'] ?? $source->authType;
-
-        if ($name === null || $name === '' || mb_strlen($name) > 100) {
-            throw new UserError('imapsync_source_name_invalid');
-        }
-        if ($description === null || mb_strlen($description) > 255) {
-            throw new UserError('imapsync_source_description_invalid');
-        }
-        if (!($stored && $host === $source->host) && !$this->isValidHost($host)) {
-            throw new UserError('imapsync_source_host_invalid');
-        }
-        if (!($stored && $port === $source->port) && ($port < 1 || $port > 65535)) {
-            throw new UserError('imapsync_source_port_invalid');
-        }
-        if (!($stored && $enc === $source->enc) && !in_array($enc, self::ENCRYPTIONS, true)) {
-            throw new UserError('imapsync_source_enc_invalid');
-        }
-        if (!in_array($authType, self::AUTH_TYPES, true)) {
-            throw new UserError('imapsync_source_auth_invalid');
-        }
-        $source->name = $name;
-        $source->description = $description;
-        $source->host = $host;
-        $source->port = $port;
-        $source->enc = $enc;
-        $source->authType = $authType;
-        $source->active = isset($input['active']) ? (int)$input['active'] : $source->active;
-    }
-
-    private function isValidHost(?string $host): bool
-    {
-        return $host !== null && $host !== '' && strlen($host) <= 253
-            && (filter_var($host, FILTER_VALIDATE_IP) || filter_var($host, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME));
-    }
-
-    /** @throws UserError */
-    private function applyOAuth(Source $source, array $input): void
-    {
-        // XOAUTH2 sources are managed by admins and domain admins only
-        if ($source->isXOAuth2() && $this->context->isUser()) {
-            throw new AccessDenied();
-        }
-        $this->oauth->applySettings($source, $input);
-    }
-
-    /**
-     * Validates scope and targets for the caller and sets them; nothing is written here.
-     * Users' sources and client_credentials sources (one app-wide token that reaches every mailbox) are always private.
-     *
-     * @throws UserError access_denied, imapsync_source_scope_empty
-     */
-    private function applyScope(Source $source, mixed $scope, mixed $domains, mixed $users): void
-    {
-        if ($source->usesClientCredentials() || $this->context->isUser()) {
-            [$source->scope, $source->domains, $source->users] = ['user', [], []];
-            return;
-        }
-        // Only admins may create globally visible sources
-        if (!is_string($scope) || !in_array($scope, self::SCOPES, true) || ($scope === 'all' && !$this->context->isAdmin())) {
-            throw new AccessDenied();
-        }
-        $domains = ($scope === 'domain') ? $this->cleanList($domains) : [];
-        $users = ($scope === 'user') ? $this->cleanList($users) : [];
-        if ($scope === 'domain' && empty($domains)) {
-            throw new UserError('imapsync_source_scope_empty');
-        }
-        foreach ($domains as $domain) {
-            if (!$this->context->canAccessDomain($domain)) {
-                throw new AccessDenied();
-            }
-        }
-        foreach ($users as $user) {
-            if (!$this->context->canAccessMailbox($user)) {
-                throw new AccessDenied();
-            }
-        }
-        [$source->scope, $source->domains, $source->users] = [$scope, $domains, $users];
-    }
-
-    /** @return string[] trimmed, lowercase, unique, non-empty scalar values */
-    private function cleanList(mixed $list): array
-    {
-        $clean = [];
-        foreach ((array)$list as $value) {
-            if (is_scalar($value) && trim((string)$value) !== '') {
-                $clean[] = strtolower(trim((string)$value));
-            }
-        }
-        return array_values(array_unique($clean));
-    }
-
-    /** A saved, active client_credentials source gets its token now instead of at the next cron run (error in lastRefreshError) */
-    private function fetchToken(Source $source): void
-    {
-        if ($source->usesClientCredentials() && $source->active === 1) {
-            $source->lastRefreshError = $this->oauth->refreshSourceToken($source);
-        }
-    }
-
-    /** Tokens never follow a source to another OAuth client or IMAP server, so these values reset them */
-    private static function connection(Source $source): array
-    {
-        return [(string)$source->authType, $source->host, $source->port, (string)$source->enc, $source->oauthFlow,
-            (string)$source->tokenEndpoint, (string)$source->clientId];
-    }
-}

data/web/inc/lib/mailcow/MailboxGroup/Controller/MailboxGroupApiController.php

@@ -1,173 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailboxGroup\Controller;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\MailboxGroup\Model\Group;
-use Mailcow\MailboxGroup\Service\AccountHooks;
-use Mailcow\MailboxGroup\Service\GroupService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php and edit.php (routes `mailbox-group`), plus the hook adapters the legacy
- * code calls outside a transaction (a database error becomes a `mysql_error` message).
- * Messages are logged as ['mailbox_group', action, 'group', data].
- */
-class MailboxGroupApiController
-{
-    public function __construct(private GroupService $groups, private AccountHooks $hooks, private TemplateService $templates,
-        private ApiResponse $response)
-    {
-    }
-
-    /** POST add/mailbox-group */
-    public function addGroup(array $attr): bool
-    {
-        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
-            $group = $this->groups->create($attr);
-            $this->warnings($group->warnings, $log);
-            return $this->response->success(['group_added', htmlspecialchars($group->name)], $log);
-        });
-    }
-
-    /** POST edit/mailbox-group: one message per id */
-    public function editGroups(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
-            $this->groups->requireEditor();
-            return $this->batch((array)($data['id'] ?? []), $log, function (int $id) use ($attr, $log): array {
-                $group = $this->groups->update($id, $attr);
-                $this->warnings($group->warnings, $log);
-                return ['group_modified', htmlspecialchars($group->name)];
-            });
-        });
-    }
-
-    /** POST delete/mailbox-group: one message per id */
-    public function deleteGroups(array $items): bool
-    {
-        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
-            $this->groups->requireEditor();
-            return $this->batch($items, $log, function (int $id) use ($log): array {
-                $group = $this->groups->delete($id);
-                $this->warnings($group->warnings, $log);
-                return ['group_removed', htmlspecialchars($group->name)];
-            });
-        });
-    }
-
-    /** GET mailbox-group/all[/<domain,domain>] */
-    public function getGroups(?string $domains = null): array
-    {
-        $list = array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
-        return array_values(array_filter(array_map(fn(int $id): array => $this->getGroup($id), $this->groups->visibleIds($list))));
-    }
-
-    /** GET mailbox-group/<id>, also edit.php */
-    public function getGroup(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $group = ($id === null) ? null : $this->groups->get($id);
-        return ($group === null) ? [] : $this->groups->toArray($group);
-    }
-
-    /** GET mailbox-group/member/<username>: groups of a mailbox (mailbox edit page, user page) */
-    public function getMemberships(string $username): array
-    {
-        return $this->groups->membershipsOf($username);
-    }
-
-    public function mailboxDeleted(string $username): void
-    {
-        $this->hook('mailbox_deleted', $username, fn() => $this->hooks->onMailboxDeleted($username));
-    }
-
-    public function domainDeleted(string $domain): void
-    {
-        $this->hook('domain_deleted', $domain, fn() => $this->hooks->onDomainDeleted($domain));
-    }
-
-    public function addressDeleted(string $address): void
-    {
-        $this->hook('address_deleted', $address, fn() => $this->hooks->onAddressDeleted($address));
-    }
-
-    public function addressRenamed(string $old, string $new): void
-    {
-        $this->hook('address_renamed', [$old, $new], fn() => $this->hooks->onAddressRenamed($old, $new));
-    }
-
-    /** A mailbox template was edited: apply it where it is the effective group template */
-    public function templateChanged(int $templateId): void
-    {
-        $this->hook('template_changed', $templateId, function () use ($templateId): void {
-            $this->warnings($this->templates->onTemplateChanged($templateId), ['mailbox_group', 'edit', 'template', $templateId]);
-        });
-    }
-
-    private function warnings(array $warnings, array $log): void
-    {
-        foreach ($warnings as $msg) {
-            $this->response->warning($msg, $log);
-        }
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, mixed $data, callable $work): bool
-    {
-        $log = ['mailbox_group', $action, 'group', $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    private function hook(string $action, mixed $data, callable $work): void
-    {
-        try {
-            $work();
-        } catch (PDOException $e) {
-            $this->response->danger(['mysql_error', $e->getMessage()], ['mailbox_group', $action, 'hook', $data]);
-        }
-        $this->response->flush();
-    }
-
-    /** One message per id: the success message returned by $work(id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id)), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/MailboxGroup/Legacy/LegacyMailbox.php

@@ -1,59 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailboxGroup\Legacy;
-
-/**
- * Bridge to the procedural mailbox code. The global calls sit in protected methods so tests can replace them.
- */
-class LegacyMailbox
-{
-    /**
-     * Applies a mailbox template like the IdP code does: as system caller, with the current name, so only the
-     * template attributes change.
-     *
-     * @return ?string null on success, else the message key of the legacy error
-     */
-    public function applyTemplate(string $username, string $name, string $template): ?string
-    {
-        return $this->asSystem(fn(): mixed => $this->callTemplate($username, $name, $template));
-    }
-
-    /**
-     * Runs a legacy call as system caller (access_all_exception); the caller's session messages are kept.
-     *
-     * @return ?string null unless the call returned false, else the message key of the legacy error
-     */
-    protected function asSystem(callable $call): ?string
-    {
-        $hasReturn = array_key_exists('return', $_SESSION ?? []);
-        $return = $_SESSION['return'] ?? null;
-        $accessAll = $_SESSION['access_all_exception'] ?? null;
-        $_SESSION['return'] = [];
-        $_SESSION['access_all_exception'] = '1';
-        try {
-            if ($call() !== false) {
-                return null;
-            }
-            foreach ($_SESSION['return'] as $message) {
-                if (($message['type'] ?? '') === 'danger') {
-                    return (string)(is_array($message['msg']) ? $message['msg'][0] : $message['msg']);
-                }
-            }
-            return 'generic_server_error';
-        } finally {
-            if ($hasReturn) {
-                $_SESSION['return'] = $return;
-            } else {
-                unset($_SESSION['return']);
-            }
-            $_SESSION['access_all_exception'] = $accessAll ?? '0';
-        }
-    }
-
-    protected function callTemplate(string $username, string $name, string $template): mixed
-    {
-        return mailbox('edit', 'mailbox_from_template', ['username' => $username, 'name' => $name, 'template' => $template]);
-    }
-}

data/web/inc/lib/mailcow/MailboxGroup/MailboxGroupModule.php

@@ -1,62 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailboxGroup;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Dovecot\DovecotModule;
-use Mailcow\MailboxGroup\Controller\MailboxGroupApiController;
-use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
-use Mailcow\MailboxGroup\Repository\GroupRepository;
-use Mailcow\MailboxGroup\Repository\LookupRepository;
-use Mailcow\MailboxGroup\Service\AccountHooks;
-use Mailcow\MailboxGroup\Service\ChangeEffects;
-use Mailcow\MailboxGroup\Service\GroupService;
-use Mailcow\MailboxGroup\Service\IdpSyncService;
-use Mailcow\MailboxGroup\Service\SharingService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use Mailcow\Sogo\SogoModule;
-use PDO;
-
-/**
- * Composition root of the mailbox group module.
- *
- *   process_add_return(MailboxGroupModule::fromGlobals()->api->addGroup($attr));
- *   MailboxGroupModule::fromGlobals(Context::system())->templates->templateForLogin($username, $legacyTemplate);
- *   MailboxGroupModule::create($pdo, Context::system())->sharing->reconcile();  // cron
- */
-class MailboxGroupModule
-{
-    public function __construct(
-        public readonly GroupService $groups,
-        public readonly TemplateService $templates,
-        public readonly AccountHooks $hooks,
-        public readonly IdpSyncService $idp,
-        public readonly SharingService $sharing,
-        public readonly MailboxGroupApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context): self
-    {
-        $repo = new GroupRepository($pdo);
-        $lookup = new LookupRepository($pdo);
-        $dovecot = DovecotModule::create()->dovecot;
-        $sogo = SogoModule::create()->sogo;
-        $templates = new TemplateService($lookup, new LegacyMailbox());
-        $effects = new ChangeEffects($templates, $sogo, $repo, $dovecot);
-        $sharing = new SharingService($repo, $dovecot, $sogo, $effects);
-        $groups = new GroupService($repo, $lookup, $templates, $effects, $sharing, $context);
-        $hooks = new AccountHooks($repo, $effects);
-        return new self($groups, $templates, $hooks, new IdpSyncService($repo, $lookup, $effects), $sharing,
-            new MailboxGroupApiController($groups, $hooks, $templates, new ApiResponse()));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION; $context overrides the session caller */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
-    }
-}

data/web/inc/lib/mailcow/MailboxGroup/Model/Group.php

@@ -1,108 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailboxGroup\Model;
-
-/**
- * A mailbox group (table `mailbox_group`) with its members and send-as grants.
- *
- * The group tables are the source of truth. GroupRepository::rebuild() derives the rows other services read:
- * the alias row of the distribution address and the sender_acl rows of the grants, both tagged with `group_id`.
- * Not to be confused with SOGo resources of kind 'group' (`mailbox.kind`).
- */
-class Group
-{
-    public const SOURCE_MANUAL = 'manual';
-    public const SOURCE_IDP = 'idp';
-
-    public ?int $id = null;
-    public string $domain = '';
-    public string $name = '';
-    public string $description = '';
-    /** Distribution address, null = group only used for grants and templates */
-    public ?string $address = null;
-    /** Members may send as the address */
-    public int $senderAllowed = 0;
-    public int $sogoVisible = 0;
-    /** Only senders of the domain may write to the address */
-    public int $internal = 0;
-    public ?int $templateId = null;
-    /** Template name, read only */
-    public ?string $template = null;
-    public int $priority = 0;
-    /** IdP group id, set = IdP memberships are synced */
-    public ?string $idpRef = null;
-    public int $active = 1;
-    public ?string $created = null;
-    public ?string $modified = null;
-    /** @var array<string, string> username => source */
-    public array $members = [];
-    /** @var string[] addresses or @domain */
-    public array $sendAs = [];
-    /** @var array[] {owner, rights, subscribe, state, last_error}: members may open these mailboxes (ACL group=mcg_<id>) */
-    public array $imapGrants = [];
-    /** @var array[] {owner, type, folder, access, subscribe, state, last_error}: SOGo folders shared with the members */
-    public array $sogoGrants = [];
-    /** @var array[] warnings of the last write (msg values), not stored */
-    public array $warnings = [];
-
-    /** Builds a group from a `mailbox_group` row (`template_name` optional) */
-    public static function fromRow(array $row): self
-    {
-        $group = new self();
-        $group->id = isset($row['id']) ? (int)$row['id'] : null;
-        $group->domain = (string)($row['domain'] ?? '');
-        $group->name = (string)($row['name'] ?? '');
-        $group->description = (string)($row['description'] ?? '');
-        $group->address = isset($row['address']) ? (string)$row['address'] : null;
-        $group->senderAllowed = (int)($row['sender_allowed'] ?? 0);
-        $group->sogoVisible = (int)($row['sogo_visible'] ?? 0);
-        $group->internal = (int)($row['internal'] ?? 0);
-        $group->templateId = isset($row['template_id']) ? (int)$row['template_id'] : null;
-        $group->template = isset($row['template_name']) ? (string)$row['template_name'] : null;
-        $group->priority = (int)($row['priority'] ?? 0);
-        $group->idpRef = isset($row['idp_ref']) ? (string)$row['idp_ref'] : null;
-        $group->active = (int)($row['active'] ?? 1);
-        $group->created = isset($row['created']) ? (string)$row['created'] : null;
-        $group->modified = isset($row['modified']) ? (string)$row['modified'] : null;
-        return $group;
-    }
-
-    /** @return string[] */
-    public function usernames(): array
-    {
-        return array_keys($this->members);
-    }
-
-    /** API representation (GET mailbox-group) */
-    public function toArray(bool $canEdit): array
-    {
-        $members = [];
-        foreach ($this->members as $username => $source) {
-            $members[] = ['username' => (string)$username, 'source' => $source];
-        }
-        return [
-            'id' => $this->id,
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'address' => $this->address,
-            'sender_allowed' => $this->senderAllowed,
-            'sogo_visible' => $this->sogoVisible,
-            'internal' => $this->internal,
-            'template_id' => $this->templateId,
-            'template' => $this->template,
-            'priority' => $this->priority,
-            'idp_ref' => $this->idpRef,
-            'members' => $members,
-            'send_as' => $this->sendAs,
-            'imap_grants' => $this->imapGrants,
-            'sogo_grants' => $this->sogoGrants,
-            'active' => $this->active,
-            'created' => $this->created,
-            'modified' => $this->modified,
-            'can_edit' => $canEdit,
-        ];
-    }
-}