[Rspamd] Update to 4.2.3

[Web] Update composer libraries

data/Dockerfiles/acme/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -412,6 +412,63 @@ class DockerApi:
 
         sogo_return = container.exec_run(["/bin/bash", "-c", f"sogo-tool rename-user '{old_username}' '{new_username}'"], user='sogo')
         return self.exec_run_handler('generic', sogo_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: subscribe
+  # Subscribes users to a folder of the owner, or unsubscribes them; values are passed as environment and
+  # arguments, never through the shell
+  def container_post__exec__sogo__subscribe(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      folder = str(request_json.get('folder', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or not re.match(r'^(Calendar|Contacts)/[A-Za-z0-9._-]+$', folder):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner, folder and users required' }, indent=4), media_type="application/json")
+      script = 'for user in "$@"; do sogo-tool manage-acl "$SOGO_ACTION" "$SOGO_OWNER" "$SOGO_FOLDER" "$user" || exit 1; done'
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe"] + users, user='sogo',
+        environment={ 'SOGO_ACTION': 'unsubscribe' if request_json.get('unsubscribe') else 'subscribe', 'SOGO_OWNER': owner, 'SOGO_FOLDER': folder })
+      return self.exec_run_handler('generic', subscribe_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: init_user
+  # Creates the personal folders of users that never logged in to SOGo (e.g. shared mailboxes)
+  def container_post__exec__sogo__init_user(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not users:
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
+      init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
+      return self.exec_run_handler('generic', init_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
+  # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
+  def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or any(c in owner for c in '/*%'):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner and users required' }, indent=4), media_type="application/json")
+      if request_json.get('unsubscribe'):
+        # Without access the user no longer sees the folders, their names come from the owner
+        script = ('set -o pipefail; for user in "$@"; do doveadm mailbox list -u "$SUB_OWNER" | while IFS= read -r folder; do '
+          'case "$folder" in Shared|Shared/*) continue;; esac; doveadm mailbox unsubscribe -u "$user" "Shared/$SUB_OWNER/$folder" || exit 1; '
+          'done || exit 1; done')
+      else:
+        # No visible folder means no access yet (ACL, auth cache): fail, so the subscription is retried
+        script = ('for user in "$@"; do folders=$(doveadm mailbox list -u "$user" "Shared/$SUB_OWNER/*") && [ -n "$folders" ] || exit 1; '
+          'while IFS= read -r folder; do doveadm mailbox subscribe -u "$user" "$folder" || exit 1; done <<< "$folders"; done')
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe_shared"] + users, environment={ 'SUB_OWNER': owner })
+      return self.exec_run_handler('generic', subscribe_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: get_acl
   def container_post__exec__doveadm__get_acl(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -439,7 +496,8 @@ class DockerApi:
           if len(acls) >= 2:
             for acl in acls[1:]:
               user_id, rights = acl.split(maxsplit=1)
-              user_id = user_id.split('=')[1]
+              # user=<name> as plain name, other identifiers (group=...) keep their type
+              user_id = user_id.split('=', 1)[1] if user_id.startswith('user=') else user_id
               mailbox_seen.append(mailbox)
               formatted_acls.append({ 'user': id, 'id': user_id, 'mailbox': mailbox, 'rights': rights.split() })
         elif "Shared" in shared_folder and "/" in shared_folder:
@@ -476,7 +534,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' 'user={id}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -510,8 +568,50 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' 'user={id}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
+  # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
+  # environment, never through the shell
+  def container_post__exec__doveadm__set_acl_all(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      user = request_json.get('user', '')
+      id = self.acl_identifier(request_json.get('id', ''))
+      rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
+      if not user or id in ('user=', 'group='):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
+      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
+      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
+        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
+      acl_return = container.exec_run(["/bin/bash", "-c", script],
+        environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
+      return self.exec_run_handler('generic', acl_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: auth_cache_flush
+  # Drops cached userdb results (e.g. acl_groups) of the given users, all users if none are given
+  def container_post__exec__doveadm__auth_cache_flush(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
+      return self.exec_run_handler('generic', flush_return)
+
+  # Rights doveadm acl set accepts from the API
+  acl_rights = ["admin", "create", "delete", "expunge", "insert", "lookup", "post", "read", "write", "write-deleted", "write-seen"]
+
+  # ACL identifier for doveadm: "user=<name>" for a plain name, "user=..." / "group=..." as given
+  def acl_identifier(self, id):
+    if id.startswith('user=') or id.startswith('group='):
+      return id
+    return 'user=' + id
 
 
   # Collect host stats

data/Dockerfiles/dovecot/Dockerfile

@@ -1,12 +1,48 @@
-FROM alpine:3.21
+# Alpine >= 3.22 only ships Dovecot 2.4, Dovecot 2.3 is built from the Alpine 3.21 packaging
+FROM alpine:3.24 AS builder
+
+# Last commit of the aports 3.21-stable branch
+ARG APORTS_COMMIT=55126a11a4b9dcd14de3068e3348e9951d144ca6
+
+RUN apk add --no-cache alpine-sdk git \
+  && git init -q /aports \
+  && cd /aports \
+  && git remote add origin https://github.com/alpinelinux/aports.git \
+  && git sparse-checkout set main/dovecot community/dovecot-fts-flatcurve \
+  && git fetch -q --depth 1 --filter=blob:none origin ${APORTS_COMMIT} \
+  && git checkout -q FETCH_HEAD \
+  && abuild-keygen -a -n \
+  && cp /root/.abuild/*.rsa.pub /etc/apk/keys/ \
+  && apk update \
+  && cd /aports/main/dovecot \
+  # newer gettext does not overwrite the shipped config.rpath without -f
+  && sed -i 's/autoreconf -vi/autoreconf -fvi/' APKBUILD \
+  && abuild -F -r \
+  # flatcurve must be built against the Dovecot 2.3 headers
+  && apk add --no-cache /root/packages/main/*/dovecot-2*.apk /root/packages/main/*/dovecot-dev-*.apk \
+  && cd /aports/community/dovecot-fts-flatcurve \
+  && abuild -F -r \
+  && mkdir /packages \
+  && cp /root/.abuild/*.rsa.pub /packages/ \
+  && for pkg in dovecot dovecot-dev dovecot-lmtpd dovecot-lua dovecot-ldap dovecot-mysql dovecot-sql \
+    dovecot-submissiond dovecot-pigeonhole-plugin dovecot-pop3d dovecot-fts-flatcurve; do \
+    cp /root/packages/*/*/${pkg}-[0-9]*.apk /packages/; \
+  done
+
+
+FROM alpine:3.24
 
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.19
+ARG GOSU_VERSION=1.20
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
+# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, this covers Perl DBI, client.cnf covers Dovecot
+ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
+
+COPY --from=builder /packages /tmp/packages
 
 # Add groups and users before installing Dovecot to not break compatibility
 RUN addgroup -g 5000 vmail \
@@ -18,6 +54,7 @@ RUN addgroup -g 5000 vmail \
   && adduser -D -u 5000 -G vmail -h /var/vmail vmail \
   && adduser -D -G dovecot -u 401 -h /dev/null -s /sbin/nologin dovecot \
   && adduser -D -G dovenull -u 402 -h /dev/null -s /sbin/nologin dovenull \
+  && cp /tmp/packages/*.rsa.pub /etc/apk/keys/ \
   && apk add --no-cache --update \
   bash \
   bind-tools \
@@ -98,22 +135,15 @@ RUN addgroup -g 5000 vmail \
   supervisor \
   tzdata \
   wget \
-  dovecot \
-  dovecot-dev \
-  dovecot-lmtpd \
-  dovecot-lua \
-  dovecot-ldap \
-  dovecot-mysql \
-  dovecot-sql \
-  dovecot-submissiond \
-  dovecot-pigeonhole-plugin \
-  dovecot-pop3d \
-  dovecot-fts-flatcurve \
+  /tmp/packages/*.apk \
+  && rm -rf /tmp/packages \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
   && chmod +x /usr/local/bin/gosu \
   && gosu nobody true
 
+# Dovecot's MySQL driver reads [client] from /etc/my.cnf, Connector/C ignores the !includedir there
+COPY client.cnf /etc/my.cnf
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf

data/Dockerfiles/dovecot/client.cnf

@@ -0,0 +1,3 @@
+[client]
+ssl = false
+ssl-verify-server-cert = false

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -127,7 +127,8 @@ cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 # Autogenerated by mailcow
 driver = mysql
 connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
-user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
+# acl_groups: active mailbox groups of the user, IMAP grants are ACLs on group=mcg_<id>
+user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule, (SELECT GROUP_CONCAT(CONCAT('mcg_', gm.group_id)) FROM mailbox_group_member gm JOIN mailbox_group g ON g.id = gm.group_id AND g.active = 1 WHERE gm.username = mailbox.username) AS acl_groups FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
 iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
 EOF
 
@@ -304,6 +305,15 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 # Envsubst maildir_gc
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+done
+echo "DB schema is ${DBV_NOW}"
+
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do
   if [[ ! -z $(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"${DBNAME}\" AND TABLE_NAME = 'versions'") ]]; then

data/Dockerfiles/netfilter/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/netfilter/docker-entrypoint.sh

@@ -17,8 +17,8 @@ if [ $nftables_found -gt $iptables_found ]; then
 fi
 
 if [ $nftables_found -eq 0 ] && [ $nftables_found -eq $iptables_found ]; then
-  nftables_lines=$(nft list ruleset | wc -l)
-  iptables_lines=$(iptables-save | wc -l)
+  nftables_lines=$(nft list ruleset 2>/dev/null | wc -l)
+  iptables_lines=$(iptables-save 2>/dev/null | wc -l)
   if [ $nftables_lines -gt $iptables_lines ]; then
     backend=nftables
   else

data/Dockerfiles/netfilter/main.py

@@ -1,548 +1,57 @@
-#!/usr/bin/env python3
-
-DEBUG = False
-
-import re
-import os
-import sys
-import time
-import atexit
-import signal
-import ipaddress
-from collections import Counter
-from random import randint
-from threading import Thread
-from threading import Lock
-import redis
-import json
-import dns.resolver
-import dns.exception
-import uuid
-from modules.Logger import Logger
-from modules.IPTables import IPTables
-from modules.NFTables import NFTables
-
-def logdebug(msg):
-  if DEBUG:
-    logger.logInfo("DEBUG: %s" % msg)
-
-# Globals
-WHITELIST = []
-BLACKLIST = []
-bans = {}
-quit_now = False
-exit_code = 0
-lock = Lock()
-chain_name = "MAILCOW"
-r = None
-pubsub = None
-clear_before_quit = False
-
-def refreshF2boptions():
-  global f2boptions
-  global quit_now
-  global exit_code
-  f2boptions = {}
-
-  if not r.get('F2B_OPTIONS'):
-    f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
-    f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
-    f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
-    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
-    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
-    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
-    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
-  else:
-    try:
-      f2boptions = json.loads(r.get('F2B_OPTIONS'))
-    except ValueError as e:
-      logger.logCrit(
-        'Error loading F2B options: F2B_OPTIONS is not json. Exception: %s' % e)
-      quit_now = True
-      exit_code = 2
-
-  verifyF2boptions(f2boptions)
-  r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
-
-def verifyF2boptions(f2boptions):
-  verifyF2boption(f2boptions, 'ban_time', 1800)
-  verifyF2boption(f2boptions, 'max_ban_time', 10000)
-  verifyF2boption(f2boptions, 'ban_time_increment', True)
-  verifyF2boption(f2boptions, 'max_attempts', 10)
-  verifyF2boption(f2boptions, 'retry_window', 600)
-  verifyF2boption(f2boptions, 'netban_ipv4', 32)
-  verifyF2boption(f2boptions, 'netban_ipv6', 128)
-  verifyF2boption(f2boptions, 'banlist_id', str(uuid.uuid4()))
-  verifyF2boption(f2boptions, 'manage_external', 0)
-
-def verifyF2boption(f2boptions, f2boption, f2bdefault):
-  f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
-
-def refreshF2bregex():
-  global f2bregex
-  global quit_now
-  global exit_code
-  if not r.get('F2B_REGEX'):
-    f2bregex = {}
-    f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-    f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
-    f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
-    f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
-    f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
-    f2bregex[6] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[7] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[8] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-    f2bregex[9] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
-    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
-  else:
-    try:
-      f2bregex = {}
-      f2bregex = json.loads(r.get('F2B_REGEX'))
-    except ValueError:
-      logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
-      quit_now = True
-      exit_code = 2
-
-def get_ip(address):
-  ip = ipaddress.ip_address(address)
-  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
-    ip = ip.ipv4_mapped
-  if ip.is_private or ip.is_loopback:
-    return False
-
-  return ip
-
-def ban(address):
-  global f2boptions
-  global lock
-  logdebug("ban() called with address=%s" % address)
-  refreshF2boptions()
-  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-  RETRY_WINDOW = int(f2boptions['retry_window'])
-  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
-  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
-
-  ip = get_ip(address)
-  if not ip:
-    logdebug("No valid IP -- skipping ban()")
-    return
-  address = str(ip)
-  self_network = ipaddress.ip_network(address)
-
-  with lock:
-    temp_whitelist = set(WHITELIST)
-    logdebug("Checking if %s overlaps with any WHITELIST entries" % self_network)
-    if temp_whitelist:
-      for wl_key in temp_whitelist:
-        wl_net = ipaddress.ip_network(wl_key, False)
-        logdebug("Checking overlap between %s and %s" % (self_network, wl_net))
-        if wl_net.overlaps(self_network):
-          logger.logInfo(
-            'Address %s is allowlisted by rule %s' % (self_network, wl_net))
-          return
-
-  net = ipaddress.ip_network(
-    (address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
-  net = str(net)
-  logdebug("Ban net: %s" % net)
-
-  if not net in bans:
-    bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
-    logdebug("Initing new ban counter for %s" % net)
-
-  current_attempt = time.time()
-  logdebug("Current attempt ts=%s, previous: %s, retry_window: %s" %
-           (current_attempt, bans[net]['last_attempt'], RETRY_WINDOW))
-  if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
-    bans[net]['attempts'] = 0
-    logdebug("Ban counter for %s reset as window expired" % net)
-
-  bans[net]['attempts'] += 1
-  bans[net]['last_attempt'] = current_attempt
-  logdebug("%s attempts now %d" % (net, bans[net]['attempts']))
-
-  if bans[net]['attempts'] >= MAX_ATTEMPTS:
-    cur_time = int(round(time.time()))
-    NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
-    logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
-    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
-      with lock:
-        logdebug("Calling tables.banIPv4(%s)" % net)
-        tables.banIPv4(net)
-    elif int(f2boptions['manage_external']) != 1:
-      with lock:
-        logdebug("Calling tables.banIPv6(%s)" % net)
-        tables.banIPv6(net)
-
-    logdebug("Updating F2B_ACTIVE_BANS[%s]=%d" %
-              (net, cur_time + NET_BAN_TIME))
-    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
-  else:
-    logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
-      MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
-
-def unban(net):
-  global lock
-  logdebug("Calling unban() with net=%s" % net)
-  if not net in bans:
-    logger.logInfo(
-      '%s is not banned, skipping unban and deleting from queue (if any)' % net)
-    r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-    return
-  logger.logInfo('Unbanning %s' % net)
-  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
-    with lock:
-      logdebug("Calling tables.unbanIPv4(%s)" % net)
-      tables.unbanIPv4(net)
-  else:
-    with lock:
-      logdebug("Calling tables.unbanIPv6(%s)" % net)
-      tables.unbanIPv6(net)
-  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
-  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-  if net in bans:
-    logdebug("Unban for %s, setting attempts=0, ban_counter+=1" % net)
-    bans[net]['attempts'] = 0
-    bans[net]['ban_counter'] += 1
-
-def permBan(net, unban=False):
-  global f2boptions
-  global lock
-
-  is_unbanned = False
-  is_banned = False
-  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv4(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv4(net)
-  else:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv6(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv6(net)
-
-
-  if is_unbanned:
-    r.hdel('F2B_PERM_BANS', '%s' % net)
-    logger.logCrit('Removed host/network %s from denylist' % net)
-  elif is_banned:
-    r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
-    logger.logCrit('Added host/network %s to denylist' % net)
-
-def clear():
-  global lock
-  logger.logInfo('Clearing all bans')
-  for net in bans.copy():
-    logdebug("Unbanning net: %s" % net)
-    unban(net)
-  with lock:
-    logdebug("Clearing IPv4/IPv6 table")
-    tables.clearIPv4Table()
-    tables.clearIPv6Table()
-    try:
-      if r is not None:
-        r.delete('F2B_ACTIVE_BANS')
-        r.delete('F2B_PERM_BANS')
-    except Exception as ex:
-      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
-
-def watch():
-  global pubsub
-  global quit_now
-  global exit_code
-
-  logger.logInfo('Watching Redis channel F2B_CHANNEL')
-  pubsub.subscribe('F2B_CHANNEL')
-
-  while not quit_now:
-    try:
-      for item in pubsub.listen():
-        refreshF2bregex()
-        for rule_id, rule_regex in f2bregex.items():
-          if item['data'] and item['type'] == 'message':
-            try:
-              result = re.search(rule_regex, item['data'])
-            except re.error:
-              result = False
-            if result:
-              addr = result.group(1)
-              ip = ipaddress.ip_address(addr)
-              if ip.is_private or ip.is_loopback:
-                continue
-              logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
-              ban(addr)
-    except Exception as ex:
-      logger.logWarn('Error reading log line from pubsub: %s' % ex)
-      pubsub = None
-      quit_now = True
-      exit_code = 2
-
-def snat4(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
-
-def snat6(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
-
-def autopurge():
-  global f2boptions
-  logdebug("autopurge thread started")
-  while not quit_now:
-    logdebug("autopurge tick")
-    time.sleep(10)
-    refreshF2boptions()
-    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
-    logdebug("QUEUE_UNBAN: %s" % QUEUE_UNBAN)
-    if QUEUE_UNBAN:
-      for net in QUEUE_UNBAN:
-        logdebug("Autopurge: unbanning queued net: %s" % net)
-        unban(str(net))
-    # Only check expiry for actively banned IPs:
-    active_bans = r.hgetall('F2B_ACTIVE_BANS')
-    now = time.time()
-    for net_str, expire_str in active_bans.items():
-      logdebug("Checking ban expiry for (actively banned): %s" % net_str)
-      # Defensive: always process if timer missing or expired
-      try:
-        expire = float(expire_str)
-      except Exception:
-        logdebug("Invalid expire time for %s; unbanning" % net_str)
-        unban(net_str)
-        continue
-      time_left = expire - now
-      logdebug("Time left for %s: %.1f seconds" % (net_str, time_left))
-      if time_left <= 0:
-        logdebug("Ban expired for %s" % net_str)
-        unban(net_str)
-
-def mailcowChainOrder():
-  global lock
-  global quit_now
-  global exit_code
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      quit_now, exit_code = tables.checkIPv4ChainOrder()
-      if quit_now: return
-      quit_now, exit_code = tables.checkIPv6ChainOrder()
-
-def calcNetBanTime(ban_counter):
-  global f2boptions
-
-  BAN_TIME = int(f2boptions['ban_time'])
-  MAX_BAN_TIME = int(f2boptions['max_ban_time'])
-  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
-  NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
-  NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
-  return NET_BAN_TIME
-
-def isIpNetwork(address):
-  try:
-    ipaddress.ip_network(address, False)
-  except ValueError:
-    return False
-  return True
-
-def genNetworkList(list):
-  resolver = dns.resolver.Resolver()
-  hostnames = []
-  networks = []
-  for key in list:
-    if isIpNetwork(key):
-      networks.append(key)
-    else:
-      hostnames.append(key)
-  for hostname in hostnames:
-    hostname_ips = []
-    for rdtype in ['A', 'AAAA']:
-      try:
-        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
-      except dns.exception.Timeout:
-        logger.logInfo('Hostname %s timedout on resolve' % hostname)
-        break
-      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-        continue
-      except dns.exception.DNSException as dnsexception:
-        logger.logInfo('%s' % dnsexception)
-        continue
-      for rdata in answer:
-        hostname_ips.append(rdata.to_text())
-    networks.extend(hostname_ips)
-  return set(networks)
-
-def whitelistUpdate():
-  global lock
-  global quit_now
-  global WHITELIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_WHITELIST')
-    new_whitelist = []
-    if list:
-      new_whitelist = genNetworkList(list)
-    with lock:
-      if Counter(new_whitelist) != Counter(WHITELIST):
-        WHITELIST = new_whitelist
-        logger.logInfo('Allowlist was changed, it has %s entries' % len(WHITELIST))
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def blacklistUpdate():
-  global quit_now
-  global BLACKLIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_BLACKLIST')
-    new_blacklist = []
-    if list:
-      new_blacklist = genNetworkList(list)
-    if Counter(new_blacklist) != Counter(BLACKLIST):
-      addban = set(new_blacklist).difference(BLACKLIST)
-      delban = set(BLACKLIST).difference(new_blacklist)
-      BLACKLIST = new_blacklist
-      logger.logInfo('Denylist was changed, it has %s entries' % len(BLACKLIST))
-      if addban:
-        for net in addban:
-          permBan(net=net)
-      if delban:
-        for net in delban:
-          permBan(net=net, unban=True)
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def sigterm_quit(signum, frame):
-  global clear_before_quit
-  logdebug("SIGTERM received, setting clear_before_quit to True and exiting")
-  clear_before_quit = True
-  sys.exit(exit_code)
-
-def before_quit():
-  logdebug("before_quit called, clear_before_quit=%s" % clear_before_quit)
-  if clear_before_quit:
-    clear()
-  if pubsub is not None:
-    pubsub.unsubscribe()
-
-if __name__ == '__main__':
-  logger = Logger()
-  logdebug("Sys.argv: %s" % sys.argv)
-  atexit.register(before_quit)
-  signal.signal(signal.SIGTERM, sigterm_quit)
-
-  backend = sys.argv[1]
-  logdebug("Backend: %s" % backend)
-  if backend == "nftables":
-    logger.logInfo('Using NFTables backend')
-    tables = NFTables(chain_name, logger)
-  else:
-    logger.logInfo('Using IPTables backend')
-    logger.logWarn(
-        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
-        "Please switch to nftables on your host to ensure complete compatibility."
-    )
-    time.sleep(5)
-    tables = IPTables(chain_name, logger)
-
-  clear()
-  logger.logInfo("Initializing mailcow netfilter chain")
-  tables.initChainIPv4()
-  tables.initChainIPv6()
-
-  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE", "").lower() in ("y", "yes"):
-    logger.logInfo(f"Skipping {chain_name} isolation")
-  else:
-    logger.logInfo(f"Setting {chain_name} isolation")
-    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
-
-  # connect to redis
-  while True:
-    try:
-      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
-      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
-      logdebug(
-        "Connecting redis (SLAVEOF_IP:%s, PORT:%s)" % (redis_slaveof_ip, redis_slaveof_port))
-      if "".__eq__(redis_slaveof_ip):
-        r = redis.StrictRedis(
-          host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
-      else:
-        r = redis.StrictRedis(
-          host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0, password=os.environ['REDISPASS'])
-      r.ping()
-      pubsub = r.pubsub()
-    except Exception as ex:
-      logdebug(
-        'Redis connection failed: %s - trying again in 3 seconds' % (ex))
-      time.sleep(3)
-    else:
-      break
-  logger.set_redis(r)
-  logdebug("Redis connection established, setting up F2B keys")
-
-  if r.exists('F2B_LOG'):
-    logdebug("Renaming F2B_LOG to NETFILTER_LOG")
-    r.rename('F2B_LOG', 'NETFILTER_LOG')
-  r.delete('F2B_ACTIVE_BANS')
-  r.delete('F2B_PERM_BANS')
-
-  refreshF2boptions()
-
-  watch_thread = Thread(target=watch)
-  watch_thread.daemon = True
-  watch_thread.start()
-
-  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv4Address:
-        snat4_thread = Thread(target=snat4, args=(snat_ip,))
-        snat4_thread.daemon = True
-        snat4_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
-
-  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT6_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv6Address:
-        snat6_thread = Thread(target=snat6,args=(snat_ip,))
-        snat6_thread.daemon = True
-        snat6_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
-
-  autopurge_thread = Thread(target=autopurge)
-  autopurge_thread.daemon = True
-  autopurge_thread.start()
-
-  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
-  mailcowchainwatch_thread.daemon = True
-  mailcowchainwatch_thread.start()
-
-  blacklistupdate_thread = Thread(target=blacklistUpdate)
-  blacklistupdate_thread.daemon = True
-  blacklistupdate_thread.start()
-
-  whitelistupdate_thread = Thread(target=whitelistUpdate)
-  whitelistupdate_thread.daemon = True
-  whitelistupdate_thread.start()
-
-  while not quit_now:
-    time.sleep(0.5)
-
-  logdebug("Exiting with code %s" % exit_code)
-  sys.exit(exit_code)
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import signal
+import redis
+from modules.Logger import Logger
+from modules.Netfilter import Netfilter
+
+chain_name = "MAILCOW"
+
+def connect_redis(logger):
+  if os.getenv('REDIS_SLAVEOF_IP', ''):
+    host, port = os.getenv('REDIS_SLAVEOF_IP'), os.getenv('REDIS_SLAVEOF_PORT', '')
+  else:
+    host, port = os.getenv('IPV4_NETWORK', '172.22.1') + '.249', 6379
+  while True:
+    try:
+      # log lines are not always valid UTF-8
+      r = redis.StrictRedis(host=host, port=port, db=0, password=os.environ['REDISPASS'], decode_responses=True,
+                            encoding_errors='replace', socket_connect_timeout=5, socket_timeout=30)
+      r.ping()
+      return r
+    except Exception as ex:
+      logger.logWarn('Redis connection failed: %s - trying again in 3 seconds' % ex)
+      time.sleep(3)
+
+if __name__ == '__main__':
+  logger = Logger()
+
+  backend = sys.argv[1]
+  if backend == "nftables":
+    from modules.NFTables import NFTables
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    from modules.IPTables import IPTables
+    logger.logInfo('Using IPTables backend')
+    logger.logWarn(
+        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
+        "Please switch to nftables on your host to ensure complete compatibility."
+    )
+    time.sleep(5)
+    tables = IPTables(chain_name, logger)
+
+  r = connect_redis(logger)
+  logger.set_redis(r)
+
+  netfilter = Netfilter(r, tables, logger, os.environ)
+  signal.signal(signal.SIGTERM, lambda signum, frame: netfilter.stop.set())
+  signal.signal(signal.SIGINT, lambda signum, frame: netfilter.stop.set())
+
+  netfilter.setup()
+  netfilter.run()
+  netfilter.shutdown()
+  sys.exit(0)

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -0,0 +1,236 @@
+import ipaddress
+import time
+
+
+class BanManager:
+  """Failed attempts and automatic bans (Redis F2B_ACTIVE_BANS: network => expiry), the denylist (F2B_PERM_BANS)
+  and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
+  (chains, SNAT), because the backends must not run in parallel."""
+
+  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
+  EXCUSE_TIME = 60
+
+  def __init__(self, r, tables, options, logger, lock, clock=time.time):
+    self.r = r
+    self.tables = tables
+    self.options = options
+    self.logger = logger
+    self.lock = lock
+    self.clock = clock
+    self.counters = {}
+    self.banned = set()
+    self.denied = {}
+    self.allowlist = []
+    self.excused = {}
+
+  def excuse(self, ip):
+    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
+    with self.lock:
+      now = self.clock()
+      if self.excused.get(ip, 0) <= now:
+        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
+          ip, self.EXCUSE_TIME))
+      self.excused[ip] = now + self.EXCUSE_TIME
+
+  def attempt(self, ip):
+    """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
+    options = self.options.values
+    prefix = options['netban_ipv4'] if ip.version == 4 else options['netban_ipv6']
+    net = str(ipaddress.ip_network('%s/%d' % (ip, prefix), strict=False))
+    with self.lock:
+      now = self.clock()
+      allowed = self.allowed_by(ip)
+      if allowed is not None:
+        self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
+        return
+      if self.excused.get(ip, 0) > now:
+        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
+        return
+      if net in self.banned:
+        # log lines of connections from before the ban
+        return
+      counter = self.counter(net)
+      if now - counter['last_attempt'] > options['retry_window']:
+        counter['attempts'] = 0
+      counter['attempts'] += 1
+      counter['last_attempt'] = now
+      if counter['attempts'] < options['max_attempts']:
+        self.logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
+          options['max_attempts'] - counter['attempts'], options['retry_window'], net))
+        return
+
+      ban_time = self.options.ban_time(counter['ban_counter'])
+      self.logger.logCrit('Banning %s for %d minutes' % (net, ban_time / 60))
+      # Redis first: a firewall rule without expiry would never be lifted
+      self.r.hset('F2B_ACTIVE_BANS', net, int(round(now + ban_time)))
+      self.banned.add(net)
+      if options['manage_external'] != 1:
+        self.apply([net])
+
+  def unban(self, net):
+    """Lifts the automatic ban of a network; the rule stays while the denylist bans the same network"""
+    with self.lock:
+      normalized = self.normalize(net)
+      if normalized is None:
+        self.logger.logWarn('Removing invalid network %s from the active bans' % net)
+      else:
+        self.logger.logInfo('Unbanning %s' % net)
+        if normalized not in self.denied:
+          self.remove([normalized])
+      self.r.hdel('F2B_ACTIVE_BANS', net)
+      self.r.hdel('F2B_QUEUE_UNBAN', net)
+      if net in self.banned:
+        self.banned.discard(net)
+        counter = self.counter(net)
+        counter['attempts'] = 0
+        counter['ban_counter'] += 1
+
+  def unban_queued(self, entry):
+    """Unban request of the UI: a banned network, or a network that contains banned ones (allowlisting)"""
+    network = self.network(entry)
+    if network is None:
+      self.logger.logWarn('Removing invalid network %s from the unban queue' % entry)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+      return
+    with self.lock:
+      banned = set(self.r.hkeys('F2B_ACTIVE_BANS')) | self.banned
+      targets = [net for net in banned if self.overlaps(net, network)]
+      for net in targets:
+        self.unban(net)
+      if not targets and str(network) not in self.denied:
+        # a rule without known ban, e.g. left by an older netfilter
+        if self.remove([str(network)]):
+          self.logger.logInfo('Unbanning %s' % network)
+        else:
+          self.logger.logInfo('%s is not banned, removing it from the unban queue' % network)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+
+  def purge(self):
+    """Processes the unban queue of the UI and lifts expired bans"""
+    for entry in self.r.hkeys('F2B_QUEUE_UNBAN'):
+      self.unban_queued(entry)
+    now = self.clock()
+    for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+      try:
+        expired = self.network(net) is None or float(expires) <= now
+      except (TypeError, ValueError):
+        expired = True
+      if expired:
+        self.unban(net)
+    with self.lock:
+      # bans whose Redis entry is gone, e.g. deleted by hand
+      for net in self.banned - set(self.r.hkeys('F2B_ACTIVE_BANS')):
+        self.unban(net)
+    self.forget_idle(now)
+
+  def restore(self):
+    """Applies the unexpired automatic bans of Redis again after a start of netfilter and drops the expired ones"""
+    now = self.clock()
+    restored = []
+    with self.lock:
+      for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+        try:
+          valid = self.network(net) is not None and float(expires) > now
+        except (TypeError, ValueError):
+          valid = False
+        if not valid:
+          self.r.hdel('F2B_ACTIVE_BANS', net)
+          continue
+        self.banned.add(net)
+        self.counter(net)
+        restored.append(net)
+      if restored and self.options.values['manage_external'] != 1:
+        self.apply(restored)
+    if restored:
+      self.logger.logInfo('Restored %d active bans' % len(restored))
+
+  def reapply(self):
+    """Applies all bans again, e.g. after the MAILCOW chain had to be recreated"""
+    with self.lock:
+      if self.options.values['manage_external'] != 1:
+        self.apply(list(self.banned) + list(self.denied))
+
+  def set_denylist(self, entries):
+    """Bans the networks of the denylist permanently and lifts the bans of removed entries"""
+    networks = {}
+    for entry in entries:
+      net = self.normalize(entry)
+      if net is None:
+        self.logger.logWarn('Skipping invalid denylist entry %s' % entry)
+        continue
+      networks[net] = entry
+    with self.lock:
+      if networks.keys() == self.denied.keys():
+        return
+      self.logger.logInfo('Denylist was changed, it has %d entries' % len(networks))
+      added = [net for net in networks if net not in self.denied]
+      removed = [net for net in self.denied if net not in networks]
+      if added and self.options.values['manage_external'] != 1:
+        self.apply(added)
+      for net in added:
+        self.r.hset('F2B_PERM_BANS', networks[net], int(round(self.clock())))
+        self.logger.logCrit('Added host/network %s to denylist' % networks[net])
+      # an automatic ban of the same network keeps its rule
+      auto_banned = {self.normalize(net) for net in self.banned}
+      self.remove([net for net in removed if net not in auto_banned])
+      for net in removed:
+        self.r.hdel('F2B_PERM_BANS', self.denied[net])
+        self.logger.logCrit('Removed host/network %s from denylist' % self.denied[net])
+      self.denied = networks
+
+  def set_allowlist(self, entries):
+    networks = []
+    for entry in entries:
+      network = self.network(entry)
+      if network is None:
+        self.logger.logWarn('Skipping invalid allowlist entry %s' % entry)
+        continue
+      networks.append(network)
+    with self.lock:
+      if set(networks) != set(self.allowlist):
+        self.allowlist = networks
+        self.logger.logInfo('Allowlist was changed, it has %d entries' % len(networks))
+
+  def allowed_by(self, ip):
+    address = ipaddress.ip_network(ip)
+    for network in self.allowlist:
+      if network.version == address.version and network.overlaps(address):
+        return network
+    return None
+
+  def counter(self, net):
+    return self.counters.setdefault(net, {'attempts': 0, 'last_attempt': 0.0, 'ban_counter': 0})
+
+  def forget_idle(self, now):
+    """Drops counters of networks that were never banned and had no attempt within the retry window"""
+    window = self.options.values['retry_window']
+    with self.lock:
+      for net in [net for net, counter in self.counters.items()
+                  if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
+        del self.counters[net]
+      for ip in [ip for ip, until in self.excused.items() if until <= now]:
+        del self.excused[ip]
+
+  def apply(self, nets):
+    for net in self.tables.ban(nets):
+      self.logger.logWarn('Could not add the firewall rule for %s' % net)
+
+  def remove(self, nets):
+    return self.tables.unban(nets) if nets else []
+
+  @staticmethod
+  def network(entry):
+    try:
+      return ipaddress.ip_network(entry, strict=False)
+    except (TypeError, ValueError):
+      return None
+
+  @classmethod
+  def normalize(cls, entry):
+    network = cls.network(entry)
+    return str(network) if network is not None else None
+
+  @classmethod
+  def overlaps(cls, net, network):
+    other = cls.network(net)
+    return other is not None and other.version == network.version and other.overlaps(network)

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,6 +1,7 @@
 import iptc
 import time
-import os
+import ipaddress
+import contextlib
 
 class IPTables:
   def __init__(self, chain_name, logger):
@@ -8,62 +9,10 @@ class IPTables:
     self.logger = logger
 
   def initChainIPv4(self):
-    if not iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name) in iptc.Table(iptc.Table.FILTER).chains:
-      iptc.Table(iptc.Table.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
-      rule = iptc.Rule()
-      rule.src = '0.0.0.0/0'
-      rule.dst = '0.0.0.0/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
+    self.init_chain(iptc.Table(iptc.Table.FILTER))
 
   def initChainIPv6(self):
-    if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name) in iptc.Table6(iptc.Table6.FILTER).chains:
-      iptc.Table6(iptc.Table6.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
-      rule = iptc.Rule6()
-      rule.src = '::/0'
-      rule.dst = '::/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
-
-  def checkIPv4ChainOrder(self):
-    filter_table = iptc.Table(iptc.Table.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkIPv6ChainOrder(self):
-    filter_table = iptc.Table6(iptc.Table6.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    forward_chain = iptc.Chain(filter_table, 'FORWARD')
-    input_chain = iptc.Chain(filter_table, 'INPUT')
-    for chain in [forward_chain, input_chain]:
-      target_found = False
-      for position, item in enumerate(chain.rules):
-        if item.target.name == self.chain_name:
-          target_found = True
-          if position > 2:
-            self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-            err = True
-            exit_code = 2
-      if not target_found:
-        self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain(iptc.Table6(iptc.Table6.FILTER))
 
   def clearIPv4Table(self):
     self.clearTable(iptc.Table(iptc.Table.FILTER))
@@ -71,68 +20,133 @@ class IPTables:
   def clearIPv6Table(self):
     self.clearTable(iptc.Table6(iptc.Table6.FILTER))
 
-  def clearTable(self, filter_table):
-    filter_table.autocommit = False
-    forward_chain = iptc.Chain(filter_table, "FORWARD")
-    input_chain = iptc.Chain(filter_table, "INPUT")
-    mailcow_chain = iptc.Chain(filter_table, self.chain_name)
-    if mailcow_chain in filter_table.chains:
-      for rule in mailcow_chain.rules:
-        mailcow_chain.delete_rule(rule)
-      for rule in forward_chain.rules:
-        if rule.target.name == self.chain_name:
-          forward_chain.delete_rule(rule)
-      for rule in input_chain.rules:
-        if rule.target.name == self.chain_name:
-          input_chain.delete_rule(rule)
-      filter_table.delete_chain(self.chain_name)
-    filter_table.commit()
+  @contextlib.contextmanager
+  def transaction(self, table):
+    # One commit for all changes, autocommit rewrites the whole table for every change
+    table.autocommit = False
+    table.refresh()
+    try:
+      yield
+      table.commit()
+    finally:
+      table.refresh()
+      table.autocommit = True
+
+  def new_rule(self, table):
+    return iptc.Rule6() if isinstance(table, iptc.Table6) else iptc.Rule()
+
+  def family(self, table):
+    return 'IPv6' if isinstance(table, iptc.Table6) else 'IPv4'
+
+  def init_chain(self, filter_table):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
     filter_table.refresh()
-    filter_table.autocommit = True
+    if filter_table.is_chain(self.chain_name):
+      filter_table.flush_entries(self.chain_name)
+    else:
+      filter_table.create_chain(self.chain_name)
+    self.ensure_jumps(filter_table)
 
-  def banIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    recreated = False
+    for filter_table in [iptc.Table(iptc.Table.FILTER), iptc.Table6(iptc.Table6.FILTER)]:
+      filter_table.refresh()
+      if not filter_table.is_chain(self.chain_name):
+        filter_table.create_chain(self.chain_name)
+        self.logger.logWarn(f'MAILCOW {self.family(filter_table)} chain was missing and has been recreated')
+        recreated = True
+      self.ensure_jumps(filter_table)
+    return recreated
 
-  def banIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_jumps(self, filter_table):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jump (DOCKER-USER) at the top
+    for name in ['INPUT', 'FORWARD']:
+      chain = iptc.Chain(filter_table, name)
+      positions = [position for position, rule in enumerate(chain.rules) if rule.target.name == self.chain_name]
+      if positions == [0]: continue
 
-  def unbanIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules: 
-      return False
-    chain.delete_rule(rule)
-    return True
+      with self.transaction(filter_table):
+        for rule in [rule for rule in chain.rules if rule.target.name == self.chain_name]:
+          chain.delete_rule(rule)
+        rule = self.new_rule(filter_table)
+        rule.target = iptc.Target(rule, self.chain_name)
+        chain.insert_rule(rule)
+      if positions:
+        self.logger.logInfo(f'Moved MAILCOW jump to the top of the {self.family(filter_table)} {name} chain, it was at position {positions[0] + 1}')
+      else:
+        self.logger.logInfo(f'Added MAILCOW jump to the {self.family(filter_table)} {name} chain')
 
-  def unbanIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules:
-      return False
-    chain.delete_rule(rule)
-    return True
+  def clearTable(self, filter_table):
+    try:
+      with self.transaction(filter_table):
+        if not filter_table.is_chain(self.chain_name):
+          return
+        for chain in filter_table.chains:
+          for rule in chain.rules:
+            if rule.target.name == self.chain_name:
+              chain.delete_rule(rule)
+        filter_table.flush_entries(self.chain_name)
+        filter_table.delete_chain(self.chain_name)
+      self.logger.logInfo(f'Clear completed: {self.family(filter_table)}')
+    except Exception as ex:
+      self.logger.logWarn(f'Error clearing the {self.family(filter_table)} MAILCOW chain: {ex}')
+
+  def ban_rule_network(self, rule):
+    # Source network of a ban rule, None for every other rule
+    if rule.target.name not in ('REJECT', 'DROP') or rule.matches or rule.in_interface or rule.out_interface:
+      return None
+    try:
+      return ipaddress.ip_network(rule.src, strict=False)
+    except ValueError:
+      return None
+
+  def split_tables(self, sources):
+    networks = {4: [], 6: []}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      networks[network.version].append(network)
+    tables = [(iptc.Table(iptc.Table.FILTER), networks[4]), (iptc.Table6(iptc.Table6.FILTER), networks[6])]
+    return [(table, table_networks) for table, table_networks in tables if table_networks]
+
+  def ban(self, sources):
+    # One REJECT rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for filter_table, networks in self.split_tables(sources):
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          existing = {self.ban_rule_network(rule) for rule in chain.rules}
+          for network in dict.fromkeys(networks):
+            if network in existing: continue
+            rule = self.new_rule(filter_table)
+            rule.src = str(network)
+            rule.target = iptc.Target(rule, "REJECT")
+            chain.insert_rule(rule)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        failed += networks
+    return [str(network) for network in failed]
+
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for filter_table, networks in self.split_tables(sources):
+      targets = set(networks)
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          found = set()
+          for rule in chain.rules:
+            network = self.ban_rule_network(rule)
+            if network in targets:
+              chain.delete_rule(rule)
+              found.add(network)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        continue
+      removed += found
+    return [str(network) for network in removed]
 
   def snat4(self, snat_target, source):
     try:
@@ -188,7 +202,7 @@ class IPTables:
           if item == new_rule:
             if position != 0:
               chain.delete_rule(new_rule)
-    
+
       table.commit()
       table.autocommit = True
     except:
@@ -215,7 +229,9 @@ class IPTables:
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
     try:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+      table = iptc.Table(iptc.Table.FILTER)
+      table.refresh()
+      chain = iptc.Chain(table, self.chain_name)
 
       # insert mailcow isolation rule
       rule = iptc.Rule()
@@ -231,7 +247,7 @@ class IPTables:
       chain.insert_rule(rule, position=0)
 
       # insert mailcow isolation exception rule
-      if _allow != "":
+      if _allow:
         rule = iptc.Rule()
         rule.src = _allow
         rule.in_interface = f'!{_interface}'
@@ -249,4 +265,4 @@ class IPTables:
       return True
     except Exception as e:
       self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
-      return False
+      return False

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -1,8 +1,9 @@
 import nftables
 import ipaddress
-import os
 
 class NFTables:
+  ISOLATION_COMMENTS = ('mailcow isolation', 'mailcow isolation allow')
+
   def __init__(self, chain_name, logger):
     self.chain_name = chain_name
     self.logger = logger
@@ -12,41 +13,15 @@ class NFTables:
     self.nft.set_handle_output(True)
     self.nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
                             'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
+    self.mailcow_chains = set()
 
     self.search_current_chains()
 
   def initChainIPv4(self):
-    self.insert_mailcow_chains("ip")
+    self.init_chain("ip")
 
   def initChainIPv6(self):
-    self.insert_mailcow_chains("ip6")
-
-  def checkIPv4ChainOrder(self):
-    return self.checkChainOrder("ip")
-
-  def checkIPv6ChainOrder(self):
-    return self.checkChainOrder("ip6")
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    for chain in ['input', 'forward']:
-      chain_position = self.check_mailcow_chains(filter_table, chain)
-      if chain_position is None: continue
-
-      if chain_position is False:
-        self.logger.logCrit(f'MAILCOW target not found in {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-      if chain_position > 0:
-        chain_position += 1
-        self.logger.logCrit(f'MAILCOW target is in position {chain_position} in the {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain("ip6")
 
   def clearIPv4Table(self):
     self.clearTable("ip")
@@ -54,84 +29,19 @@ class NFTables:
   def clearIPv6Table(self):
     self.clearTable("ip6")
 
-  def clearTable(self, _family):
-    is_empty_dict = True
-    json_command = self.get_base_dict()
-    chain_handle = self.get_chain_handle(_family, "filter", self.chain_name)
-    # if no handle, the chain doesn't exists
-    if chain_handle is not None:
-      is_empty_dict = False
-      # flush chain
-      mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-      flush_chain = {'flush': {'chain': mailcow_chain}}
-      json_command["nftables"].append(flush_chain)
-
-    # remove rule in forward chain
-    # remove rule in input chain
-    chains_family = [self.nft_chain_names[_family]['filter']['input'],
-                    self.nft_chain_names[_family]['filter']['forward'] ]
-
-    for chain_base in chains_family:
-      if not chain_base: continue
-
-      rules_handle = self.get_rules_handle(_family, "filter", chain_base)
-      if rules_handle is not None:
-        for r_handle in rules_handle:
-          is_empty_dict = False
-          mailcow_rule = {'family':_family,
-                          'table': 'filter',
-                          'chain': chain_base,
-                          'handle': r_handle }
-          delete_rules = {'delete': {'rule': mailcow_rule} }
-          json_command["nftables"].append(delete_rules)
-
-    # remove chain
-    # after delete all rules referencing this chain
-    if chain_handle is not None:
-      mc_chain_handle = {'family':_family,
-                        'table': 'filter',
-                        'name': self.chain_name,
-                        'handle': chain_handle }
-      delete_chain = {'delete': {'chain': mc_chain_handle} }
-      json_command["nftables"].append(delete_chain)
-
-    if is_empty_dict == False:
-      if self.nft_exec_dict(json_command):
-        self.logger.logInfo(f"Clear completed: {_family}")
-
-  def banIPv4(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip")
-    return self.nft_exec_dict(ban_dict)
-
-  def banIPv6(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip6")
-    return self.nft_exec_dict(ban_dict)
-
-  def unbanIPv4(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
-  def unbanIPv6(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip6")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
   def snat4(self, snat_target, source):
     self.snat_rule("ip", snat_target, source)
 
   def snat6(self, snat_target, source):
     self.snat_rule("ip6", snat_target, source)
 
-
-  def nft_exec_dict(self, query: dict):
+  def nft_exec_dict(self, query: dict, quiet: bool = False):
     if not query: return False
 
     rc, output, error = self.nft.json_cmd(query)
     if rc != 0:
-      #self.logger.logCrit(f"Nftables Error: {error}")
+      if not quiet:
+        self.logger.logWarn(f"nftables: {str(error).strip() or f'error {rc}'}")
       return False
 
     # Prevent returning False or empty string on commands that do not produce output
@@ -144,118 +54,165 @@ class NFTables:
     return {'nftables': [{ 'metainfo': { 'json_schema_version': 1} } ] }
 
   def search_current_chains(self):
+    nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
+                       'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
     nft_chain_priority = {'ip': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} },
-                      'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+                          'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+    mailcow_chains = set()
 
     # Command: 'nft list chains'
     _list = {'list' : {'chains': 'null'} }
     command = self.get_base_dict()
     command['nftables'].append(_list)
     kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset['nftables']:
-        chain = _object.get("chain")
-        if not chain: continue
+    if not kernel_ruleset:
+      return
 
-        _family = chain['family']
-        _table = chain['table']
-        _hook = chain.get("hook")
-        _priority = chain.get("prio")
-        _name = chain['name']
-
-        if _family not in self.nft_chain_names: continue
-        if _table not in self.nft_chain_names[_family]: continue
-        if _hook not in self.nft_chain_names[_family][_table]: continue
-        if _priority is None: continue
-
-        _saved_priority = nft_chain_priority[_family][_table][_hook]
-        if _saved_priority is None or _priority < _saved_priority:
-          # at this point, we know the chain has:
-          # hook and priority set
-          # and it has the lowest priority
-          nft_chain_priority[_family][_table][_hook] = _priority
-          self.nft_chain_names[_family][_table][_hook] = _name
-
-  def search_for_chain(self, kernel_ruleset: dict, chain_name: str):
-    found = False
-    for _object in kernel_ruleset["nftables"]:
+    for _object in kernel_ruleset['nftables']:
       chain = _object.get("chain")
-      if not chain:
-        continue
-      ch_name = chain.get("name")
-      if ch_name == chain_name:
-        found = True
-        break
-    return found
+      if not chain: continue
+
+      _family = chain['family']
+      _table = chain['table']
+      _hook = chain.get("hook")
+      _priority = chain.get("prio")
+      _name = chain['name']
+
+      if _family not in nft_chain_names: continue
+      if _table not in nft_chain_names[_family]: continue
+      if _table == 'filter' and _name == self.chain_name:
+        mailcow_chains.add(_family)
+      if _hook not in nft_chain_names[_family][_table]: continue
+      if _priority is None: continue
+
+      _saved_priority = nft_chain_priority[_family][_table][_hook]
+      if _saved_priority is None or _priority < _saved_priority:
+        # at this point, we know the chain has:
+        # hook and priority set
+        # and it has the lowest priority
+        nft_chain_priority[_family][_table][_hook] = _priority
+        nft_chain_names[_family][_table][_hook] = _name
+
+    self.nft_chain_names = nft_chain_names
+    self.mailcow_chains = mailcow_chains
+
+  def has_filter_chains(self, _family: str):
+    # no base chains: the filter table does not exist, e.g. ip6 if docker has ipv6 disabled
+    return any(self.nft_chain_names[_family]['filter'].values())
+
+  def list_rules(self, _family: str, _table: str, _chain: str):
+    # Command: 'nft list chain {family} {table} {chain}', None if the chain does not exist
+    _list = {'list': {'chain': {'family': _family, 'table': _table, 'name': _chain}}}
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return None
+    return [_object["rule"] for _object in kernel_ruleset["nftables"] if _object.get("rule")]
 
   def get_chain_dict(self, _family: str, _name: str):
-    # nft (add | create) chain [<family>] <table> <name> 
+    # nft (add | create) chain [<family>] <table> <name>
     _chain_opts = {'family': _family, 'table': 'filter', 'name': _name  }
     _add = {'add': {'chain': _chain_opts} }
     final_chain = self.get_base_dict()
     final_chain["nftables"].append(_add)
     return final_chain
 
-  def get_mailcow_jump_rule_dict(self, _family: str, _chain: str):
-    _jump_rule = self.get_base_dict()
-    _expr_opt=[]
-    _expr_counter = {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}
-    _counter_dict = {'counter': _expr_counter}
-    _expr_opt.append(_counter_dict)
-
-    _jump_opts = {'jump': {'target': self.chain_name} }
-
-    _expr_opt.append(_jump_opts)
-
+  def get_mailcow_jump_rule(self, _family: str, _chain: str):
+    _expr_opt = [{'counter': {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}},
+                 {'jump': {'target': self.chain_name}}]
     _rule_params = {'family': _family,
                     'table': 'filter',
                     'chain': _chain,
                     'expr': _expr_opt,
                     'comment': "mailcow" }
+    return {'insert': {'rule': _rule_params} }
 
-    _add_rule = {'insert': {'rule': _rule_params} }
+  def is_mailcow_jump(self, rule: dict):
+    # by target, jumps restored by iptables-restore carry no nft comment
+    return any(isinstance(expr, dict) and isinstance(expr.get('jump'), dict) and expr['jump'].get('target') == self.chain_name
+               for expr in rule.get('expr', []))
 
-    _jump_rule["nftables"].append(_add_rule)
+  def init_chain(self, _family: str):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
+    if not self.has_filter_chains(_family): return
 
-    return _jump_rule
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+        self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    else:
+      flush_chain = {'flush': {'chain': {'family': _family, 'table': 'filter', 'name': self.chain_name}}}
+      command = self.get_base_dict()
+      command['nftables'].append(flush_chain)
+      self.nft_exec_dict(command)
 
-  def insert_mailcow_chains(self, _family: str):
-    nft_input_chain = self.nft_chain_names[_family]['filter']['input']
-    nft_forward_chain = self.nft_chain_names[_family]['filter']['forward']
-    # Command: 'nft list table <family> filter'
-    _table_opts = {'family': _family, 'name': 'filter'}
-    _list = {'list': {'table': _table_opts} }
+    self.ensure_jumps(_family)
+
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    self.search_current_chains()
+    recreated = False
+    for _family in ['ip', 'ip6']:
+      if not self.has_filter_chains(_family): continue
+
+      if _family not in self.mailcow_chains:
+        if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+          self.logger.logWarn(f"MAILCOW {_family} chain was missing and has been recreated")
+          recreated = True
+      self.ensure_jumps(_family)
+    return recreated
+
+  def ensure_jumps(self, _family: str):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jumps (DOCKER-USER, DOCKER-FORWARD) at the top
+    for hook in ['input', 'forward']:
+      base_chain = self.nft_chain_names[_family]['filter'][hook]
+      if not base_chain: continue
+
+      rules = self.list_rules(_family, 'filter', base_chain)
+      if rules is None: continue
+
+      positions = [position for position, rule in enumerate(rules) if self.is_mailcow_jump(rule)]
+      if positions == [0]: continue
+
+      # delete and insert in one transaction, the chain is never without jump
+      command = self.get_base_dict()
+      for position in positions:
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': base_chain, 'handle': rules[position]['handle']}
+        command['nftables'].append({'delete': {'rule': _rule_opts}})
+      command['nftables'].append(self.get_mailcow_jump_rule(_family, base_chain))
+      if self.nft_exec_dict(command):
+        if positions:
+          self.logger.logInfo(f"Moved MAILCOW jump to the top of the {_family} {base_chain} chain, it was at position {positions[0] + 1}")
+        else:
+          self.logger.logInfo(f"Added MAILCOW jump to the {_family} {base_chain} chain")
+
+  def clearTable(self, _family: str):
+    # Jumps first, the chain can only be deleted without references. Separate steps, so one failing step
+    # does not roll back the others
+    _list = {'list': {'table': {'family': _family, 'name': 'filter'}}}
     command = self.get_base_dict()
     command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      # chain
-      if not self.search_for_chain(kernel_ruleset, self.chain_name):
-        cadena = self.get_chain_dict(_family, self.chain_name)
-        if self.nft_exec_dict(cadena):
-          self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return
 
-      input_jump_found, forward_jump_found = False, False
+    delete_jumps = self.get_base_dict()
+    for _object in kernel_ruleset["nftables"]:
+      rule = _object.get("rule")
+      if rule and self.is_mailcow_jump(rule):
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': rule['chain'], 'handle': rule['handle']}
+        delete_jumps['nftables'].append({'delete': {'rule': _rule_opts}})
+    if len(delete_jumps['nftables']) > 1:
+      self.nft_exec_dict(delete_jumps)
 
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if nft_input_chain and rule["chain"] == nft_input_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            input_jump_found = True
-        if nft_forward_chain and rule["chain"] == nft_forward_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            forward_jump_found = True
-
-      if not input_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_input_chain)
-        self.nft_exec_dict(command)
-
-      if not forward_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_forward_chain)
-        self.nft_exec_dict(command)
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      return
+    mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+    command = self.get_base_dict()
+    command['nftables'].append({'flush': {'chain': mailcow_chain}})
+    command['nftables'].append({'delete': {'chain': mailcow_chain}})
+    if self.nft_exec_dict(command):
+      self.logger.logInfo(f"Clear completed: {_family}")
 
   def delete_nat_rule(self, _family:str, _chain: str, _handle:str):
     delete_command = self.get_base_dict()
@@ -285,23 +242,14 @@ class NFTables:
     # no postrouting chain, may occur if docker has ipv6 disabled.
     if not chain_name: return
 
-    # Command: nft list chain <family> nat <chain_name>
-    _chain_opts = {'family': _family, 'table': 'nat', 'name': chain_name}
-    _list = {'list':{'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if not kernel_ruleset:
+    rules = self.list_rules(_family, 'nat', chain_name)
+    if rules is None:
       return
 
     rule_position = 0
     rule_handle = None
     rule_found = False
-    for _object in kernel_ruleset["nftables"]:
-      if not _object.get("rule"):
-        continue
-
-      rule = _object["rule"]
+    for rule in rules:
       if not rule.get("comment") or not rule["comment"] == "mailcow":
         rule_position +=1
         continue
@@ -314,23 +262,28 @@ class NFTables:
     target_net = ipaddress.ip_network(snat_target, strict=False)
 
     if rule_found:
-      saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
-      saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
+      try:
+        saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
+        saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
 
-      daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
-      daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
+        daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
+        daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
 
-      target_ip = rule["expr"][3]["snat"]["addr"]
+        target_ip = rule["expr"][3]["snat"]["addr"]
 
-      saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
-      daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
-      current_target_net = ipaddress.ip_network(target_ip, strict=False)
+        saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
+        daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
+        current_target_net = ipaddress.ip_network(target_ip, strict=False)
 
-      match = all((
-                dest_net == saddr_net,
-                dest_net == daddr_net,
-                target_net == current_target_net
-              ))
+        match = all((
+                  dest_net == saddr_net,
+                  dest_net == daddr_net,
+                  target_net == current_target_net
+                ))
+      except (KeyError, IndexError, TypeError, ValueError):
+        # rule of another format, replace it
+        saddr_net = dest_net
+        match = False
       try:
         if rule_position == 0:
           if not match:
@@ -377,49 +330,13 @@ class NFTables:
       except:
         self.logger.logCrit(f"Error running SNAT on {_family}, retrying...")
 
-  def get_chain_handle(self, _family: str, _table: str, chain_name: str):
-    chain_handle = None
-    # Command: 'nft list chains {family}'
-    _list = {'list': {'chains': {'family': _family} } }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("chain"):
-          continue
-        chain = _object["chain"]
-        if chain["family"] == _family and chain["table"] == _table and chain["name"] == chain_name:
-          chain_handle = chain["handle"]
-          break
-    return chain_handle
-
   def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
-    rule_handle = []
-    # Command: 'nft list chain {family} {table} {chain_name}'
-    _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
-          if rule.get("comment") and rule["comment"] == _comment_filter:
-            rule_handle.append(rule["handle"])
-    return rule_handle
-
-  def get_ban_ip_dict(self, ipaddr: str, _family: str):
-    json_command = self.get_base_dict()
+    rules = self.list_rules(_family, _table, chain_name) or []
+    return [rule["handle"] for rule in rules if rule.get("comment") == _comment_filter]
 
+  def get_ban_rule(self, network, _family: str):
     expr_opt = []
-    ipaddr_net = ipaddress.ip_network(ipaddr, strict=False)
-    right_dict = {'prefix': {'addr': str(ipaddr_net.network_address), 'len': int(ipaddr_net.prefixlen) } }
+    right_dict = {'prefix': {'addr': str(network.network_address), 'len': int(network.prefixlen) } }
 
     left_dict = {'payload': {'protocol': _family, 'field': 'saddr'} }
     match_dict = {'op': '==', 'left': left_dict, 'right': right_dict }
@@ -432,228 +349,115 @@ class NFTables:
 
     rule_dict = {'family': _family, 'table': "filter", 'chain': self.chain_name, 'expr': expr_opt}
 
-    base_dict = {'insert': {'rule': rule_dict} }
-    json_command["nftables"].append(base_dict)
+    return {'insert': {'rule': rule_dict} }
 
-    return json_command
+  def ban_rule_network(self, rule: dict, _family: str):
+    # Source network of a ban rule (saddr match and drop), None for every other rule
+    if rule.get("comment") in self.ISOLATION_COMMENTS:
+      return None
+    exprs = [expr for expr in rule.get("expr", []) if isinstance(expr, dict)]
+    if not any('drop' in expr or 'reject' in expr for expr in exprs):
+      return None
+    for expr in exprs:
+      match = expr.get("match")
+      if not isinstance(match, dict) or match.get("op") != "==":
+        continue
+      left = match.get("left")
+      payload = left.get("payload") if isinstance(left, dict) else None
+      if not isinstance(payload, dict) or payload.get("protocol") != _family or payload.get("field") != "saddr":
+        continue
+      right = match.get("right")
+      try:
+        if isinstance(right, dict) and isinstance(right.get("prefix"), dict):
+          return ipaddress.ip_network(f'{right["prefix"]["addr"]}/{right["prefix"]["len"]}', strict=False)
+        if isinstance(right, str):
+          return ipaddress.ip_network(right, strict=False)
+      except (KeyError, ValueError):
+        return None
+    return None
 
-  def get_unban_ip_dict(self, ipaddr:str, _family: str):
-    json_command = self.get_base_dict()
-    # Command: 'nft list chain {s_family} filter  MAILCOW'
-    _chain_opts = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    rule_handle = None
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
+  def split_families(self, sources):
+    families = {}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      families.setdefault('ip' if network.version == 4 else 'ip6', []).append(network)
+    return families
 
-        rule = _object["rule"]["expr"][0]["match"]
-        if not "payload" in rule["left"]:
-          continue
-        left_opt = rule["left"]["payload"]
-        if not left_opt["protocol"] == _family:
-          continue
-        if not left_opt["field"] =="saddr":
-          continue
+  def ban(self, sources):
+    # One drop rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if rules is None:
+        failed += networks
+        continue
+      existing = {self.ban_rule_network(rule, _family) for rule in rules}
+      missing = [network for network in dict.fromkeys(networks) if network not in existing]
+      if not missing: continue
 
-        # ip currently banned
-        rule_right = rule["right"]
-        if isinstance(rule_right, dict):
-          current_rule_ip = rule_right["prefix"]["addr"] + '/' + str(rule_right["prefix"]["len"])
-        else:
-          current_rule_ip = rule_right
-        current_rule_net = ipaddress.ip_network(current_rule_ip)
+      command = self.get_base_dict()
+      for network in missing:
+        command['nftables'].append(self.get_ban_rule(network, _family))
+      if not self.nft_exec_dict(command):
+        failed += missing
+    return [str(network) for network in failed]
 
-        # ip to ban
-        candidate_net = ipaddress.ip_network(ipaddr, strict=False)
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if not rules: continue
 
-        if current_rule_net == candidate_net:
-          rule_handle = _object["rule"]["handle"]
-          break
-
-      if rule_handle is not None:
-        mailcow_rule = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule_handle}
-        delete_rule = {'delete': {'rule': mailcow_rule} }
-        json_command["nftables"].append(delete_rule)
-      else:
-        return False
-
-    return json_command
-
-  def check_mailcow_chains(self, family: str, chain: str):
-    position = 0
-    rule_found = False
-    chain_name = self.nft_chain_names[family]['filter'][chain]
-
-    if not chain_name: return None
-
-    _chain_opts = {'family': family, 'table': 'filter', 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts}}
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-        rule = _object["rule"]
-        if rule.get("comment") and rule["comment"] == "mailcow":
-          rule_found = True
-          break
-
-        position+=1
-
-    return position if rule_found else False
+      targets = set(networks)
+      command = self.get_base_dict()
+      found = set()
+      for rule in rules:
+        network = self.ban_rule_network(rule, _family)
+        if network in targets:
+          _rule_opts = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule['handle']}
+          command['nftables'].append({'delete': {'rule': _rule_opts}})
+          found.add(network)
+      if found and self.nft_exec_dict(command):
+        removed += found
+    return [str(network) for network in removed]
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
-    family = "ip"
-    table = "filter"
-    comment_filter_drop = "mailcow isolation"
-    comment_filter_allow = "mailcow isolation allow"
-    json_command = self.get_base_dict()
+    # One rule per port: iptables-nft cannot show an anonymous set, a host that saves and restores the ruleset
+    # with iptables-save (e.g. netfilter-persistent) would get a drop for all ports
+    for comment in self.ISOLATION_COMMENTS:
+      for handle in self.get_rules_handle("ip", "filter", self.chain_name, comment):
+        self.delete_filter_rule("ip", self.chain_name, handle)
 
-    # Delete old mailcow isolation rules
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-
-    # insert mailcow isolation rule
-    _match_dict_drop = [
-      {
-        "match": {
-          "op": "!=",
-          "left": {
-            "meta": {
-              "key": "iifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "meta": {
-              "key": "oifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "payload": {
-              "protocol": "tcp",
-              "field": "dport"
-            }
-          },
-          "right": {
-            "set": _dports
-          }
-        }
-      },
-      {
-        "counter": {
-          "packets": 0,
-          "bytes": 0
-        }
-      },
-      {
-        "drop": None
-      }
-    ]
-    rule_drop = { "insert": { "rule": {
-      "family": family,
-      "table": table,
-      "chain": self.chain_name,
-      "comment": comment_filter_drop,
-      "expr": _match_dict_drop
-    }}}
-    json_command["nftables"].append(rule_drop)
-
-    # insert mailcow isolation allow rule
-    if _allow != "":
-      _match_dict_allow = [
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "ip",
-                "field": "saddr"
-              }
-            },
-            "right": _allow
-          }
-        },
-        {
-          "match": {
-            "op": "!=",
-            "left": {
-              "meta": {
-                "key": "iifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "meta": {
-                "key": "oifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "tcp",
-                "field": "dport"
-              }
-            },
-            "right": {
-              "set": _dports
-            }
-          }
-        },
-        {
-          "counter": {
-            "packets": 0,
-            "bytes": 0
-          }
-        },
-        {
-          "accept": None
-        }
-      ]
-      rule_allow = { "insert": { "rule": {
-        "family": family,
-        "table": table,
-        "chain": self.chain_name,
-        "comment": comment_filter_allow,
-        "expr": _match_dict_allow
-      }}}
-      json_command["nftables"].append(rule_allow)
-
-    success = self.nft_exec_dict(json_command)
-    if success == False:
+    if not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[0], {"drop": None})):
       self.logger.logCrit(f"Error adding {self.chain_name} isolation")
       return False
 
-    return True
+    # separate, an invalid address must not prevent the drop rules. Inserted last, so they are above them
+    if _allow and not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[1], {"accept": None}, _allow)):
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation exception for {_allow}")
+      return False
+
+    return True
+
+  def get_isolation_dict(self, _interface:str, _dports:list, _comment:str, _verdict:dict, _saddr:str = ""):
+    json_command = self.get_base_dict()
+    for port in _dports:
+      expr = []
+      if _saddr:
+        expr.append({"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": _saddr}})
+      expr += [
+        {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": int(port)}},
+        {"counter": {"packets": 0, "bytes": 0}},
+        _verdict
+      ]
+      json_command["nftables"].append({"insert": {"rule": {
+        "family": "ip",
+        "table": "filter",
+        "chain": self.chain_name,
+        "comment": _comment,
+        "expr": expr
+      }}})
+    return json_command

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -0,0 +1,200 @@
+import ipaddress
+import threading
+import dns.exception
+import dns.resolver
+from modules.BanManager import BanManager
+from modules.Options import Options
+from modules.Regexes import Regexes
+
+
+class Netfilter:
+  """Runs the workers of netfilter. Every worker is a loop that logs errors and goes on, so neither a broken log
+  line nor a Redis restart can stop the expiry of bans."""
+
+  ISOLATION_INTERFACE = 'br-mailcow'
+  ISOLATION_PORTS = [3306, 6379, 8983, 12345]
+
+  def __init__(self, r, tables, logger, env):
+    self.r = r
+    self.tables = tables
+    self.logger = logger
+    self.env = env
+    # reentrant, bans.reapply() runs within the chain check
+    self.lock = threading.RLock()
+    self.stop = threading.Event()
+    self.options = Options(logger)
+    self.regexes = Regexes(logger)
+    self.bans = BanManager(r, tables, self.options, logger, self.lock)
+    self.snat4_target = self.snat_target('SNAT_TO_SOURCE', 4)
+    self.snat6_target = self.snat_target('SNAT6_TO_SOURCE', 6)
+    self.resolved = {}
+    self.workers = {
+      'watch': (self.watch, 3),
+      'purge': (self.purge, 10),
+      'chains': (self.check_chains, 10),
+      'lists': (self.update_lists, 60),
+    }
+    if self.snat4_target or self.snat6_target:
+      self.workers['snat'] = (self.snat, 10)
+    self.threads = {}
+
+  def setup(self):
+    if self.r.exists('F2B_LOG'):
+      self.r.rename('F2B_LOG', 'NETFILTER_LOG')
+    # filled again by the lists worker
+    self.r.delete('F2B_PERM_BANS')
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    with self.lock:
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+      self.logger.logInfo('Initializing mailcow netfilter chain')
+      self.tables.initChainIPv4()
+      self.tables.initChainIPv6()
+      self.isolate()
+    self.bans.restore()
+
+  def run(self):
+    for name in self.workers:
+      self.start(name)
+    while not self.stop.wait(5):
+      for name, thread in list(self.threads.items()):
+        if not thread.is_alive():
+          self.logger.logCrit('Worker %s stopped unexpectedly, restarting it' % name)
+          self.start(name)
+
+  def shutdown(self):
+    """Removes the firewall rules, the active bans stay in Redis and are restored on the next start"""
+    self.stop.set()
+    for thread in self.threads.values():
+      thread.join(timeout=5)
+    with self.lock:
+      self.logger.logInfo('Clearing all bans')
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+    try:
+      self.r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      self.logger.logWarn('Error clearing redis key F2B_PERM_BANS: %s' % ex)
+
+  def start(self, name):
+    task, interval = self.workers[name]
+    thread = threading.Thread(target=self.loop, args=(name, task, interval), name=name, daemon=True)
+    self.threads[name] = thread
+    thread.start()
+
+  def loop(self, name, task, interval):
+    while not self.stop.is_set():
+      try:
+        task()
+      except Exception as ex:
+        self.logger.logWarn('Error in worker %s: %s' % (name, ex))
+      self.stop.wait(interval)
+
+  def watch(self):
+    pubsub = self.r.pubsub(ignore_subscribe_messages=True)
+    try:
+      pubsub.subscribe('F2B_CHANNEL')
+      self.logger.logInfo('Watching Redis channel F2B_CHANNEL')
+      while not self.stop.is_set():
+        message = pubsub.get_message(timeout=1.0)
+        if message and message['type'] == 'message' and message['data']:
+          self.handle_line(message['data'])
+    finally:
+      try:
+        pubsub.close()
+      except Exception:
+        pass
+
+  def handle_line(self, line):
+    # a broken line must not end the watcher
+    try:
+      excused = Regexes.temporary_failure(line)
+      if excused is not None:
+        ip = Regexes.parse_ip(excused)
+        if self.external(ip):
+          self.bans.excuse(ip)
+        return
+      result = self.regexes.match(line)
+      if result is None:
+        return
+      rule_id, address = result
+      ip = Regexes.parse_ip(address)
+      if ip is None:
+        self.logger.logWarn('Rule id %s matched an invalid address %s (%s)' % (rule_id, address, line))
+        return
+      if not self.external(ip):
+        return
+      self.logger.logWarn('%s matched rule id %s (%s)' % (ip, rule_id, line))
+      self.bans.attempt(ip)
+    except Exception as ex:
+      self.logger.logWarn('Error processing log line %r: %s' % (line, ex))
+
+  @staticmethod
+  def external(ip):
+    # private and loopback addresses are never banned
+    return ip is not None and not (ip.is_private or ip.is_loopback)
+
+  def purge(self):
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    self.bans.purge()
+
+  def check_chains(self):
+    with self.lock:
+      if self.tables.ensure_chains():
+        self.isolate()
+        self.bans.reapply()
+
+  def snat(self):
+    with self.lock:
+      if self.snat4_target:
+        self.tables.snat4(self.snat4_target, self.env.get('IPV4_NETWORK', '172.22.1') + '.0/24')
+      if self.snat6_target:
+        self.tables.snat6(self.snat6_target, self.env.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
+
+  def update_lists(self):
+    self.bans.set_allowlist(self.resolve(self.r.hkeys('F2B_WHITELIST')))
+    self.bans.set_denylist(self.resolve(self.r.hkeys('F2B_BLACKLIST')))
+
+  def isolate(self):
+    if self.env.get('DISABLE_NETFILTER_ISOLATION_RULE', '').lower() in ('y', 'yes'):
+      self.logger.logInfo('Skipping MAILCOW isolation')
+      return
+    self.logger.logInfo('Setting MAILCOW isolation')
+    self.tables.create_mailcow_isolation_rule(self.ISOLATION_INTERFACE, self.ISOLATION_PORTS, self.env.get('MAILCOW_REPLICA_IP', ''))
+
+  def resolve(self, entries):
+    """Networks of the entries; hostnames are resolved, on DNS errors to their last known addresses"""
+    resolver = dns.resolver.Resolver()
+    networks = set()
+    for entry in entries:
+      if BanManager.network(entry) is not None:
+        networks.add(entry)
+        continue
+      addresses = set()
+      for rdtype in ['A', 'AAAA']:
+        try:
+          answer = resolver.resolve(qname=entry, rdtype=rdtype, lifetime=3)
+        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+          continue
+        except dns.exception.DNSException as ex:
+          self.logger.logInfo('Could not resolve %s: %s' % (entry, ex))
+          addresses = self.resolved.get(entry, set())
+          break
+        addresses.update(rdata.to_text() for rdata in answer)
+      self.resolved[entry] = addresses
+      networks |= addresses
+    return networks
+
+  def snat_target(self, name, version):
+    value = self.env.get(name, '')
+    if not value or value == 'n':
+      return None
+    try:
+      if ipaddress.ip_address(value).version == version:
+        return value
+    except ValueError:
+      pass
+    self.logger.logWarn('%s is not a valid IPv%d address' % (value, version))
+    return None

data/Dockerfiles/netfilter/modules/Options.py

@@ -0,0 +1,120 @@
+import json
+import uuid
+
+
+class Options:
+  """Fail2ban options of the mailcow UI (Redis F2B_OPTIONS). The values are replaced as a whole and never changed
+  in place, so a thread that took them (options.values) keeps a consistent set."""
+
+  DEFAULTS = {
+    'ban_time': 1800,
+    'max_ban_time': 10000,
+    'ban_time_increment': True,
+    'max_attempts': 10,
+    'retry_window': 600,
+    'netban_ipv4': 32,
+    'netban_ipv6': 128,
+    'manage_external': 0,
+  }
+  # Single keys of old installations, read while F2B_OPTIONS does not exist
+  LEGACY_KEYS = {
+    'ban_time': 'F2B_BAN_TIME',
+    'max_ban_time': 'F2B_MAX_BAN_TIME',
+    'ban_time_increment': 'F2B_BAN_TIME_INCREMENT',
+    'max_attempts': 'F2B_MAX_ATTEMPTS',
+    'retry_window': 'F2B_RETRY_WINDOW',
+    'netban_ipv4': 'F2B_NETBAN_IPV4',
+    'netban_ipv6': 'F2B_NETBAN_IPV6',
+  }
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.values = self.normalize({})
+
+  def refresh(self, r):
+    """Reads the options of the UI; options it lacks are added to Redis once, existing ones are never overwritten"""
+    raw = r.get('F2B_OPTIONS')
+    if raw:
+      try:
+        stored = json.loads(raw)
+      except ValueError as ex:
+        self.logger.logWarn('F2B_OPTIONS is no valid JSON, keeping the current options: %s' % ex)
+        return
+      if not isinstance(stored, dict):
+        self.logger.logWarn('F2B_OPTIONS is no JSON object, keeping the current options')
+        return
+    else:
+      stored = {key: r.get(name) for key, name in self.LEGACY_KEYS.items()}
+      stored = {key: value for key, value in stored.items() if value is not None}
+
+    values = self.normalize(stored)
+    self.values = values
+    # First start or old installation: store the complete set, else only what the UI has not saved yet
+    missing = {key: value for key, value in values.items() if not raw or stored.get(key) is None}
+    if missing:
+      self.add_missing(r, missing)
+
+  def ban_time(self, ban_counter):
+    """Seconds of the next ban of a network that was banned ban_counter times before"""
+    values = self.values
+    ban_time = values['ban_time']
+    if values['ban_time_increment']:
+      # the exponent is capped, max_ban_time limits the result anyway
+      ban_time = ban_time * 2 ** min(ban_counter, 32)
+    return max(values['ban_time'], min(ban_time, values['max_ban_time']))
+
+  @classmethod
+  def normalize(cls, stored):
+    values = dict(stored)
+    for key, default in cls.DEFAULTS.items():
+      if isinstance(default, bool):
+        values[key] = cls.to_bool(stored.get(key), default)
+      else:
+        values[key] = cls.to_int(stored.get(key), default)
+    values['ban_time'] = max(1, values['ban_time'])
+    values['max_ban_time'] = max(1, values['max_ban_time'])
+    values['max_attempts'] = max(1, values['max_attempts'])
+    values['retry_window'] = max(1, values['retry_window'])
+    # A short prefix would ban whole regions or everything (/0)
+    values['netban_ipv4'] = min(32, max(8, values['netban_ipv4']))
+    values['netban_ipv6'] = min(128, max(8, values['netban_ipv6']))
+    values['manage_external'] = 1 if values['manage_external'] == 1 else 0
+    if not values.get('banlist_id'):
+      values['banlist_id'] = str(uuid.uuid4())
+    return values
+
+  @staticmethod
+  def to_int(value, default):
+    try:
+      return int(value)
+    except (TypeError, ValueError):
+      return default
+
+  @staticmethod
+  def to_bool(value, default):
+    if isinstance(value, bool):
+      return value
+    if isinstance(value, (int, float)):
+      return value != 0
+    if isinstance(value, str):
+      return value.strip().lower() in ('1', 'true', 'yes', 'on')
+    return default
+
+  def add_missing(self, r, missing):
+    # WATCH keeps a save of the UI between reading and writing
+    def fill(pipe):
+      raw = pipe.get('F2B_OPTIONS')
+      current = json.loads(raw) if raw else {}
+      if not isinstance(current, dict):
+        return
+      added = {key: value for key, value in missing.items() if current.get(key) is None}
+      if not added:
+        return
+      current.update(added)
+      pipe.multi()
+      pipe.set('F2B_OPTIONS', json.dumps(current, ensure_ascii=False))
+
+    try:
+      r.transaction(fill, 'F2B_OPTIONS')
+    except Exception as ex:
+      self.logger.logWarn('Could not add default options to F2B_OPTIONS: %s' % ex)

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -0,0 +1,88 @@
+import ipaddress
+import json
+import re
+
+
+class Regexes:
+  """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
+
+  DEFAULTS = {
+    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
+    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
+    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
+    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
+    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
+    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
+    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
+    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+  }
+  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
+  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
+  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.rules = []
+    self.source = None
+
+  def refresh(self, r):
+    """Compiles the rules of the UI again when they changed; invalid rules are skipped"""
+    raw = r.get('F2B_REGEX')
+    if not raw:
+      raw = json.dumps(self.DEFAULTS, ensure_ascii=False)
+      r.set('F2B_REGEX', raw, nx=True)
+    if raw == self.source:
+      return
+    try:
+      stored = json.loads(raw)
+    except ValueError as ex:
+      self.logger.logWarn('F2B_REGEX is no valid JSON, keeping the current rules: %s' % ex)
+      return
+    if not isinstance(stored, dict):
+      self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
+      return
+
+    rules = []
+    for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
+      try:
+        rules.append((str(rule_id), re.compile(regex)))
+      except (re.error, TypeError) as ex:
+        self.logger.logWarn('Skipping regex of rule id %s: %s' % (rule_id, ex))
+    self.rules = rules
+    self.source = raw
+
+  def match(self, line):
+    """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    for rule_id, regex in self.rules:
+      result = regex.search(line)
+      if result and result.lastindex and result.group(1):
+        return rule_id, result.group(1)
+    return None
+
+  @classmethod
+  def temporary_failure(cls, line):
+    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
+    result = cls.TEMPORARY_FAILURE.search(line)
+    return result.group(1) if result else None
+
+  @staticmethod
+  def parse_ip(text):
+    """IP address of a captured address, IPv4-mapped IPv6 as IPv4; a trailing colon (e.g. "1.2.3.4:") is
+    tolerated. None if the text is no address."""
+    for candidate in (text, text.rstrip(':')):
+      try:
+        ip = ipaddress.ip_address(candidate)
+      except ValueError:
+        continue
+      if ip.version == 6 and ip.ipv4_mapped:
+        return ip.ipv4_mapped
+      return ip
+    return None
+
+  @staticmethod
+  def sort_key(rule_id):
+    try:
+      return (0, int(rule_id))
+    except (TypeError, ValueError):
+      return (1, str(rule_id))

data/Dockerfiles/olefy/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/olefy/olefy.py

@@ -67,7 +67,7 @@ logger.debug('olefy listen address string: {} (type {})'.format(olefy_listen_add
 if not olefy_listen_addr_string:
     olefy_listen_addr = ""
 else:
-    addr_re = re.compile('[\[" \]]')
+    addr_re = re.compile(r'[\[" \]]')
     olefy_listen_addr = addr_re.sub('', olefy_listen_addr_string.replace("'", "")).split(',')
 
 # log runtime variables
@@ -206,8 +206,9 @@ class AIO(asyncio.Protocol):
         self.transport.close()
 
 
-# start the listeners
-loop = asyncio.get_event_loop()
+# start the listeners, Python >= 3.14 no longer creates a loop in get_event_loop()
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
 # each client connection will create a new protocol instance
 coro = loop.create_server(AIO, olefy_listen_addr, olefy_listen_port)
 server = loop.run_until_complete(coro)

data/Dockerfiles/rspamd/Dockerfile

@@ -2,7 +2,7 @@ FROM debian:trixie-slim
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG RSPAMD_VER=rspamd_4.2.1-1~b7a16ae
+ARG RSPAMD_VER=rspamd_4.2.3-1~50bde75
 ARG CODENAME=trixie
 ENV LC_ALL=C
 

data/Dockerfiles/sogo/Dockerfile

@@ -17,7 +17,7 @@ ARG SOPE_VERSION=SOPE-5.12.12
 # Security patches to apply (space-separated commit hashes)
 ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.19
+ARG GOSU_VERSION=1.20
 ENV LC_ALL=C
 
 # Install dependencies, build SOPE and SOGo, then clean up (all in one layer to minimize image size)

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -13,13 +13,11 @@ do
   sleep 3
 done
 
-# Wait for updated schema
+# Wait for updated schema (DB_VERSION from docker-compose.yml)
 DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
-while [[ "${DBV_NOW}" != "${DBV_NEW}" ]]; do
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
   echo "Waiting for schema update..."
   DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-  DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
   sleep 5
 done
 echo "DB schema is ${DBV_NOW}"

data/Dockerfiles/unbound/Dockerfile

@@ -1,10 +1,10 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
 # install unbound from alpine:edge to get security patches
 # minimum version is pinned
-RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main "unbound>=1.26.1-r0"
+RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main "unbound>=1.26.2-r0"
 
 # install other packages from regular alpine stable repo
 RUN apk add --update --no-cache \

data/Dockerfiles/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/conf/dovecot/auth/mailcowauth.php

@@ -101,7 +101,16 @@ if ($result === false){
   // Init Identity Provider
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
-  $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+  try {
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+  } catch (\Mailcow\Auth\IdpUnavailable $e) {
+    // temporary failure for Dovecot, no failed login
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
+    http_response_code(503); // Service Unavailable
+    echo json_encode($return);
+    session_destroy();
+    exit;
+  }
   if ($result) {
     error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
     set_sasl_log($post['username'], $post['real_rip'], $post['service']);

data/conf/dovecot/auth/passwd-verify.lua

@@ -41,6 +41,12 @@ function auth_password_verify(request, password)
     return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unreachable"
   end
 
+  -- 5xx: the backend or the identity provider cannot answer, a temporary failure and no failed login
+  if c >= 500 then
+    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unavailable"
+  end
+
   if c ~= 200 and c ~= 401 then
     dovecot.i_info("HTTP request failed with " .. tostring(c) .. " for user " .. request.user)
     return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Upstream error"

data/conf/dovecot/dovecot.conf

@@ -67,9 +67,11 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
+# cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
+# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
 passdb {
   driver = lua
-  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
 }
 # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
 service doveadm {

data/conf/phpfpm/crons/group-grants.php

@@ -0,0 +1,65 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Group Shares") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/group-grants.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+$sharing = \Mailcow\MailboxGroup\MailboxGroupModule::create($pdo, \Mailcow\Core\Context::system())->sharing;
+
+// Every minute: subscriptions of changed groups; failed ones are retried and logged by the full run
+if ((int)date('i') % 15 !== 0) {
+  $sharing->syncSubscriptions();
+  exit(0);
+}
+
+// Every 15 minutes: IMAP ACLs on new top-level folders and failed grants, SOGo grants of owners that opened SOGo meanwhile
+foreach ($sharing->reconcile() as $warning) {
+  logMsg("warning", $warning[0] . ": " . html_entity_decode(implode(", ", array_slice($warning, 1))));
+}

data/conf/phpfpm/crons/identity-sync.php

@@ -0,0 +1,95 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Identity Sync") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+
+// The legacy mailbox functions run as admin
+$_SESSION['mailcow_cc_username'] = "admin";
+$_SESSION['mailcow_cc_role'] = "admin";
+$_SESSION['acl']['tls_policy'] = "1";
+$_SESSION['acl']['quarantine_notification'] = "1";
+$_SESSION['acl']['quarantine_category'] = "1";
+$_SESSION['acl']['ratelimit'] = "1";
+$_SESSION['acl']['sogo_access'] = "1";
+$_SESSION['acl']['protocol_access'] = "1";
+$_SESSION['acl']['mailbox_relayhost'] = "1";
+$_SESSION['acl']['unlimited_quota'] = "1";
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/identity-sync.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+// Users, offboarding and groups of Keycloak or LDAP; requested runs first, else every sync_interval minutes.
+// The IdP is only contacted when a run is due.
+$iam_settings = identity_provider('get');
+$identity = \Mailcow\Identity\IdentityModule::create($pdo, \Mailcow\Core\Context::system(), $iam_settings, function () {
+  return identity_provider('init');
+});
+$run = $identity->sync->runDue();
+if ($run !== null) {
+  $summary = $run['summary'];
+  if ($run['status'] == 'error') {
+    logMsg("err", ucfirst($run['mode']) . " failed: " . $summary['error']);
+  } else {
+    $counts = array();
+    foreach ($summary['counts'] as $key => $count) {
+      if ($count > 0) $counts[] = $key . "=" . $count;
+    }
+    logMsg(($run['status'] == 'blocked') ? "warning" : "info", ucfirst($run['mode']) . " " . $run['status'] . ": " . (empty($counts) ? "no changes" : implode(", ", $counts)));
+    foreach ($summary['details']['conflicts'] as $conflict) {
+      logMsg("warning", $conflict);
+    }
+    foreach ($summary['errors'] ?? array() as $error) {
+      logMsg("err", $error);
+    }
+  }
+}
+session_destroy();

data/conf/phpfpm/crons/keycloak-sync.php

@@ -1,231 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "Keycloak Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "keycloak" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Keycloak Provider
-$iam_provider = identity_provider('init');
-
-// Loop until all users have been retrieved
-while (true) {
-  // Get admin access token
-  $admin_token = identity_provider("get-keycloak-admin-token");
-
-  // Make the API request to retrieve the users
-  $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users?first=$start&max=$max";
-  $ch = curl_init();
-  curl_setopt($ch, CURLOPT_URL, $url);
-  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-  curl_setopt($ch, CURLOPT_HTTPHEADER, [
-    "Content-Type: application/json",
-    "Authorization: Bearer " . $admin_token
-  ]);
-  $response = curl_exec($ch);
-  $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-  curl_close($ch);
-
-  if ($code != 200){
-    logMsg("err", "Received HTTP {$code}");
-    session_destroy();
-    exit;
-  }
-  try {
-    $response = json_decode($response, true);
-  } catch (Exception $e) {
-    logMsg("err", $e->getMessage());
-    break;
-  }
-  if (!is_array($response)){
-    logMsg("err", "Received malformed response from keycloak api");
-    break;
-  }
-  if (count($response) == 0) {
-    break;
-  }
-
-  // Process the batch of users
-  foreach ($response as $user) {
-    if (empty($user['email'])){
-      logMsg("warning", "No email address in keycloak found for user " . $user['name']);
-      continue;
-    }
-
-    // try get mailbox user
-    $stmt = $pdo->prepare("SELECT
-      mailbox.*,
-      domain.active AS d_active
-      FROM `mailbox`
-      INNER JOIN domain on mailbox.domain = domain.domain
-      WHERE `kind` NOT REGEXP 'location|thing|group'
-        AND `username` = :user");
-    $stmt->execute(array(':user' => $user['email']));
-    $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-    // check if matching attribute mapping exists
-    $user_template = $user['attributes']['mailcow_template'][0];
-    $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-    $_SESSION['access_all_exception'] = '1';
-    if (!$row && intval($iam_settings['import_users']) == 1){
-      if ($mapper_key === false){
-        if (!empty($iam_settings['default_template'])) {
-          $mbox_template = $iam_settings['default_template'];
-          logMsg("warning", "Using default template for user " . $user['email']);
-        } else {
-          logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-          continue;
-        }
-      } else {
-        $mbox_template = $iam_settings['templates'][$mapper_key];
-      }
-      // mailbox user does not exist, create...
-      logMsg("info", "Creating user " . $user['email']);
-      $create_res = mailbox('add', 'mailbox_from_template', array(
-        'domain' => explode('@', $user['email'])[1],
-        'local_part' => explode('@', $user['email'])[0],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'authsource' => 'keycloak',
-        'template' => $mbox_template
-      ));
-      if (!$create_res){
-        logMsg("err", "Could not create user " . $user['email']);
-        continue;
-      }
-    } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "keycloak") {
-      if ($mapper_key === false){
-        logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-        continue;
-      }
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-      // mailbox user does exist, sync attribtues...
-      logMsg("info", "Syncing attributes for user " . $user['email']);
-      mailbox('edit', 'mailbox_from_template', array(
-        'username' => $user['email'],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'template' => $mbox_template
-      ));
-    } else {
-      // skip mailbox user
-      logMsg("info", "Skipping user " . $user['email']);
-    }
-    $_SESSION['access_all_exception'] = '0';
-
-    sleep(0.025);
-  }
-
-  // Update the pagination variables for the next batch
-  $start += $max;
-  sleep(1);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/conf/phpfpm/crons/ldap-sync.php

@@ -1,198 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "LDAP Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "ldap" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Provider
-$iam_provider = identity_provider('init');
-
-// Get ldap users
-$ldap_query = $iam_provider->query();
-if (!empty($iam_settings['filter'])) {
-  $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
-}
-$response = $ldap_query->where($iam_settings['username_field'], "*")
-  ->where($iam_settings['attribute_field'], "*")
-  ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname'])
-  ->paginate($max);
-
-// Process the users
-foreach ($response as $user) {
-  // try get mailbox user
-  $stmt = $pdo->prepare("SELECT
-    mailbox.*,
-    domain.active AS d_active
-    FROM `mailbox`
-    INNER JOIN domain on mailbox.domain = domain.domain
-    WHERE `kind` NOT REGEXP 'location|thing|group'
-      AND `username` = :user");
-  $stmt->execute(array(':user' => $user[$iam_settings['username_field']][0]));
-  $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-  // check if matching attribute mapping exists
-  $user_template = $user[$iam_settings['attribute_field']][0];
-  $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-  if (empty($user[$iam_settings['username_field']][0])){
-    logMsg("warning", "Skipping user " . $user['displayname'][0] . " due to empty LDAP ". $iam_settings['username_field'] . " property.");
-    continue;
-  }
-
-  $_SESSION['access_all_exception'] = '1';
-  if (!$row && intval($iam_settings['import_users']) == 1){
-    if ($mapper_key === false){
-      if (!empty($iam_settings['default_template'])) {
-        $mbox_template = $iam_settings['default_template'];
-      } else {
-        logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-        continue;
-      }
-    } else {
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-    }
-    // mailbox user does not exist, create...
-    logMsg("info", "Creating user " .  $user[$iam_settings['username_field']][0]);
-    $create_res = mailbox('add', 'mailbox_from_template', array(
-      'domain' => explode('@',  $user[$iam_settings['username_field']][0])[1],
-      'local_part' => explode('@',  $user[$iam_settings['username_field']][0])[0],
-      'name' => $user['displayname'][0],
-      'authsource' => 'ldap',
-      'template' => $mbox_template
-    ));
-    if (!$create_res){
-      logMsg("err", "Could not create user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-  } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "ldap") {
-    if ($mapper_key === false){
-      logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-    $mbox_template = $iam_settings['templates'][$mapper_key];
-    // mailbox user does exist, sync attribtues...
-    logMsg("info", "Syncing attributes for user " . $user[$iam_settings['username_field']][0]);
-    mailbox('edit', 'mailbox_from_template', array(
-      'username' =>  $user[$iam_settings['username_field']][0],
-      'name' => $user['displayname'][0],
-      'template' => $mbox_template
-    ));
-  } else {
-    // skip mailbox user
-    logMsg("info", "Skipping user " .  $user[$iam_settings['username_field']][0]);
-  }
-  $_SESSION['access_all_exception'] = '0';
-
-  sleep(0.025);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/web/admin/mailbox.php

@@ -10,6 +10,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 $js_minifier->add('/web/js/site/imapsync.js');
+$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -39,6 +40,7 @@ $template_data = [
   'global_filters' => mailbox('get', 'global_filter_details'),
   'domains' => $domains,
   'mailboxes' => $mailboxes,
+  'mbox_templates' => mailbox('get', 'mailbox_templates'),
   'lang_mailbox' => json_encode($lang['mailbox']),
   'lang_rl' => json_encode($lang['ratelimit']),
   'lang_edit' => json_encode($lang['edit']),

data/web/admin/system.php

@@ -12,6 +12,7 @@ $fido2_data = fido2(array("action" => "get_friendly_names"));
 $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
+$js_minifier->add('/web/js/site/identity.js');
 
 // all domains
 $domains = mailbox('get', 'domains');

data/web/api/openapi.yaml

@@ -1260,9 +1260,123 @@ paths:
                 tls_enforce_out:
                   description: force oubound tmail tls encryption
                   type: boolean
+                shared:
+                  description: >-
+                    shared mailbox: no login of its own, members of groups open it through shares;
+                    password and password2 may be empty (a random password is set)
+                  type: boolean
               type: object
       summary: Create mailbox
 
+  /api/v1/add/mailbox-group:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - log:
+                        - mailbox_group
+                        - add
+                        - group
+                        - domain: example.org
+                          name: Sales
+                          local_part: sales
+                          members:
+                            - alice@example.org
+                      msg:
+                        - group_added
+                        - Sales
+                      type: success
+              schema:
+                properties:
+                  log:
+                    description: contains request object
+                    items: {}
+                    type: array
+                  msg:
+                    items: {}
+                    type: array
+                  type:
+                    enum:
+                      - success
+                      - danger
+                      - warning
+                      - error
+                    type: string
+                type: object
+          description: OK
+          headers: {}
+      tags:
+        - Groups
+      description: >-
+        Create a mailbox group (admins; domain admins with the "mailbox_groups"
+        ACL for their domains). With an address the group is a distribution
+        list; its alias is maintained by the group and read-only in the alias
+        API.
+      operationId: Create group
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                domain: example.org
+                name: Sales
+                local_part: sales
+                members:
+                  - alice@example.org
+                  - bob@example.org
+                send_as:
+                  - shared@example.org
+                sender_allowed: "1"
+                active: "1"
+              properties:
+                domain:
+                  description: domain of the group (required)
+                  type: string
+                name:
+                  description: unique per domain, max. 100 characters
+                  type: string
+                description:
+                  description: max. 255 characters
+                  type: string
+                local_part:
+                  description: local part of the group address (domain of the group), empty = no address
+                  type: string
+                address:
+                  description: full group address instead of local_part, empty = no address
+                  type: string
+                members:
+                  description: >-
+                    mailboxes (array or list separated by commas, spaces or
+                    newlines); replaces the manual members you may access
+                  type: array
+                send_as:
+                  description: addresses (mailbox or alias) or @domain the members may send as
+                  type: array
+                sender_allowed:
+                  description: members may send as the group address
+                  type: boolean
+                sogo_visible:
+                  description: show the group address as SOGo identity (needs sender_allowed)
+                  type: boolean
+                internal:
+                  description: only accept mail from senders of the domain
+                  type: boolean
+                template_id:
+                  description: mailbox template applied to the members (admin only), empty = none
+                  type: number
+                priority:
+                  description: highest priority wins if a mailbox is in several groups with a template (admin only)
+                  type: number
+                active:
+                  type: boolean
+              type: object
+      summary: Create group
   /api/v1/add/oauth2-client:
     post:
       responses:
@@ -2673,6 +2787,103 @@ paths:
                   type: object
               type: object
       summary: Delete mailbox
+  /api/v1/delete/mailbox-group:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - log:
+                        - mailbox_group
+                        - delete
+                        - group
+                        - id:
+                            - "7"
+                      msg:
+                        - group_removed
+                        - Sales
+                      type: success
+              schema:
+                properties:
+                  log:
+                    description: contains request object
+                    items: {}
+                    type: array
+                  msg:
+                    items: {}
+                    type: array
+                  type:
+                    enum:
+                      - success
+                      - danger
+                      - warning
+                      - error
+                    type: string
+                type: object
+          description: OK
+          headers: {}
+      tags:
+        - Groups
+      description: >-
+        Delete groups by ID. Members keep their settings; their group alias and
+        send-as rights of the group are removed.
+      operationId: Delete groups
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                - "7"
+              properties:
+                items:
+                  description: contains list of group IDs you want to delete
+                  type: object
+              type: object
+      summary: Delete groups
+  /api/v1/delete/identity-link:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - log:
+                        - identity_provider
+                        - delete
+                        - link
+                        - - alice@example.org
+                      msg:
+                        - iam_link_removed
+                        - alice@example.org
+                      type: success
+          description: OK
+          headers: {}
+      tags:
+        - Identity Provider
+      description: >-
+        Removes the link between mailboxes and their Identity Provider users (admins only). The mailbox is
+        linked again by email on the next sync run or login, e.g. after its address was given to another
+        Identity Provider user.
+      operationId: Delete identity links
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                - alice@example.org
+              items:
+                type: string
+              type: array
+      summary: Delete identity links
   /api/v1/delete/mailq:
     post:
       responses:
@@ -4025,6 +4236,9 @@ paths:
                     sogo_access:
                       description: is access to SOGo webmail active or not
                       type: boolean
+                    shared:
+                      description: shared mailbox, no login of its own (IMAP, SMTP, SOGo, app passwords, mailcow UI)
+                      type: boolean
                   type: object
                 items:
                   description: contains list of mailboxes you want update
@@ -4111,6 +4325,219 @@ paths:
                   type: object
               type: object
       summary: Update mailbox custom attributes
+  /api/v1/edit/mailbox-group:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - log:
+                        - mailbox_group
+                        - edit
+                        - group
+                        - id:
+                            - "7"
+                          add_members:
+                            - carol@example.org
+                      msg:
+                        - group_modified
+                        - Sales
+                      type: success
+              schema:
+                properties:
+                  log:
+                    description: contains request object
+                    items: {}
+                    type: array
+                  msg:
+                    items: {}
+                    type: array
+                  type:
+                    enum:
+                      - success
+                      - danger
+                      - warning
+                      - error
+                    type: string
+                type: object
+          description: OK
+          headers: {}
+      tags:
+        - Groups
+      description: >-
+        Update groups. Attributes you do not send keep their value. Members
+        synced from the identity provider cannot be removed here
+        (group_member_idp).
+      operationId: Update groups
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                items:
+                  - "7"
+                attr:
+                  add_members:
+                    - carol@example.org
+              properties:
+                attr:
+                  properties:
+                    name:
+                      description: unique per domain, max. 100 characters
+                      type: string
+                    description:
+                      description: max. 255 characters
+                      type: string
+                    local_part:
+                      description: local part of the group address (domain of the group), empty = no address
+                      type: string
+                    address:
+                      description: full group address instead of local_part, empty = no address
+                      type: string
+                    members:
+                      description: >-
+                        mailboxes (array or list separated by commas, spaces or
+                        newlines); replaces the manual members you may access
+                      type: array
+                    send_as:
+                      description: addresses (mailbox or alias) or @domain the members may send as
+                      type: array
+                    sender_allowed:
+                      description: members may send as the group address
+                      type: boolean
+                    sogo_visible:
+                      description: show the group address as SOGo identity (needs sender_allowed)
+                      type: boolean
+                    internal:
+                      description: only accept mail from senders of the domain
+                      type: boolean
+                    template_id:
+                      description: mailbox template applied to the members (admin only), empty = none
+                      type: number
+                    priority:
+                      description: highest priority wins if a mailbox is in several groups with a template (admin only)
+                      type: number
+                    active:
+                      type: boolean
+                    add_members:
+                      description: mailboxes to add
+                      type: array
+                    remove_members:
+                      description: manual members to remove
+                      type: array
+                    reapply_template:
+                      description: apply the effective templates to all members again (admin only)
+                      type: boolean
+                    imap_grants:
+                      description: >-
+                        replaces the mailbox access of the members (Dovecot ACL on every folder of the
+                        mailbox, shown as Shared/<mailbox>); rights read, write or full; you need access
+                        to each mailbox; subscribe = the group-grants cron subscribes the folders for every
+                        member once (within 15 minutes)
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          owner:
+                            type: string
+                          rights:
+                            type: string
+                            enum: [read, write, full]
+                          subscribe:
+                            type: boolean
+                            default: false
+                      example:
+                        - owner: shared@example.org
+                          rights: write
+                          subscribe: true
+                    sogo_grants:
+                      description: >-
+                        replaces the SOGo folders shared with the members; folder = SOGo folder ID
+                        (default personal); pending until the mailbox opened SOGo once; subscribe = the
+                        group-grants cron subscribes every member once (within 15 minutes)
+                      type: array
+                      items:
+                        type: object
+                        properties:
+                          owner:
+                            type: string
+                          type:
+                            type: string
+                            enum: [Calendar, Contacts]
+                          folder:
+                            type: string
+                            default: personal
+                          access:
+                            type: string
+                            enum: [read, write]
+                          subscribe:
+                            type: boolean
+                            default: false
+                      example:
+                        - owner: shared@example.org
+                          type: Calendar
+                          access: write
+                          subscribe: true
+                  type: object
+                items:
+                  description: contains list of group IDs you want to update
+                  type: object
+              type: object
+      summary: Update groups
+  /api/v1/edit/identity-provider-sync:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - log:
+                        - identity_provider
+                        - edit
+                        - sync
+                        - mode: preview
+                      msg: iam_sync_requested
+                      type: success
+          description: OK
+          headers: {}
+      tags:
+        - Identity Provider
+      description: >-
+        Queues a sync run of the Identity Provider (Keycloak and LDAP, admins only). The identity-sync cron
+        starts it within a minute. `preview` lists the changes without applying them, `run` applies them,
+        `force` applies them without the change limit (offboarding_limit).
+      operationId: Request identity sync run
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                items:
+                  - sync
+                attr:
+                  mode: preview
+              properties:
+                items:
+                  type: array
+                  default: ["sync"]
+                attr:
+                  type: object
+                  properties:
+                    mode:
+                      type: string
+                      enum: [preview, run, force]
+                      default: run
+              type: object
+      summary: Request identity sync run
   /api/v1/edit/mailq:
     post:
       responses:
@@ -5898,6 +6325,203 @@ paths:
       description: You can list all mailboxes existing in system.
       operationId: Get mailboxes
       summary: Get mailboxes
+  "/api/v1/get/mailbox-group/{id}":
+    get:
+      parameters:
+        - description: id of the group, or "all" (optionally followed by /domain1,domain2)
+          example: all
+          in: path
+          name: id
+          required: true
+          schema:
+            type: string
+        - description: e.g. api-key-string
+          example: api-key-string
+          in: header
+          name: X-API-Key
+          required: false
+          schema:
+            type: string
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    - id: 7
+                      domain: example.org
+                      name: Sales
+                      description: Sales team
+                      address: sales@example.org
+                      sender_allowed: 1
+                      sogo_visible: 0
+                      internal: 0
+                      template_id: 3
+                      template: Staff
+                      priority: 10
+                      idp_ref: null
+                      members:
+                        - username: alice@example.org
+                          source: manual
+                        - username: bob@example.org
+                          source: idp
+                      send_as:
+                        - shared@example.org
+                      imap_grants:
+                        - owner: shared@example.org
+                          rights: write
+                          subscribe: 1
+                          state: applied
+                          last_error: null
+                      sogo_grants:
+                        - owner: shared@example.org
+                          type: Calendar
+                          folder: personal
+                          access: write
+                          subscribe: 1
+                          state: applied
+                          last_error: null
+                      active: 1
+                      created: "2026-10-25 12:00:00"
+                      modified: null
+                      can_edit: true
+          description: OK
+          headers: {}
+      tags:
+        - Groups
+      description: >-
+        Groups visible to the caller (admins all, domain admins their domains).
+      operationId: Get groups
+      summary: Get groups
+  "/api/v1/get/mailbox-group/member/{username}":
+    get:
+      parameters:
+        - description: mailbox
+          example: alice@example.org
+          in: path
+          name: username
+          required: true
+          schema:
+            type: string
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    groups:
+                      - id: 7
+                        domain: example.org
+                        name: Sales
+                        address: sales@example.org
+                        sender_allowed: 1
+                        active: 1
+                        priority: 10
+                        template: Staff
+                        source: manual
+                    template: Staff
+                    send_as:
+                      - sales@example.org
+                      - shared@example.org
+          description: OK
+          headers: {}
+      tags:
+        - Groups
+      description: >-
+        Groups of a mailbox, its effective group template and the addresses it
+        may send as through groups.
+      operationId: Get group memberships
+      summary: Get group memberships
+  /api/v1/get/identity-provider/sync-status:
+    get:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    provider: keycloak
+                    can_sync: true
+                    sync_enabled: true
+                    counts:
+                      accounts: 120
+                      linked: 118
+                      idp_disabled: 2
+                      missing: 1
+                    runs:
+                      - id: 42
+                        provider: keycloak
+                        mode: run
+                        status: ok
+                        requested_by: ""
+                        created: "2026-10-25 12:00:00"
+                        started: "2026-10-25 12:00:01"
+                        finished: "2026-10-25 12:00:03"
+                        summary:
+                          counts:
+                            creates: 1
+                            deactivate: 1
+                            memberships: 2
+                          details:
+                            creates:
+                              - new@example.org
+                            deactivate:
+                              - left@example.org
+                            conflicts: []
+                          errors: []
+                          blocked: false
+                          mode: run
+          description: OK
+          headers: {}
+      tags:
+        - Identity Provider
+      description: >-
+        Sync status (admins only): mailboxes of the Identity Provider and their links, and the last sync runs
+        with their summary. Run status: pending, running, ok, blocked (change limit), error (listing failed,
+        nothing was changed).
+      operationId: Get identity sync status
+      summary: Get identity sync status
+  "/api/v1/get/identity-link/{username}":
+    get:
+      parameters:
+        - description: mailbox
+          example: alice@example.org
+          in: path
+          name: username
+          required: true
+          schema:
+            type: string
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    provider: keycloak
+                    external_id: 8d5da28d-7541-428b-8356-a9704f73e7f4
+                    idp_disabled: 0
+                    missing_since: null
+          description: OK
+          headers: {}
+      tags:
+        - Identity Provider
+      description: >-
+        Identity Provider user a mailbox is linked to (admins, and domain admins for their mailboxes).
+        `idp_disabled` = login was disabled by the sync, `missing_since` = not listed by the Identity Provider.
+      operationId: Get identity link
+      summary: Get identity link
   /api/v1/get/mailq/all:
     get:
       responses:
@@ -7199,7 +7823,7 @@ paths:
                       description: An optional LDAP search filter to refine which users can authenticate.
                       type: string
                     attribute_field:
-                      description: Specifies an LDAP attribute that holds a specific value which can be mapped to a mailbox template using the Attribute Mapping section. Required if `authsource` is ldap.
+                      description: (Optional) Specifies an LDAP attribute that holds a specific value which can be mapped to a mailbox template using the Attribute Mapping section.
                       type: string
                     binddn:
                       description: The Distinguished Name (DN) of the LDAP user that will be used to authenticate and perform LDAP searches. This account should have sufficient permissions to read the required attributes. Required if `authsource` is ldap.
@@ -7220,6 +7844,53 @@ paths:
                       description: Specifies the OIDC scopes requested during authentication.
                       type: string
                       default: "openid profile email mailcow_template"
+                    offboarding:
+                      description: (Keycloak, LDAP) Users disabled in the Identity Provider, or missing in two complete sync runs, get login disabled (mailbox active = 2, mail is still received). Only mailboxes disabled by the sync are enabled again.
+                      type: boolean
+                      default: false
+                    offboarding_limit:
+                      description: (Keycloak, LDAP) Sync runs that would disable more mailboxes or remove more group memberships than this share in percent (at least 5) are blocked.
+                      type: number
+                      default: 10
+                    group_sync:
+                      description: (Keycloak, LDAP) Syncs the members of Identity Provider groups to mailbox groups. Members added in mailcow stay.
+                      type: boolean
+                      default: false
+                    group_autocreate:
+                      description: (Keycloak, LDAP) Creates a mailbox group for every Identity Provider group without one. The group mail becomes its address if it is free.
+                      type: boolean
+                      default: false
+                    group_default_domain:
+                      description: (Keycloak, LDAP) Domain of new groups whose mail is not in a mailcow domain.
+                      type: string
+                    group_paths:
+                      description: (Keycloak) Comma-separated group paths, e.g. `/Sales` or `/mailcow/*` (all groups below /mailcow). Needs the `query-groups` permission.
+                      type: string
+                    id_attribute:
+                      description: (LDAP) Attribute with a stable user id. Empty = objectGUID, entryUUID, nsUniqueId or ipaUniqueID. Changing it links all mailboxes again by email.
+                      type: string
+                    disabled_filter:
+                      description: (LDAP) Filter for disabled users. Active Directory accounts with ACCOUNTDISABLE in userAccountControl are always disabled.
+                      type: string
+                    group_basedn:
+                      description: (LDAP) Base DN of the group search, default `basedn`.
+                      type: string
+                    group_filter:
+                      description: (LDAP) Filter for the synced groups, required for group sync.
+                      type: string
+                    group_name_attribute:
+                      description: (LDAP) Attribute with the group name.
+                      type: string
+                      default: cn
+                    group_member_mode:
+                      description: "(LDAP) Source of the memberships: memberof (memberOf of the users), member (member attribute of the groups) or ad_chain (Active Directory incl. nested groups)."
+                      type: string
+                      enum: [memberof, member, ad_chain]
+                      default: memberof
+                    group_member_attribute:
+                      description: (LDAP) Member attribute for group_member_mode member, e.g. member or uniqueMember.
+                      type: string
+                      default: member
             examples:
               keycloak:
                 value:
@@ -7320,6 +7991,8 @@ tags:
     description: Use mailcow as a oAuth server
   - name: API Keys
     description: Manage API keys
+  - name: Groups
+    description: Mailbox groups (distribution lists, send-as rights, templates)
   - name: Routing
     description: Define your own email routes
   - name: Resources

data/web/domainadmin/mailbox.php

@@ -10,6 +10,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 $js_minifier->add('/web/js/site/imapsync.js');
+$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');

data/web/edit.php

@@ -99,6 +99,22 @@ if (isset($_SESSION['mailcow_cc_role'])) {
         $template = 'edit/oauth2client.twig';
         $template_data = ['oauth2client' => $oauth2client];
     }
+    elseif (isset($_GET['mailbox-group']) &&
+      is_numeric($_GET['mailbox-group'])) {
+        $result = \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->getGroup($_GET['mailbox-group']);
+        $template = 'edit/mailbox-group.twig';
+        // Member candidates: mailboxes of the domains the caller manages
+        $mailboxes = [];
+        foreach (mailbox('get', 'domains') as $domain) {
+          foreach (mailbox('get', 'mailboxes', $domain) as $mailbox) {
+            $mailboxes[] = $mailbox;
+          }
+        }
+        $template_data = [
+          'mailboxes' => $mailboxes,
+          'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
+        ];
+    }
     elseif (isset($_GET['api-key']) &&
       is_numeric($_GET['api-key'])) {
         $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
@@ -141,6 +157,8 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'get_tls_policy' => $get_tls_policy,
           'rlyhosts' => $rlyhosts,
           'sender_acl_handles' => mailbox('get', 'sender_acl_handles', $mailbox),
+          'mailbox_groups' => \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->getMemberships($mailbox),
+          'identity_link' => \Mailcow\Identity\IdentityModule::fromGlobals()->api->getLink($mailbox),
           'user_acls' => acl('get', 'user', $mailbox),
           'mailbox_details' => $result,
           'iam_settings' => $iam_settings,
@@ -237,6 +255,7 @@ else {
 }
 
 $js_minifier->add('/web/js/site/imapsync.js');
+$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/edit.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 

data/web/inc/footer.inc.php

@@ -73,6 +73,7 @@ $globalVariables = [
   'lang_success' => json_encode($lang['success']),
   'lang_danger' => json_encode($lang['danger']),
   'lang_syncjobs' => json_encode($lang['syncjobs']),
+  'lang_groups' => json_encode($lang['groups']),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.auth.inc.php

@@ -36,7 +36,18 @@ function check_login($user, $pass, $extra = null) {
 
   // Try validate user
   if (!isset($role) || $role == "user") {
-    $result = user_login($user, $pass, $extra);
+    try {
+      $result = user_login($user, $pass, $extra);
+    } catch (\Mailcow\Auth\IdpUnavailable $e) {
+      // no failed login, netfilter must not count it
+      error_log("mailcow UI: Identity provider unavailable for " . $user . " by " . $_SERVER['REMOTE_ADDR'] . ": " . $e->getMessage());
+      $_SESSION['return'][] =  array(
+        'type' => 'danger',
+        'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
+        'msg' => 'iam_unavailable'
+      );
+      return false;
+    }
     if ($result !== false) {
       $real_rip = ($_SERVER['HTTP_X_REAL_IP'] ?? $_SERVER['REMOTE_ADDR']);
       set_sasl_log($user, $real_rip, $extra['service']);
@@ -262,12 +273,20 @@ function user_login($user, $pass, $extra = null){
 
   // check if user has access to service (imap, smtp, pop3, sieve) if service is set
   $row['attributes'] = json_decode($row['attributes'], true);
+  // Shared mailboxes have no login of their own, members open them through group shares
+  if (($row['attributes']['shared'] ?? '0') == '1') {
+    return false;
+  }
   if ($extra['service'] != 'NONE') {
     $key = strtolower($extra['service']) . "_access";
     if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
       return false;
     }
   }
+  // IdP accounts: refuse inactive mailboxes and domains before asking the IdP (no template side effects)
+  if ($row['authsource'] != 'mailcow' && ($row['active'] != '1' || $row['d_active'] != '1')) {
+    return false;
+  }
   switch ($row['authsource']) {
     case 'keycloak':
       // user authsource is keycloak, try using via rest flow
@@ -462,6 +481,7 @@ function apppass_login($user, $pass, $extra = null){
       AND `mailbox`.`active` = '1'
       AND `domain`.`active` = '1'
       AND `app_passwd`.`active` = '1'
+      AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
       AND `app_passwd`.`mailbox` = :user"
   );
   // fetch password data
@@ -507,11 +527,14 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     return false;
   }
   if (!$iam_provider) {
-    return false;
+    throw new \Mailcow\Auth\IdpUnavailable('Keycloak is not configured completely');
   }
 
   // get access_token for service account of mailcow client
   $admin_token = identity_provider("get-keycloak-admin-token");
+  if (!$admin_token) {
+    throw new \Mailcow\Auth\IdpUnavailable('Could not get an admin token from Keycloak');
+  }
 
   // get the mailcow_password attribute from keycloak user
   $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users";
@@ -525,15 +548,23 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
       'Authorization: Bearer ' . $admin_token,
       'Content-Type: application/json'
   ));
-  $user_res = json_decode(curl_exec($curl), true)[0];
+  if (intval($iam_settings['ignore_ssl_error']) == 1) {
+    curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
+    curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
+  }
+  $users = json_decode(curl_exec($curl), true);
   $code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
   curl_close($curl);
+  // the password is checked below, a failed request is never a wrong password
   if ($code != 200) {
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $user, '*', 'Identity Provider returned HTTP ' . $code),
-      'msg' => 'generic_server_error'
-    );
+    throw new \Mailcow\Auth\IdpUnavailable('Keycloak returned HTTP ' . $code);
+  }
+  // exact=true can still return several users with that email
+  if (!is_array($users) || count($users) != 1) {
+    return false;
+  }
+  $user_res = $users[0];
+  if (empty($user_res['enabled'])) {
     return false;
   }
   if (!isset($user_res['attributes']['mailcow_password']) || !is_array($user_res['attributes']['mailcow_password'])){
@@ -560,18 +591,28 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   }
 
   // get mapped template
-  $user_template = $user_res['attributes']['mailcow_template'][0];
-  $mapper_key = array_search($user_template, $iam_settings['mappers']);
+  $name = trim(($user_res['firstName'] ?? '') . ' ' . ($user_res['lastName'] ?? ''));
+  $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
+  $mapped_template = $idp_login->mappedTemplate($user_res['attributes']['mailcow_template'] ?? array());
 
   if (!$create) {
+    if (!$idp_login->verifyLink('keycloak', $user, (string)($user_res['id'] ?? ''))) {
+      $_SESSION['return'][] =  array(
+        'type' => 'danger',
+        'log' => array(__FUNCTION__, $user, '*', 'The mailbox is linked to another IdP user'),
+        'msg' => 'generic_server_error'
+      );
+      return false;
+    }
     // login success
-    if ($mapper_key !== false) {
+    $update_template = $idp_login->templateFor($user, $mapped_template);
+    if ($update_template !== null) {
       // update user
       $_SESSION['access_all_exception'] = '1';
       mailbox('edit', 'mailbox_from_template', array(
         'username' => $user,
-        'name' => $user_res['name'],
-        'template' => $iam_settings['templates'][$mapper_key]
+        'name' => $name,
+        'template' => $update_template
       ));
       $_SESSION['access_all_exception'] = '0';
     }
@@ -590,7 +631,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     return false;
   }
   // check if matching attribute exist
-  if (empty($iam_settings['mappers']) || !$user_template || $mapper_key === false) {
+  if ($mapped_template === null) {
     if (!empty($iam_settings['default_template'])) {
       $mbox_template = $iam_settings['default_template'];
     } else {
@@ -602,7 +643,16 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
       return false;
     }
   } else {
-    $mbox_template = $iam_settings['templates'][$mapper_key];
+    $mbox_template = $mapped_template;
+  }
+
+  if (!$idp_login->mayCreate('keycloak', $user, (string)($user_res['id'] ?? ''))) {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $user, '*', 'The IdP user is linked to another mailbox'),
+      'msg' => 'generic_server_error'
+    );
+    return false;
   }
 
   // create mailbox
@@ -610,7 +660,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   $create_res = mailbox('add', 'mailbox_from_template', array(
     'domain' => explode('@', $user)[1],
     'local_part' => explode('@', $user)[0],
-    'name' => $user_res['name'],
+    'name' => $name,
     'authsource' => 'keycloak',
     'template' => $mbox_template
   ));
@@ -624,6 +674,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
     );
     return false;
   }
+  $idp_login->storeLink('keycloak', $user, (string)($user_res['id'] ?? ''));
 
   return 'user';
 }
@@ -646,7 +697,8 @@ function ldap_mbox_login($user, $pass, $extra = null){
     return false;
   }
   if (!$iam_provider) {
-    return false;
+    // identity_provider('init') could not connect
+    throw new \Mailcow\Auth\IdpUnavailable('No connection to the LDAP server');
   }
 
   try {
@@ -655,47 +707,42 @@ function ldap_mbox_login($user, $pass, $extra = null){
       $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
     }
     $ldap_query = $ldap_query->where($iam_settings['username_field'], '=', $user)
-      ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn']);
+      ->select(array_values(array_filter(array_merge([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn'],
+        \Mailcow\Identity\Directory\LdapDirectory::idAttributes($iam_settings['id_attribute'] ?? '')))));
 
     $user_res = $ldap_query->firstOrFail();
-  } catch (Exception $e) {
-    // clear $_SESSION['return'] to not leak data
-    $_SESSION['return'] = array();
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
-      'msg' => 'generic_server_error'
-    );
+  } catch (\LdapRecord\Query\ObjectNotFoundException $e) {
     return false;
-  }
-  try {
-    if (!$iam_provider->auth()->attempt($user_res['dn'], $pass)) {
-      return false;
-    }
   } catch (Exception $e) {
-    // clear $_SESSION['return'] to not leak data
-    $_SESSION['return'] = array();
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $user, '*', $e->getMessage()),
-      'msg' => 'generic_server_error'
-    );
+    throw new \Mailcow\Auth\IdpUnavailable('LDAP search failed: ' . $e->getMessage(), 0, $e);
+  }
+  $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
+  if (!$idp_login->verifyLdapPassword($iam_provider, (string)$user_res['dn'], (string)$pass)) {
     return false;
   }
 
   // get mapped template
-  $user_template = $user_res[$iam_settings['attribute_field']][0];
-  $mapper_key = array_search($user_template, $iam_settings['mappers']);
+  $mapped_template = $idp_login->mappedTemplate($user_res[$iam_settings['attribute_field']] ?? array());
+  $external_id = \Mailcow\Identity\Directory\LdapDirectory::entryId($user_res, $iam_settings['id_attribute'] ?? '');
 
   if (!$create) {
+    if (!$idp_login->verifyLink('ldap', $user, $external_id)) {
+      $_SESSION['return'][] =  array(
+        'type' => 'danger',
+        'log' => array(__FUNCTION__, $user, '*', 'The mailbox is linked to another IdP user'),
+        'msg' => 'generic_server_error'
+      );
+      return false;
+    }
     // login success
-    if ($mapper_key !== false) {
+    $update_template = $idp_login->templateFor($user, $mapped_template);
+    if ($update_template !== null) {
       // update user
       $_SESSION['access_all_exception'] = '1';
       mailbox('edit', 'mailbox_from_template', array(
         'username' => $user,
         'name' => $user_res['displayname'][0],
-        'template' => $iam_settings['templates'][$mapper_key]
+        'template' => $update_template
       ));
       $_SESSION['access_all_exception'] = '0';
     }
@@ -714,7 +761,7 @@ function ldap_mbox_login($user, $pass, $extra = null){
     return false;
   }
   // check if matching attribute exist
-  if (empty($iam_settings['mappers']) || !$user_template || $mapper_key === false) {
+  if ($mapped_template === null) {
     if (!empty($iam_settings['default_template'])) {
       $mbox_template = $iam_settings['default_template'];
     } else {
@@ -726,7 +773,16 @@ function ldap_mbox_login($user, $pass, $extra = null){
       return false;
     }
   } else {
-    $mbox_template = $iam_settings['templates'][$mapper_key];
+    $mbox_template = $mapped_template;
+  }
+
+  if (!$idp_login->mayCreate('ldap', $user, $external_id)) {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $user, '*', 'The IdP user is linked to another mailbox'),
+      'msg' => 'generic_server_error'
+    );
+    return false;
   }
 
   // create mailbox
@@ -748,6 +804,7 @@ function ldap_mbox_login($user, $pass, $extra = null){
     );
     return false;
   }
+  $idp_login->storeLink('ldap', $user, $external_id);
 
   return 'user';
 }

data/web/inc/functions.fail2ban.inc.php

@@ -60,13 +60,14 @@ function fail2ban($_action, $_data = null, $_extra = null) {
         $queue_unban = $redis->hGetAll('F2B_QUEUE_UNBAN');
         if (is_array($active_bans)) {
           foreach ($active_bans as $network => $banned_until) {
-            $queued_for_unban = (isset($queue_unban[$network]) && $queue_unban[$network] == 1) ? 1 : 0;
-            $difference = $banned_until - time();
+            // Expired bans are lifted by netfilter within seconds
+            $difference = max(0, intval($banned_until) - time());
+            $queued_for_unban = ($difference == 0 || (isset($queue_unban[$network]) && $queue_unban[$network] == 1)) ? 1 : 0;
             $f2b_options['active_bans'][] = array(
               'queued_for_unban' => $queued_for_unban,
               'network' => $network,
               'ip' => strtok($network,'/'),
-              'banned_until' => sprintf('%02dh %02dm %02ds', ($difference/3600), ($difference/60%60), $difference%60)
+              'banned_until' => sprintf('%02dh %02dm %02ds', intdiv($difference, 3600), intdiv($difference, 60) % 60, $difference % 60)
             );
           }
         }
@@ -108,16 +109,15 @@ function fail2ban($_action, $_data = null, $_extra = null) {
             );
             return false;
           }
-          // Rules will also be recreated on log events, but rules may seem empty for a second in the UI
-          docker('post', 'netfilter-mailcow', 'restart');
+          // netfilter recreates the default rules within 10 seconds
           $fail_count = 0;
           $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
-          while (empty($regex_result) && $fail_count < 10) {
+          while (empty($regex_result) && $fail_count < 15) {
             $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
             $fail_count++;
             sleep(1);
           }
-          if ($fail_count >= 10) {
+          if (empty($regex_result)) {
             $_SESSION['return'][] = array(
               'type' => 'danger',
               'log' => array(__FUNCTION__, $_action, $_data_log),
@@ -141,7 +141,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
           $_SESSION['return'][] = array(
             'type' => 'success',
             'log' => array(__FUNCTION__, $_action, $_data_log),
-            'msg' => array('object_modified', htmlspecialchars($network))
+            'msg' => 'f2b_modified'
           );
           return true;
         }
@@ -172,7 +172,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
               if (valid_network($network)) {
                 try {
                   $redis->hSet('F2B_WHITELIST', $network, 1);
-                  $redis->hDel('F2B_BLACKLIST', $network, 1);
+                  $redis->hDel('F2B_BLACKLIST', $network);
                   $redis->hSet('F2B_QUEUE_UNBAN', $network, 1);
                 }
                 catch (RedisException $e) {
@@ -199,14 +199,13 @@ function fail2ban($_action, $_data = null, $_extra = null) {
               if (valid_network($network) && !in_array($network, array(
                 '0.0.0.0',
                 '0.0.0.0/0',
-                getenv('IPV4_NETWORK') . '0/24',
-                getenv('IPV4_NETWORK') . '0',
+                getenv('IPV4_NETWORK') . '.0/24',
+                getenv('IPV4_NETWORK') . '.0',
                 getenv('IPV6_NETWORK')
               ))) {
                 try {
                   $redis->hSet('F2B_BLACKLIST', $network, 1);
-                  $redis->hDel('F2B_WHITELIST', $network, 1);
-                  //$response = docker('post', 'netfilter-mailcow', 'restart');
+                  $redis->hDel('F2B_WHITELIST', $network);
                 }
                 catch (RedisException $e) {
                   $_SESSION['return'][] = array(
@@ -261,62 +260,60 @@ function fail2ban($_action, $_data = null, $_extra = null) {
       $f2b_options['ban_time'] = ($ban_time < 60) ? 60 : $ban_time;
       $f2b_options['ban_time_increment'] = ($ban_time_increment == 1) ? true : false;
       $f2b_options['max_ban_time'] = ($max_ban_time < 60) ? 60 : $max_ban_time;
-      $f2b_options['netban_ipv4'] = ($netban_ipv4 < 8) ? 8 : $netban_ipv4;
-      $f2b_options['netban_ipv6'] = ($netban_ipv6 < 8) ? 8 : $netban_ipv6;
-      $f2b_options['netban_ipv4'] = ($netban_ipv4 > 32) ? 32 : $netban_ipv4;
-      $f2b_options['netban_ipv6'] = ($netban_ipv6 > 128) ? 128 : $netban_ipv6;
+      $f2b_options['netban_ipv4'] = min(32, max(8, $netban_ipv4));
+      $f2b_options['netban_ipv6'] = min(128, max(8, $netban_ipv6));
       $f2b_options['max_attempts'] = ($max_attempts < 1) ? 1 : $max_attempts;
       $f2b_options['retry_window'] = ($retry_window < 1) ? 1 : $retry_window;
-      $f2b_options['banlist_id'] = $is_now['banlist_id'];
+      $f2b_options['banlist_id'] = (!empty($is_now['banlist_id'])) ? $is_now['banlist_id'] : uuid4();
       $f2b_options['manage_external'] = ($manage_external > 0) ? 1 : 0;
+      $wl_items = array();
+      if (!empty($wl)) {
+        foreach (array_filter(array_map('trim', preg_split("/( |,|;|\n)/", $wl))) as $wl_item) {
+          if (valid_network($wl_item) || valid_hostname($wl_item)) {
+            $wl_items[$wl_item] = 1;
+          }
+          else {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log),
+              'msg' => array('network_host_invalid', $wl_item)
+            );
+          }
+        }
+      }
+      $bl_items = array();
+      if (!empty($bl)) {
+        foreach (array_filter(array_map('trim', preg_split("/( |,|;|\n)/", $bl))) as $bl_item) {
+          if (valid_network($bl_item) && !in_array($bl_item, array(
+            '0.0.0.0',
+            '0.0.0.0/0',
+            getenv('IPV4_NETWORK') . '.0/24',
+            getenv('IPV4_NETWORK') . '.0',
+            getenv('IPV6_NETWORK')
+          ))) {
+            $bl_items[$bl_item] = 1;
+          }
+          else {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log),
+              'msg' => array('network_host_invalid', $bl_item)
+            );
+          }
+        }
+      }
       try {
+        // One transaction, netfilter must never see empty lists in between
+        $redis->multi();
         $redis->Set('F2B_OPTIONS', json_encode($f2b_options));
-        $redis->Del('F2B_WHITELIST');
-        $redis->Del('F2B_BLACKLIST');
-        if(!empty($wl)) {
-          $wl_array = array_map('trim', preg_split( "/( |,|;|\n)/", $wl));
-          $wl_array = array_filter($wl_array);
-          if (is_array($wl_array)) {
-            foreach ($wl_array as $wl_item) {
-              if (valid_network($wl_item) || valid_hostname($wl_item)) {
-                $redis->hSet('F2B_WHITELIST', $wl_item, 1);
-              }
-              else {
-                $_SESSION['return'][] = array(
-                  'type' => 'danger',
-                  'log' => array(__FUNCTION__, $_action, $_data_log),
-                  'msg' => array('network_host_invalid', $wl_item)
-                );
-                continue;
-              }
-            }
-          }
+        $redis->Del('F2B_WHITELIST', 'F2B_BLACKLIST');
+        if (!empty($wl_items)) {
+          $redis->hMSet('F2B_WHITELIST', $wl_items);
         }
-        if(!empty($bl)) {
-          $bl_array = array_map('trim', preg_split( "/( |,|;|\n)/", $bl));
-          $bl_array = array_filter($bl_array);
-          if (is_array($bl_array)) {
-            foreach ($bl_array as $bl_item) {
-              if (valid_network($bl_item) && !in_array($bl_item, array(
-                '0.0.0.0',
-                '0.0.0.0/0',
-                getenv('IPV4_NETWORK') . '0/24',
-                getenv('IPV4_NETWORK') . '0',
-                getenv('IPV6_NETWORK')
-              ))) {
-                $redis->hSet('F2B_BLACKLIST', $bl_item, 1);
-              }
-              else {
-                $_SESSION['return'][] = array(
-                  'type' => 'danger',
-                  'log' => array(__FUNCTION__, $_action, $_data_log),
-                  'msg' => array('network_host_invalid', $bl_item)
-                );
-                continue;
-              }
-            }
-          }
+        if (!empty($bl_items)) {
+          $redis->hMSet('F2B_BLACKLIST', $bl_items);
         }
+        $redis->exec();
       }
       catch (RedisException $e) {
         $_SESSION['return'][] = array(
@@ -348,7 +345,9 @@ function fail2ban($_action, $_data = null, $_extra = null) {
       if (is_array($_extra)) {
         $_extra = $_extra[0];
       }
-      if ($_extra != $f2b_options['banlist_id']){
+      // Timing safe, a missing or empty id never matches
+      $banlist_id = (string)($f2b_options['banlist_id'] ?? '');
+      if ($banlist_id === '' || !is_string($_extra) || !hash_equals($banlist_id, $_extra)) {
         http_response_code(404);
         return false;
       }

data/web/inc/functions.inc.php

@@ -1292,8 +1292,13 @@ function set_tfa($_data) {
       $row = $stmt->fetch(PDO::FETCH_ASSOC);
       if ($row) {
         if ($row['authsource'] == 'ldap'){
-          if (!ldap_mbox_login($username, $_data["confirm_password"], $iam_settings)) $access_denied = true;
-          else $access_denied = false;
+          try {
+            if (!ldap_mbox_login($username, $_data["confirm_password"], $iam_settings)) $access_denied = true;
+            else $access_denied = false;
+          } catch (\Mailcow\Auth\IdpUnavailable $e) {
+            $_SESSION['return'][] = array('type' => 'danger', 'log' => array(__FUNCTION__, $_data_log, $e->getMessage()), 'msg' => 'iam_unavailable');
+            return false;
+          }
         } else {
           if (!verify_hash($row['password'], $_data["confirm_password"])) $access_denied = true;
           else $access_denied = false;
@@ -2360,6 +2365,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
   $data_log = $_data;
   if (isset($data_log['client_secret'])) $data_log['client_secret'] = '*';
   if (isset($data_log['access_token'])) $data_log['access_token'] = '*';
+  if (isset($data_log['bindpass'])) $data_log['bindpass'] = '*';
 
   switch ($_action) {
     case 'get':
@@ -2370,9 +2376,12 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       foreach($rows as $row){
         switch ($row["key"]) {
           case "redirect_url_extra":
+            $settings[$row["key"]] = json_decode($row["value"]);
+          break;
           case "mappers":
           case "templates":
-            $settings[$row["key"]] = json_decode($row["value"]);
+            // Parallel lists; older rows can be JSON objects, decode as arrays to keep the keys aligned
+            $settings[$row["key"]] = (array)json_decode($row["value"], true);
           break;
           case "use_ssl":
           case "use_tls":
@@ -2455,6 +2464,13 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
 
       $_data['ignore_ssl_error']    = isset($_data['ignore_ssl_error']) ? boolval($_data['ignore_ssl_error']) : false;
       $_data['login_provisioning']  = isset($_data['login_provisioning']) ? boolval($_data['login_provisioning']) : false;
+      // Identity sync: offboarding and groups (Keycloak and LDAP)
+      $sync_settings = array('offboarding', 'offboarding_limit', 'group_sync', 'group_autocreate', 'group_default_domain');
+      $_data['offboarding']          = isset($_data['offboarding']) ? intval($_data['offboarding']) : 0;
+      $_data['offboarding_limit']    = isset($_data['offboarding_limit']) && is_numeric($_data['offboarding_limit']) ? min(100, max(0, intval($_data['offboarding_limit']))) : 10;
+      $_data['group_sync']           = isset($_data['group_sync']) ? intval($_data['group_sync']) : 0;
+      $_data['group_autocreate']     = isset($_data['group_autocreate']) ? intval($_data['group_autocreate']) : 0;
+      $_data['group_default_domain'] = isset($_data['group_default_domain']) ? strtolower(trim($_data['group_default_domain'])) : "";
       switch ($_data['authsource']) {
         case "keycloak":
           $_data['server_url']        = (!empty($_data['server_url'])) ? rtrim($_data['server_url'], '/') : null;
@@ -2463,7 +2479,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $_data['import_users']      = isset($_data['import_users']) ? intval($_data['import_users']) : 0;
           $_data['sync_interval']     = (!empty($_data['sync_interval'])) ? intval($_data['sync_interval']) : 15;
           $_data['sync_interval']     = $_data['sync_interval'] < 1 ? 1 : $_data['sync_interval'];
-          $required_settings          = array('authsource', 'server_url', 'realm', 'client_id', 'client_secret', 'redirect_url', 'version', 'mailpassword_flow', 'periodic_sync', 'import_users', 'sync_interval', 'ignore_ssl_error', 'login_provisioning');
+          $_data['group_paths']       = isset($_data['group_paths']) ? trim($_data['group_paths']) : "";
+          $required_settings          = array_merge(array('authsource', 'server_url', 'realm', 'client_id', 'client_secret', 'redirect_url', 'version', 'mailpassword_flow', 'periodic_sync', 'import_users', 'sync_interval', 'ignore_ssl_error', 'login_provisioning', 'group_paths'), $sync_settings);
         break;
         case "generic-oidc":
           $_data['authorize_url']     = (!empty($_data['authorize_url'])) ? $_data['authorize_url'] : null;
@@ -2484,7 +2501,14 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $_data['use_tls']           = isset($_data['use_tls']) && !$_data['use_ssl'] ? boolval($_data['use_tls']) : false;
           $_data['sync_interval']     = (!empty($_data['sync_interval'])) ? intval($_data['sync_interval']) : 15;
           $_data['sync_interval']     = $_data['sync_interval'] < 1 ? 1 : $_data['sync_interval'];
-          $required_settings          = array('authsource', 'host', 'port', 'basedn', 'username_field', 'filter', 'attribute_field', 'binddn', 'bindpass', 'periodic_sync', 'import_users', 'sync_interval', 'use_ssl', 'use_tls', 'ignore_ssl_error', 'login_provisioning');
+          $_data['id_attribute']      = isset($_data['id_attribute']) ? strtolower(trim($_data['id_attribute'])) : "";
+          $_data['disabled_filter']   = isset($_data['disabled_filter']) ? trim($_data['disabled_filter']) : "";
+          $_data['group_basedn']      = isset($_data['group_basedn']) ? trim($_data['group_basedn']) : "";
+          $_data['group_filter']      = isset($_data['group_filter']) ? trim($_data['group_filter']) : "";
+          $_data['group_name_attribute'] = (!empty($_data['group_name_attribute'])) ? strtolower(trim($_data['group_name_attribute'])) : "cn";
+          $_data['group_member_mode'] = isset($_data['group_member_mode']) && in_array($_data['group_member_mode'], array('memberof', 'member', 'ad_chain')) ? $_data['group_member_mode'] : "memberof";
+          $_data['group_member_attribute'] = (!empty($_data['group_member_attribute'])) ? strtolower(trim($_data['group_member_attribute'])) : "member";
+          $required_settings          = array_merge(array('authsource', 'host', 'port', 'basedn', 'username_field', 'filter', 'attribute_field', 'binddn', 'bindpass', 'periodic_sync', 'import_users', 'sync_interval', 'use_ssl', 'use_tls', 'ignore_ssl_error', 'login_provisioning', 'id_attribute', 'disabled_filter', 'group_basedn', 'group_filter', 'group_name_attribute', 'group_member_mode', 'group_member_attribute'), $sync_settings);
         break;
       }
 
@@ -2506,6 +2530,10 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         $stmt->bindParam(':value', $_data[$setting]);
         $stmt->execute();
       }
+      // Stored LDAP ids of another id attribute never match again, the mailboxes are linked by email again
+      if ($_data['authsource'] == 'ldap' && ($iam_settings['id_attribute'] ?? '') != $_data['id_attribute']) {
+        $pdo->query("UPDATE `identity_link` SET `external_id` = NULL WHERE `provider` = 'ldap'");
+      }
       $pdo->commit();
 
       // add redirect_url_extra
@@ -2533,19 +2561,26 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         $_data['mappers'] = (!is_array($_data['mappers'])) ? array($_data['mappers']) : $_data['mappers'];
         $_data['templates'] = (!is_array($_data['templates'])) ? array($_data['templates']) : $_data['templates'];
 
-        $mappers = array_filter($_data['mappers']);
-        $templates = array_filter($_data['templates']);
-        if (count($mappers) == count($templates)){
-          $mappers = json_encode($mappers);
-          $templates = json_encode($templates);
-
-          $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('mappers', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
-          $stmt->bindParam(':value', $mappers);
-          $stmt->execute();
-          $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('templates', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
-          $stmt->bindParam(':value', $templates);
-          $stmt->execute();
+        // Keep mapper/template pairs together, incomplete rows are dropped
+        $mappers = array();
+        $templates = array();
+        $template_values = array_values($_data['templates']);
+        foreach (array_values($_data['mappers']) as $i => $mapper) {
+          if ((string)$mapper === '' || (string)($template_values[$i] ?? '') === '') {
+            continue;
+          }
+          $mappers[] = $mapper;
+          $templates[] = $template_values[$i];
         }
+        $mappers = json_encode($mappers);
+        $templates = json_encode($templates);
+
+        $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('mappers', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
+        $stmt->bindParam(':value', $mappers);
+        $stmt->execute();
+        $stmt = $pdo->prepare("INSERT INTO identity_provider (`key`, `value`) VALUES ('templates', :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`);");
+        $stmt->bindParam(':value', $templates);
+        $stmt->execute();
       }
 
       // delete old access_token
@@ -2670,6 +2705,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       }
 
       $stmt = $pdo->query("DELETE FROM identity_provider;");
+      $stmt = $pdo->query("DELETE FROM identity_link;");
+      $stmt = $pdo->query("DELETE FROM identity_sync_run;");
 
       $_SESSION['return'][] =  array(
         'type' => 'success',
@@ -2790,8 +2827,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       }
 
       // get mapped template
-      $user_template = $info['mailcow_template'];
-      $mapper_key = array_search($user_template, $iam_settings['mappers']);
+      $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
+      $mapped_template = $idp_login->mappedTemplate($info['mailcow_template'] ?? array());
 
       // token valid, get mailbox
       $stmt = $pdo->prepare("SELECT
@@ -2813,13 +2850,23 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           );
           return false;
         }
-        if ($mapper_key !== false) {
+        if (!$idp_login->verifyLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
+          clear_session();
+          $_SESSION['return'][] =  array(
+            'type' => 'danger',
+            'log' => array(__FUNCTION__, $info['email'], 'The mailbox is linked to another IdP user'),
+            'msg' => 'login_failed'
+          );
+          return false;
+        }
+        $update_template = $idp_login->templateFor($info['email'], $mapped_template);
+        if ($update_template !== null) {
           // update user
           $_SESSION['access_all_exception'] = '1';
           mailbox('edit', 'mailbox_from_template', array(
             'username' => $info['email'],
             'name' => $info['name'],
-            'template' => $iam_settings['templates'][$mapper_key]
+            'template' => $update_template
           ));
           $_SESSION['access_all_exception'] = '0';
 
@@ -2834,11 +2881,11 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           $stmt->execute(array(':user' => $info['email']));
           $row = $stmt->fetch(PDO::FETCH_ASSOC);
         }
-        if ($row['active'] != 1 || $row['d_active'] != 1) {
+        if ($row['active'] != 1 || $row['d_active'] != 1 || (json_decode($row['attributes'], true)['shared'] ?? '0') == '1') {
           clear_session();
           $_SESSION['return'][] =  array(
             'type' => 'danger',
-            'log' => array(__FUNCTION__, $info['email'], 'Domain or mailbox is inactive'),
+            'log' => array(__FUNCTION__, $info['email'], 'Domain or mailbox is inactive or shared'),
             'msg' => 'login_failed'
           );
           return false;
@@ -2864,7 +2911,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
-      if (empty($iam_settings['mappers']) || empty($user_template) || $mapper_key === false){
+      if ($mapped_template === null){
         if (!empty($iam_settings['default_template'])) {
           $mbox_template = $iam_settings['default_template'];
         } else {
@@ -2877,7 +2924,17 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           return false;
         }
       } else {
-        $mbox_template = $iam_settings['templates'][$mapper_key];
+        $mbox_template = $mapped_template;
+      }
+
+      if (!$idp_login->mayCreate($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
+        clear_session();
+        $_SESSION['return'][] =  array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $info['email'], 'The IdP user is linked to another mailbox'),
+          'msg' => 'login_failed'
+        );
+        return false;
       }
 
       // create mailbox
@@ -2899,6 +2956,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         );
         return false;
       }
+      $idp_login->storeLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''));
 
       // double check if mailbox and domain is active
       $stmt = $pdo->prepare("SELECT * FROM `mailbox`
@@ -2955,6 +3013,26 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
+      // The mailbox may have been deleted or deactivated since the login
+      $stmt = $pdo->prepare("SELECT `mailbox`.`authsource` FROM `mailbox`
+        INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
+        WHERE `kind` NOT REGEXP 'location|thing|group'
+          AND `mailbox`.`active` = '1'
+          AND `domain`.`active` = '1'
+          AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
+          AND `username` = :user");
+      $stmt->execute(array(':user' => $info['email']));
+      $row = $stmt->fetch(PDO::FETCH_ASSOC);
+      if (!$row || !in_array($row['authsource'], array('keycloak', 'generic-oidc'))) {
+        clear_session();
+        $_SESSION['return'][] =  array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $info['email'], 'Mailbox or domain is inactive or not of type OIDC'),
+          'msg' => 'refresh_login_failed'
+        );
+        return false;
+      }
+
       set_user_loggedin_session($info['email']);
       $_SESSION['iam_token'] = $plain_token;
       $_SESSION['iam_refresh_token'] = $plain_refreshtoken;

data/web/inc/functions.mailbox.inc.php

@@ -938,6 +938,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $eas_access = (isset($_data['eas_access'])) ? intval($_data['eas_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['eas_access']);
           $dav_access = (isset($_data['dav_access'])) ? intval($_data['dav_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['dav_access']);
           $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : 0;
+          // Shared mailbox: no login of its own, members open it through group shares
+          $shared = (isset($_data['shared'])) ? intval($_data['shared']) : 0;
           $quarantine_notification = (isset($_data['quarantine_notification'])) ? strval($_data['quarantine_notification']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_notification']);
           $quarantine_category = (isset($_data['quarantine_category'])) ? strval($_data['quarantine_category']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_category']);
           // Validate quarantine_category
@@ -971,6 +973,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'eas_access' => strval($eas_access),
               'dav_access' => strval($dav_access),
               'relayhost' => strval($relayhost),
+              'shared' => strval($shared),
               'passwd_update' => time(),
               'mailbox_format' => strval($MAILBOX_DEFAULT_ATTRIBUTES['mailbox_format']),
               'quarantine_notification' => strval($quarantine_notification),
@@ -1050,7 +1053,11 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             );
             return false;
           }
-          if ($authsource == 'mailcow'){
+          if ($authsource == 'mailcow' && $shared == 1 && empty($password) && empty($password2)) {
+            // Nobody logs in to a shared mailbox, a random password keeps it unusable
+            $password_hashed = hash_password(bin2hex(random_bytes(32)));
+          }
+          elseif ($authsource == 'mailcow'){
             if (password_check($password, $password2) !== true) {
               return false;
             }
@@ -1268,7 +1275,16 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             }
           }
 
-          return mailbox('add', 'mailbox', $mailbox_attributes);
+          $result = mailbox('add', 'mailbox', $mailbox_attributes);
+          if ($result !== false) {
+            // Remember the applied template, see edit mailbox_from_template
+            $stmt = $pdo->prepare("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.template', :template) WHERE `username` = :username");
+            $stmt->execute(array(
+              ':template' => $_data['template'],
+              ':username' => strtolower(trim($_data['local_part'])) . '@' . idn_to_ascii(strtolower(trim($_data['domain'])), 0, INTL_IDNA_VARIANT_UTS46)
+            ));
+          }
+          return $result;
         break;
         case 'mta_sts':
           $domain       = idn_to_ascii(strtolower(trim($_data['domain'])), 0, INTL_IDNA_VARIANT_UTS46);
@@ -2208,6 +2224,14 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
+            if (!empty($is_now['group_id'])) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => array('alias_group_managed', htmlspecialchars($is_now['address']))
+              );
+              continue;
+            }
             if ($_data['expand_alias'] === true || $_data['expand_alias'] == 1) {
               $stmt = $pdo->prepare("SELECT `address` FROM `alias`
                 WHERE `address` = :address
@@ -2370,7 +2394,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 // Delete from sender_acl to prevent duplicates
                 $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE
                   `logged_in_as` = :goto AND
-                  `send_as` = :address");
+                  `send_as` = :address AND
+                  `group_id` IS NULL");
                 $stmt->execute(array(
                   ':goto' => $goto,
                   ':address' => $address
@@ -2404,6 +2429,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':active' => $active,
                 ':id' => $is_now['id']
               ));
+              if ($address != $is_now['address']) {
+                // Send-as grants of groups follow the address
+                \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
+              }
             }
             $_SESSION['return'][] = array(
               'type' => 'success',
@@ -2785,6 +2814,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               (int)$eas_access     = (isset($_data['eas_access']) && hasACLAccess("protocol_access")) ? intval($_data['eas_access']) : intval($is_now['attributes']['eas_access']);
               (int)$dav_access    = (isset($_data['dav_access']) && hasACLAccess("protocol_access")) ? intval($_data['dav_access']) : intval($is_now['attributes']['dav_access']);
               (int)$relayhost       = (isset($_data['relayhost']) && hasACLAccess("mailbox_relayhost")) ? intval($_data['relayhost']) : intval($is_now['attributes']['relayhost']);
+              (int)$shared          = (isset($_data['shared'])) ? intval($_data['shared']) : intval($is_now['attributes']['shared'] ?? 0);
               (int)$quota_m         = (isset_has_content($_data['quota'])) ? intval($_data['quota']) : ($is_now['quota'] / 1048576);
               $name                 = (!empty($_data['name'])) ? ltrim(rtrim($_data['name'], '>'), '<') : $is_now['name'];
               $domain               = $is_now['domain'];
@@ -3007,7 +3037,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $sender_acl_merged = array_merge($sender_acl_domain_admin, $sender_acl_admin);
                 // If merged array still contains "*", set it as only value
                 !in_array('*', $sender_acl_merged) ?: $sender_acl_merged = array('*');
-                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `logged_in_as` = :username");
+                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `group_id` IS NULL AND `logged_in_as` = :username");
                 $stmt->execute(array(
                   ':username' => $username
                 ));
@@ -3039,7 +3069,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 }
               }
               else {
-                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `logged_in_as` = :username");
+                $stmt = $pdo->prepare("DELETE FROM `sender_acl` WHERE `external` = 0 AND `group_id` IS NULL AND `logged_in_as` = :username");
                 $stmt->execute(array(
                   ':username' => $username
                 ));
@@ -3080,6 +3110,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   `attributes` = JSON_SET(`attributes`, '$.sieve_access', :sieve_access),
                   `attributes` = JSON_SET(`attributes`, '$.pop3_access', :pop3_access),
                   `attributes` = JSON_SET(`attributes`, '$.relayhost', :relayhost),
+                  `attributes` = JSON_SET(`attributes`, '$.shared', :shared),
                   `attributes` = JSON_SET(`attributes`, '$.smtp_access', :smtp_access),
                   `attributes` = JSON_SET(`attributes`, '$.eas_access', :eas_access),
                   `attributes` = JSON_SET(`attributes`, '$.dav_access', :dav_access),
@@ -3102,6 +3133,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':dav_access' => $dav_access,
                 ':recovery_email' => $pw_recovery_email,
                 ':relayhost' => $relayhost,
+                ':shared' => strval($shared),
                 ':username' => $username,
                 ':authsource' => $authsource
               ));
@@ -3258,7 +3290,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               "imapsync" => ['user2'],
               'bcc_maps' => ['local_dest', 'bcc_dest'],
               'recipient_maps' => ['old_dest', 'new_dest'],
-              'sender_acl' => ['logged_in_as', 'send_as']
+              'sender_acl' => ['logged_in_as', 'send_as'],
+              'identity_link' => ['username'],
+              'mailbox_group_imap_grant' => ['owner'],
+              'mailbox_group_sogo_grant' => ['owner'],
+              'mailbox_group_sogo_acl' => ['c_uid'],
+              'mailbox_group_subscription' => ['username', 'owner']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3272,6 +3309,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 
             // Sync sources created by / targeting the mailbox and its OAuth tokens (owner); a PDOException rolls back below
             \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
+            // Group memberships and send-as grants follow, group aliases are rebuilt
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
 
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
@@ -3370,10 +3409,27 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attribute_hash = sha1(json_encode($mbox_template_data["attributes"]));
           $is_now = mailbox('get', 'mailbox_details', $_data['username']);
           $name = ltrim(rtrim($_data['name'], '>'), '<');
-          if ($is_now['attributes']['attribute_hash'] == $attribute_hash && $is_now['name'] == $name)
+          // Last applied template, null for mailboxes created before it was recorded
+          $applied_template = $is_now['attributes']['template'] ?? null;
+          $stmt_template = $pdo->prepare("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.template', :template) WHERE `username` = :username");
+          if ($is_now['attributes']['attribute_hash'] == $attribute_hash && $is_now['name'] == $name) {
+            if ($applied_template === null) {
+              $stmt_template->execute(array(':template' => $_data['template'], ':username' => $_data['username']));
+            }
             return true;
+          }
 
           $mbox_template_data = json_decode($mbox_template_data["attributes"], true);
+          // Any manual edit clears attribute_hash: re-applying the same template must not undo a manual (de)activation
+          if ($applied_template === null || $applied_template === $_data['template']) {
+            unset($mbox_template_data['active']);
+          }
+          // Login disabled by the IdP sync is only lifted by the sync
+          $stmt = $pdo->prepare("SELECT `idp_disabled` FROM `identity_link` WHERE `username` = :username");
+          $stmt->execute(array(':username' => $_data['username']));
+          if (intval($stmt->fetchColumn()) == 1) {
+            unset($mbox_template_data['active']);
+          }
           $mbox_template_data['attribute_hash'] = $attribute_hash;
           $mbox_template_data['name'] = $name;
           $quarantine_attributes = array('username' => $_data['username']);
@@ -3416,6 +3472,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           if ($result === false) return $result;
           $result = acl('edit', 'user', $acl_attributes);
           if ($result === false) return $result;
+          $stmt_template->execute(array(':template' => $_data['template'], ':username' => $_data['username']));
 
           $_SESSION['return'] = array();
           return true;
@@ -3528,6 +3585,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ":template" => $_data["template"] ,
               ":attributes" => json_encode($attr)
             ));
+            // Mailboxes with this effective group template get the new settings
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->templateChanged((int)$id);
           }
 
 
@@ -3959,7 +4018,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
           // Return array $data['sender_acl_domains/addresses']['ro'] with read-only objects
           // Return array $data['sender_acl_domains/addresses']['rw'] with read-write objects (can be deleted)
-          $stmt = $pdo->prepare("SELECT REPLACE(`send_as`, '@', '') AS `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND (`send_as` LIKE '@%' OR `send_as` = '*')");
+          $stmt = $pdo->prepare("SELECT REPLACE(`send_as`, '@', '') AS `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND `group_id` IS NULL AND (`send_as` LIKE '@%' OR `send_as` = '*')");
           $stmt->execute(array(':logged_in_as' => $_data));
           $domain_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($domain_row = array_shift($domain_rows)) {
@@ -3978,7 +4037,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               $data['sender_acl_domains']['rw'][] = $domain_row['send_as'];
             }
           }
-          $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND (`send_as` NOT LIKE '@%' AND `send_as` != '*')");
+          $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :logged_in_as AND `external` = '0' AND `group_id` IS NULL AND (`send_as` NOT LIKE '@%' AND `send_as` != '*')");
           $stmt->execute(array(':logged_in_as' => $_data));
           $address_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($address_row = array_shift($address_rows)) {
@@ -4436,7 +4495,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             `sogo_visible`,
             `sender_allowed`,
             `created`,
-            `modified`
+            `modified`,
+            `group_id`,
+            (SELECT `name` FROM `mailbox_group` WHERE `mailbox_group`.`id` = `alias`.`group_id`) AS `group_name`
               FROM `alias`
                   WHERE (`id` = :id OR `address` = :address) AND `address` != `goto`");
           $stmt->execute(array(
@@ -4469,6 +4530,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $aliasdata['sogo_visible'] = $row['sogo_visible'];
           $aliasdata['sogo_visible_int'] = $row['sogo_visible'];
           $aliasdata['sender_allowed'] = $row['sender_allowed'];
+          // Distribution address of a mailbox group, read-only here
+          $aliasdata['group_id'] = $row['group_id'];
+          $aliasdata['group_name'] = $row['group_name'];
           $aliasdata['created'] = $row['created'];
           $aliasdata['modified'] = $row['modified'];
           if (!hasDomainAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $aliasdata['domain'])) {
@@ -4909,7 +4973,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $mailboxdata['is_relayed'] = $row['backupmx'];
             // Internal send-as ACL for this mailbox, mirrors the sender_acl field accepted by edit/mailbox
             $mailboxdata['sender_acl'] = array();
-            $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :username AND `external` = '0'");
+            $stmt = $pdo->prepare("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :username AND `external` = '0' AND `group_id` IS NULL");
             $stmt->execute(array(':username' => $_data));
             $sender_acl_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
             foreach ($sender_acl_rows as $sender_acl_row) {
@@ -5283,6 +5347,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':domain' => $domain,
             ));
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
             $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,
@@ -5410,6 +5475,14 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
+            if (!empty($alias_data['group_id'])) {
+              $_SESSION['return'][] = array(
+                'type' => 'danger',
+                'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                'msg' => array('alias_group_managed', htmlspecialchars($alias_data['address']))
+              );
+              continue;
+            }
 
             // Track affected mailboxes for SOGo update (capture before deletion)
             if (!empty($alias_data['goto'])) {
@@ -5430,6 +5503,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':alias_address' => $alias_data['address']
             ));
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressDeleted($alias_data['address']);
             $_SESSION['return'][] = array(
               'type' => 'success',
               'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -5470,6 +5544,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(
               ':alias_domain' => $alias_domain,
             ));
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($alias_domain);
             $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `domain` = :alias_domain");
             $stmt->execute(array(
               ':alias_domain' => $alias_domain,
@@ -5552,7 +5627,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 'msg' => 'Could not move maildir to garbage collector: variables local_part and/or domain empty'
               );
             }
-            $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `goto` = :username");
+            \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->mailboxDeleted($username);
+            $stmt = $pdo->prepare("DELETE FROM `alias` WHERE `goto` = :username AND `group_id` IS NULL");
             $stmt->execute(array(
               ':username' => $username
             ));

data/web/inc/init_db.inc.php

@@ -4,7 +4,20 @@ function init_db_schema()
   try {
     global $pdo;
 
-    $db_version = "22102026_1200";
+    // Defined once as DB_VERSION in docker-compose.yml
+    $db_version = (string)getenv('DB_VERSION');
+    if ($db_version === '') {
+      if (php_sapi_name() == "cli") {
+        echo "DB initialization failed: DB_VERSION is not set" . PHP_EOL;
+      } else {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__),
+          'msg' => 'Database not initialized: DB_VERSION is not set.'
+        );
+      }
+      return false;
+    }
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -174,34 +187,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "alias" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "address" => "VARCHAR(255) NOT NULL",
-          "goto" => "TEXT NOT NULL",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "private_comment" => "TEXT",
-          "public_comment" => "TEXT",
-          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "address" => array("address")
-          ),
-          "key" => array(
-            "domain" => array("domain")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "api" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -226,20 +211,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "sender_acl" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "logged_in_as" => "VARCHAR(255) NOT NULL",
-          "send_as" => "VARCHAR(255) NOT NULL",
-          "external" => "TINYINT(1) NOT NULL DEFAULT '0'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "templates" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -418,6 +389,360 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
+      "mailbox_group" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "name" => "VARCHAR(100) NOT NULL",
+          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          // distribution address, maintained as alias row with group_id
+          "address" => "VARCHAR(255) NULL DEFAULT NULL",
+          // members may send as the address
+          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "template_id" => "INT NULL DEFAULT NULL",
+          // highest priority wins when a mailbox is in several groups with a template
+          "priority" => "INT NOT NULL DEFAULT '0'",
+          // IdP group id: set = memberships are synced from the IdP
+          "idp_ref" => "VARCHAR(255) NULL DEFAULT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "domain_name" => array("domain", "name"),
+            "address" => array("address"),
+            "idp_ref" => array("idp_ref")
+          ),
+          "fkey" => array(
+            "fk_mailbox_group_domain" => array(
+              "col" => "domain",
+              "ref" => "domain.domain",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mailbox_group_template" => array(
+              "col" => "template_id",
+              "ref" => "templates.id",
+              "delete" => "SET NULL",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "mailbox_group_member" => array(
+        "cols" => array(
+          "group_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL",
+          "source" => "ENUM('manual', 'idp') NOT NULL DEFAULT 'manual'",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("group_id", "username")
+          ),
+          "key" => array(
+            "username" => array("username")
+          ),
+          "fkey" => array(
+            "fk_mailbox_group_member_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mailbox_group_member_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "mailbox_group_send_as" => array(
+        "cols" => array(
+          "group_id" => "INT NOT NULL",
+          // address or @domain the members may send as
+          "address" => "VARCHAR(255) NOT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("group_id", "address")
+          ),
+          "key" => array(
+            "address" => array("address")
+          ),
+          "fkey" => array(
+            "fk_mailbox_group_send_as_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "mailbox_group_imap_grant" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "group_id" => "INT NOT NULL",
+          // mailbox whose folders the members may access
+          "owner" => "VARCHAR(255) NOT NULL",
+          "rights" => "ENUM('read', 'write', 'full') NOT NULL DEFAULT 'read'",
+          // 1 = the group-grants cron subscribes the members to the folders (once per member)
+          "subscribe" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // removing = the ACL is deleted in Dovecot, then the row
+          "state" => "ENUM('pending', 'applied', 'failed', 'removing') NOT NULL DEFAULT 'pending'",
+          "last_error" => "TEXT",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "group_owner" => array("group_id", "owner")
+          ),
+          "key" => array(
+            "owner" => array("owner")
+          ),
+          "fkey" => array(
+            "fk_mailbox_group_imap_grant_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mailbox_group_imap_grant_owner" => array(
+              "col" => "owner",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "mailbox_group_sogo_grant" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "group_id" => "INT NOT NULL",
+          "owner" => "VARCHAR(255) NOT NULL",
+          "folder_type" => "ENUM('Calendar', 'Contacts') NOT NULL DEFAULT 'Calendar'",
+          "folder" => "VARCHAR(255) NOT NULL DEFAULT 'personal'",
+          "access" => "ENUM('read', 'write') NOT NULL DEFAULT 'read'",
+          // 1 = the group-grants cron subscribes the members to the folder (once per member)
+          "subscribe" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // pending = the owner has not opened SOGo yet, the folder does not exist
+          "state" => "ENUM('pending', 'applied', 'failed') NOT NULL DEFAULT 'pending'",
+          "last_error" => "TEXT",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "group_folder" => array("group_id", "owner", "folder_type", "folder")
+          ),
+          "key" => array(
+            "owner" => array("owner")
+          ),
+          "fkey" => array(
+            "fk_mailbox_group_sogo_grant_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mailbox_group_sogo_grant_owner" => array(
+              "col" => "owner",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // sogo_acl rows written for a grant, so only those are ever removed again. No FK: rows of grants
+      // removed by a cascade stay until their sogo_acl rows are deleted (GroupRepository::cleanupSogoAcl())
+      "mailbox_group_sogo_acl" => array(
+        "cols" => array(
+          "acl_id" => "INT NOT NULL",
+          "grant_id" => "INT NOT NULL",
+          "c_uid" => "VARCHAR(255) NOT NULL",
+          "c_role" => "VARCHAR(80) NOT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("acl_id")
+          ),
+          "key" => array(
+            "grant_id" => array("grant_id")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // IMAP and SOGo subscriptions made by the group-grants cron. No FK: rows of grants removed by a cascade stay
+      // until the members are unsubscribed again
+      "mailbox_group_subscription" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "kind" => "ENUM('imap', 'sogo') NOT NULL",
+          // mailbox_group_imap_grant.id or mailbox_group_sogo_grant.id
+          "grant_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL",
+          "owner" => "VARCHAR(255) NOT NULL",
+          // SOGo: Calendar/<id> or Contacts/<id>, IMAP: empty (all folders of the owner)
+          "folder" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "grant_username" => array("kind", "grant_id", "username")
+          ),
+          "key" => array(
+            "username" => array("username")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "identity_link" => array(
+        "cols" => array(
+          "username" => "VARCHAR(255) NOT NULL",
+          "provider" => "VARCHAR(20) NOT NULL",
+          // Keycloak user id (= OIDC sub), LDAP objectGUID/entryUUID/... or dn:<dn>, OIDC sub; NULL = not linked yet
+          "external_id" => "VARCHAR(255) NULL DEFAULT NULL",
+          // 1 = the sync set the mailbox to login disabled (active = 2)
+          "idp_disabled" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // first complete sync run that did not list the user
+          "missing_since" => "DATETIME NULL DEFAULT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("username")
+          ),
+          "unique" => array(
+            "provider_external_id" => array("provider", "external_id")
+          ),
+          "fkey" => array(
+            "fk_identity_link_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "identity_sync_run" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "provider" => "VARCHAR(20) NOT NULL DEFAULT ''",
+          // run = scheduled or requested, preview = dry run, force = without the deactivation limit
+          "mode" => "ENUM('run', 'preview', 'force') NOT NULL DEFAULT 'run'",
+          "status" => "ENUM('pending', 'running', 'ok', 'blocked', 'error') NOT NULL DEFAULT 'pending'",
+          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "started" => "DATETIME NULL DEFAULT NULL",
+          "finished" => "DATETIME NULL DEFAULT NULL",
+          "summary" => "MEDIUMTEXT"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "status" => array("status")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // alias and sender_acl come after mailbox_group (foreign keys)
+      "alias" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "address" => "VARCHAR(255) NOT NULL",
+          "goto" => "TEXT NOT NULL",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "private_comment" => "TEXT",
+          "public_comment" => "TEXT",
+          "sogo_visible" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "internal" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "sender_allowed" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          // set = distribution address of a mailbox group, read-only in the alias UI
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "address" => array("address"),
+            "group_id" => array("group_id")
+          ),
+          "key" => array(
+            "domain" => array("domain")
+          ),
+          "fkey" => array(
+            "fk_alias_mailbox_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "sender_acl" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "logged_in_as" => "VARCHAR(255) NOT NULL",
+          "send_as" => "VARCHAR(255) NOT NULL",
+          "external" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // set = granted through a mailbox group, rebuilt by the group module
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "group_id" => array("group_id")
+          ),
+          "fkey" => array(
+            "fk_sender_acl_mailbox_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
       "sieve_filters" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -727,7 +1052,8 @@ function init_db_schema()
           "mailbox_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'"
+          "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1278,7 +1604,9 @@ function init_db_schema()
           "redirect_uri" => "VARCHAR(2000)",
           "expires" => "TIMESTAMP NOT NULL",
           "scope" => "VARCHAR(4000)",
-          "id_token" => "VARCHAR(1000)"
+          "id_token" => "VARCHAR(1000)",
+          "code_challenge" => "VARCHAR(1000)",
+          "code_challenge_method" => "VARCHAR(20)"
         ),
         "keys" => array(
           "primary" => array(
@@ -1807,6 +2135,18 @@ function init_db_schema()
     // fill quarantine.qhash
     $pdo->query("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE ISNULL(`qhash`)");
 
+    // Group rows that lost their group_id (init_db of an older version ran) belong to their group again, so the
+    // group can update its alias and revokes send-as rights of former members. An active group without rows of its
+    // own for a grant only exists after such a downgrade; manual rows next to group rows stay manual.
+    $pdo->query("UPDATE `alias` a JOIN `mailbox_group` g ON g.`address` = a.`address` SET a.`group_id` = g.`id` WHERE a.`group_id` IS NULL");
+    $pdo->query("UPDATE `sender_acl` s
+      JOIN `mailbox_group_member` m ON m.`username` = s.`logged_in_as`
+      JOIN `mailbox_group` g ON g.`id` = m.`group_id` AND g.`active` = 1
+      JOIN `mailbox_group_send_as` ga ON ga.`group_id` = m.`group_id` AND ga.`address` = s.`send_as`
+      SET s.`group_id` = m.`group_id`
+      WHERE s.`group_id` IS NULL AND s.`external` = 0
+        AND NOT EXISTS (SELECT 1 FROM (SELECT `group_id` FROM `sender_acl` WHERE `group_id` IS NOT NULL) own WHERE own.`group_id` = m.`group_id`)");
+
   } catch (PDOException $e) {
     if (php_sapi_name() == "cli") {
       echo "DB initialization failed: " . print_r($e, true) . PHP_EOL;

data/web/inc/lib/composer.json

@@ -10,7 +10,7 @@
         "mustangostang/spyc": "^0.6.3",
         "directorytree/ldaprecord": "^3.3",
         "twig/twig": "^3.0",
-        "stevenmaguire/oauth2-keycloak": "^4.0",
+        "stevenmaguire/oauth2-keycloak": "^6.1",
         "league/oauth2-client": "^2.7",
         "bacon/bacon-qr-code": "^2.0"
     },

data/web/inc/lib/mailcow/ApiKey/Service/ApiKeyService.php

@@ -30,9 +30,7 @@ class ApiKeyService
     /** @throws AccessDenied unless an admin (login or read-write API key) */
     public function requireAdmin(): void
     {
-        if (!$this->context->isAdmin() || $this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
+        $this->context->requireAdmin();
     }
 
     /** @return ApiKey[] all keys, service keys first; [] for other callers */
@@ -146,7 +144,7 @@ class ApiKeyService
 
     private function canRead(): bool
     {
-        return $this->context->isAdmin() && !$this->context->isReadOnlyApi();
+        return $this->context->hasAdminRights() && !$this->context->isReadOnlyApi();
     }
 
     /** @throws AccessDenied when missing, UserError api_key_service_locked for service keys */

data/web/inc/lib/mailcow/Auth/AuthModule.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Auth;
+
+use Mailcow\Auth\Service\IdpLoginService;
+use Mailcow\Core\Context;
+use Mailcow\Identity\IdentityModule;
+use Mailcow\MailboxGroup\MailboxGroupModule;
+use PDO;
+
+/**
+ * Logins. For now the IdP login steps of the legacy flows (keycloak_mbox_login_rest(), ldap_mbox_login(),
+ * identity_provider('verify-sso')); the auth logic of the function files moves here step by step.
+ */
+class AuthModule
+{
+    public function __construct(public readonly IdpLoginService $idp)
+    {
+    }
+
+    /** @param array $iamSettings identity provider settings as identity_provider('get') returns them */
+    public static function create(PDO $pdo, Context $context, array $iamSettings): self
+    {
+        $identity = IdentityModule::create($pdo, $context, $iamSettings);
+        return new self(new IdpLoginService($identity->settings, $identity->links, MailboxGroupModule::create($pdo, $context)->templates));
+    }
+
+    /** Logins run before a session exists, so the default context is the system */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        $settings = $GLOBALS['iam_settings'] ?? null;
+        return self::create($GLOBALS['pdo'], $context ?? Context::system(), is_array($settings) ? $settings : identity_provider('get'));
+    }
+}

data/web/inc/lib/mailcow/Auth/IdpUnavailable.php

@@ -0,0 +1,15 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Auth;
+
+use RuntimeException;
+
+/**
+ * The identity provider could not answer a login (unreachable, timeout, server error). No failed login:
+ * callers report a temporary failure and must not count it as an attempt for netfilter.
+ */
+class IdpUnavailable extends RuntimeException
+{
+}

data/web/inc/lib/mailcow/Auth/Service/IdpLoginService.php

@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Auth\Service;
+
+use LdapRecord\Auth\BindException;
+use LdapRecord\Connection;
+use LdapRecord\LdapRecordException;
+use Mailcow\Auth\IdpUnavailable;
+use Mailcow\Identity\Model\Settings;
+use Mailcow\Identity\Service\LinkService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use Throwable;
+
+/**
+ * Steps of an IdP login (Keycloak REST, LDAP bind, OIDC) the legacy login flows call. The link checks fail
+ * closed: an error denies the login.
+ */
+class IdpLoginService
+{
+    public function __construct(private Settings $settings, private LinkService $links, private TemplateService $templates)
+    {
+    }
+
+    /**
+     * Binds as the user to check the password, then again as the configured user. False for wrong credentials,
+     * IdpUnavailable when the server cannot answer (client side codes are negative: server down, timeout, ...).
+     */
+    public function verifyLdapPassword(Connection $ldap, string $dn, string $password): bool
+    {
+        // an empty password would be an unauthenticated bind, which succeeds
+        if ($dn === '' || $password === '') {
+            return false;
+        }
+        $guard = $ldap->auth();
+        try {
+            $guard->bind($dn, $password);
+            $bound = true;
+        } catch (BindException $e) {
+            $code = $e->getDetailedError()?->getErrorCode();
+            if ($code !== null && ($code < 0 || in_array($code, [51, 52], true))) {
+                throw new IdpUnavailable('LDAP bind failed: ' . $e->getMessage(), 0, $e);
+            }
+            $bound = false;
+        } catch (LdapRecordException $e) {
+            throw new IdpUnavailable('LDAP bind failed: ' . $e->getMessage(), 0, $e);
+        }
+        try {
+            $guard->bindAsConfiguredUser();
+        } catch (LdapRecordException $e) {
+            // the login needs no further LDAP access
+        }
+        return $bound;
+    }
+
+    /** Template of the first attribute mapping matching one of the IdP attribute values, null if none */
+    public function mappedTemplate(mixed $values): ?string
+    {
+        return $this->settings->mappedTemplate((array)$values);
+    }
+
+    /** Template for the login of an existing mailbox: its effective group template, else the mapped one */
+    public function templateFor(string $username, ?string $mapped): ?string
+    {
+        try {
+            return $this->templates->templateForLogin($username, $mapped);
+        } catch (Throwable $e) {
+            error_log('mailcow: group template lookup failed for ' . $username . ': ' . $e->getMessage());
+            return $mapped;
+        }
+    }
+
+    /** The IdP user may open the mailbox, an unlinked mailbox gets linked */
+    public function verifyLink(string $provider, string $username, string $externalId): bool
+    {
+        try {
+            return $this->links->verify($provider, $username, $externalId);
+        } catch (Throwable $e) {
+            error_log('mailcow: IdP link check failed for ' . $username . ': ' . $e->getMessage());
+            return false;
+        }
+    }
+
+    /** Before a mailbox is created on login: the IdP user must not own another mailbox */
+    public function mayCreate(string $provider, string $username, string $externalId): bool
+    {
+        try {
+            return $this->links->mayCreate($provider, $username, $externalId);
+        } catch (Throwable $e) {
+            error_log('mailcow: IdP link check failed for ' . $username . ': ' . $e->getMessage());
+            return false;
+        }
+    }
+
+    /** Links a mailbox created on login */
+    public function storeLink(string $provider, string $username, string $externalId): void
+    {
+        try {
+            $this->links->link($provider, $username, $externalId);
+        } catch (Throwable $e) {
+            error_log('mailcow: IdP link could not be stored for ' . $username . ': ' . $e->getMessage());
+        }
+    }
+}

data/web/inc/lib/mailcow/Core/ApiResponse.php

@@ -38,6 +38,12 @@ class ApiResponse
         return $this->danger($error->msg(), $log);
     }
 
+    /** Something went partly wrong, but the call itself succeeded */
+    public function warning(string|array $msg, array $log): void
+    {
+        $this->add('warning', $msg, $log);
+    }
+
     /** Messages collected since the last flush() */
     public function messages(): array
     {

data/web/inc/lib/mailcow/Core/Context.php

@@ -70,6 +70,24 @@ class Context
         return $this->apiAccess === 'ro';
     }
 
+    /** An admin, or a legacy call that may ignore ownership (access_all_exception) */
+    public function hasAdminRights(): bool
+    {
+        return $this->isAdmin() || $this->accessAll;
+    }
+
+    /**
+     * Admin rights; a write also needs more than a read-only API key.
+     *
+     * @throws AccessDenied
+     */
+    public function requireAdmin(bool $write = true): void
+    {
+        if (!$this->hasAdminRights() || ($write && $this->isReadOnlyApi())) {
+            throw new AccessDenied();
+        }
+    }
+
     public function hasAcl(string $name): bool
     {
         return isset($this->acl[$name]) && $this->acl[$name] == '1';

data/web/inc/lib/mailcow/Dovecot/DovecotModule.php

@@ -0,0 +1,22 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Dovecot;
+
+use Mailcow\Dovecot\Service\DovecotService;
+
+/**
+ * Dovecot through dockerapi (ACLs, subscriptions, auth cache). Modules use it instead of calling docker().
+ */
+class DovecotModule
+{
+    public function __construct(public readonly DovecotService $dovecot)
+    {
+    }
+
+    public static function create(): self
+    {
+        return new self(new DovecotService());
+    }
+}

data/web/inc/lib/mailcow/Dovecot/Service/DovecotService.php

@@ -0,0 +1,54 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Dovecot\Service;
+
+/**
+ * doveadm in the dovecot container through dockerapi. The docker() call sits in a protected method so tests can
+ * replace it.
+ */
+class DovecotService
+{
+    /**
+     * Sets the rights of an ACL identifier (user=<name>, group=<name>) on every folder of the owner; no rights
+     * remove it.
+     *
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function setAclAll(string $owner, string $identifier, array $rights): ?string
+    {
+        return self::error($this->exec(['cmd' => 'doveadm', 'task' => 'set_acl_all', 'user' => $owner, 'id' => $identifier,
+            'rights' => array_values($rights)]));
+    }
+
+    /**
+     * Subscribes the users to every folder of the owner they see under Shared/<owner>/, or unsubscribes them.
+     *
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function subscribeShared(string $owner, array $usernames, bool $unsubscribe = false): ?string
+    {
+        return self::error($this->exec(['cmd' => 'doveadm', 'task' => 'subscribe_shared', 'owner' => $owner,
+            'users' => array_values($usernames), 'unsubscribe' => $unsubscribe]));
+    }
+
+    /** Drops the cached userdb results, so changed userdb fields apply at the next login instead of after auth_cache_ttl */
+    public function flushAuthCache(array $usernames): void
+    {
+        if (!empty($usernames)) {
+            $this->exec(['cmd' => 'doveadm', 'task' => 'auth_cache_flush', 'users' => array_values($usernames)]);
+        }
+    }
+
+    protected function exec(array $fields): mixed
+    {
+        return docker('post', 'dovecot-mailcow', 'exec', $fields);
+    }
+
+    private static function error(mixed $response): ?string
+    {
+        $result = json_decode((string)$response, true);
+        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
+    }
+}

data/web/inc/lib/mailcow/Identity/Controller/IdentityApiController.php

@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Controller;
+
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use Mailcow\Identity\Service\StatusService;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php (routes `identity-provider/sync-status`, `identity-provider-sync`,
+ * `identity-link`). Messages are logged as ['identity_provider', action, object, data].
+ */
+class IdentityApiController
+{
+    public function __construct(private StatusService $status, private ApiResponse $response)
+    {
+    }
+
+    /** GET identity-provider/sync-status */
+    public function getStatus(): array
+    {
+        try {
+            return $this->status->status();
+        } catch (UserError) {
+            return [];
+        }
+    }
+
+    /** POST edit/identity-provider-sync: attr mode = preview|run|force */
+    public function requestRun(array $attr): bool
+    {
+        return $this->handle('edit', 'sync', $attr, function (array $log) use ($attr): bool {
+            $this->status->requestRun($attr['mode'] ?? 'run');
+            return $this->response->success('iam_sync_requested', $log);
+        });
+    }
+
+    /** GET identity-link/<username> */
+    public function getLink(string $username): array
+    {
+        return $this->status->link($username) ?? [];
+    }
+
+    /** POST delete/identity-link: usernames */
+    public function unlink(array $items): bool
+    {
+        return $this->handle('delete', 'link', $items, function (array $log) use ($items): bool {
+            foreach ($items as $username) {
+                $this->status->unlink((string)$username);
+                $this->response->success(['iam_link_removed', htmlspecialchars((string)$username)], $log);
+            }
+            return true;
+        });
+    }
+
+    private function handle(string $action, string $object, mixed $data, callable $work): bool
+    {
+        $log = ['identity_provider', $action, $object, $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+}

data/web/inc/lib/mailcow/Identity/Directory/Directory.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Directory;
+
+use Mailcow\Identity\Model\Listing;
+
+/**
+ * Lists users (and groups) of an IdP for the sync. Implementations: KeycloakDirectory, LdapDirectory.
+ */
+interface Directory
+{
+    /**
+     * The complete listing. Anything that could make it incomplete (HTTP or LDAP errors, size limits, ranged
+     * attributes) throws, because the sync treats users that are not listed as removed.
+     *
+     * @throws DirectoryException
+     */
+    public function listAll(bool $withGroups): Listing;
+}

data/web/inc/lib/mailcow/Identity/Directory/DirectoryException.php

@@ -0,0 +1,14 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Directory;
+
+use RuntimeException;
+
+/**
+ * The IdP could not be listed completely; the sync run stops without writing anything.
+ */
+class DirectoryException extends RuntimeException
+{
+}

data/web/inc/lib/mailcow/Identity/Directory/KeycloakDirectory.php

@@ -0,0 +1,105 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Directory;
+
+use Mailcow\Identity\Http\KeycloakClient;
+use Mailcow\Identity\Model\DirectoryGroup;
+use Mailcow\Identity\Model\DirectoryUser;
+use Mailcow\Identity\Model\Listing;
+use Mailcow\Identity\Model\Settings;
+
+/**
+ * Keycloak realm users and the groups selected by their path. Only direct group members count.
+ */
+class KeycloakDirectory implements Directory
+{
+    private const PAGE = 100;
+
+    public function __construct(private KeycloakClient $client, private Settings $settings)
+    {
+    }
+
+    public function listAll(bool $withGroups): Listing
+    {
+        $users = [];
+        foreach ($this->paged('/users', ['briefRepresentation' => 'false']) as $row) {
+            if (isset($row['id'])) {
+                $user = self::user($row);
+                $users[$user->id] = $user;
+            }
+        }
+        return new Listing($users, $withGroups ? $this->groups($users) : null);
+    }
+
+    /** A user of the admin REST API */
+    public static function user(array $row): DirectoryUser
+    {
+        $email = (isset($row['email']) && $row['email'] !== '') ? strtolower((string)$row['email']) : null;
+        return new DirectoryUser((string)$row['id'], $email, trim(($row['firstName'] ?? '') . ' ' . ($row['lastName'] ?? '')),
+            !empty($row['enabled']), array_map('strval', (array)($row['attributes']['mailcow_template'] ?? [])));
+    }
+
+    /**
+     * @param array<string, DirectoryUser> $users
+     * @return array<string, DirectoryGroup> selected groups, members limited to listed users
+     */
+    private function groups(array $users): array
+    {
+        $groups = [];
+        foreach ($this->paged('/groups', ['briefRepresentation' => 'false']) as $row) {
+            $this->collect($row, $groups);
+        }
+        foreach ($groups as $id => $group) {
+            foreach ($this->paged('/groups/' . rawurlencode((string)$id) . '/members', ['briefRepresentation' => 'true']) as $member) {
+                if (isset($member['id'], $users[$member['id']])) {
+                    $group->memberIds[] = (string)$member['id'];
+                }
+            }
+        }
+        return $groups;
+    }
+
+    /** Adds the group and its subgroups that match the configured paths (Keycloak >= 23 lists children separately) */
+    private function collect(array $row, array &$groups): void
+    {
+        $path = (string)($row['path'] ?? '/' . ($row['name'] ?? ''));
+        if ($this->selected($path)) {
+            $mail = $row['attributes']['mail'][0] ?? null;
+            $groups[(string)$row['id']] = new DirectoryGroup((string)$row['id'], (string)($row['name'] ?? $path),
+                is_string($mail) && $mail !== '' ? strtolower($mail) : null);
+        }
+        $children = $row['subGroups'] ?? [];
+        if (empty($children) && (int)($row['subGroupCount'] ?? 0) > 0) {
+            $children = $this->paged('/groups/' . rawurlencode((string)$row['id']) . '/children', ['briefRepresentation' => 'false']);
+        }
+        foreach ($children as $child) {
+            $this->collect($child, $groups);
+        }
+    }
+
+    /** '/a' exactly, '/a/*' every group below /a, '/*' every group */
+    private function selected(string $path): bool
+    {
+        foreach ($this->settings->groupPaths as $pattern) {
+            if ($pattern === $path || (str_ends_with($pattern, '/*') && str_starts_with($path, substr($pattern, 0, -1)))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /** @return array[] all rows of a paged list endpoint */
+    private function paged(string $path, array $query): array
+    {
+        $rows = [];
+        for ($first = 0; ; $first += self::PAGE) {
+            $page = array_values($this->client->get($path, $query + ['first' => $first, 'max' => self::PAGE]));
+            $rows = array_merge($rows, $page);
+            if (count($page) < self::PAGE) {
+                return $rows;
+            }
+        }
+    }
+}

data/web/inc/lib/mailcow/Identity/Directory/LdapDirectory.php

@@ -0,0 +1,240 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Directory;
+
+use LdapRecord\Connection;
+use Mailcow\Identity\Model\DirectoryGroup;
+use Mailcow\Identity\Model\DirectoryUser;
+use Mailcow\Identity\Model\Listing;
+use Mailcow\Identity\Model\Settings;
+
+/**
+ * LDAP / Active Directory users and the groups matching the group filter.
+ *
+ * Searches run on the raw connection with paged results and check the result code of every page, because
+ * LdapRecord's paginator stops silently on size limits. Group members come from memberOf (default), from the
+ * member attribute of the groups, or from the AD in-chain rule (nested groups).
+ */
+class LdapDirectory implements Directory
+{
+    private const PAGE = 500;
+    private const ID_ATTRIBUTES = ['objectguid', 'entryuuid', 'nsuniqueid', 'ipauniqueid'];
+    private const AD_IN_CHAIN = '1.2.840.113556.1.4.1941';
+    /** userAccountControl: ACCOUNTDISABLE */
+    private const UAC_DISABLED = 0x2;
+
+    public function __construct(private ?Connection $connection, private Settings $settings)
+    {
+    }
+
+    public function listAll(bool $withGroups): Listing
+    {
+        $s = $this->settings;
+        $userFilter = $this->userFilter();
+        $disabled = [];
+        if ($s->disabledFilter !== '') {
+            foreach ($this->search($s->basedn, '(&' . $userFilter . self::wrap($s->disabledFilter) . ')', ['dn']) as $entry) {
+                $disabled[strtolower((string)$entry['dn'])] = true;
+            }
+        }
+        $attributes = array_merge([$s->usernameField, 'displayname', 'cn', 'useraccountcontrol'],
+            ($s->attributeField !== '') ? [$s->attributeField] : [],
+            self::idAttributes($s->idAttribute),
+            ($withGroups && $s->groupMemberMode === 'memberof') ? ['memberof'] : []);
+        $users = [];
+        $memberOf = [];
+        foreach ($this->search($s->basedn, $userFilter, array_values(array_unique($attributes))) as $entry) {
+            $dn = strtolower((string)$entry['dn']);
+            $email = self::first($entry, $s->usernameField);
+            $uac = self::first($entry, 'useraccountcontrol');
+            $user = new DirectoryUser(self::entryId($entry, $s->idAttribute),
+                ($email === null) ? null : strtolower($email),
+                self::first($entry, 'displayname') ?? self::first($entry, 'cn') ?? '',
+                !isset($disabled[$dn]) && !($uac !== null && ((int)$uac & self::UAC_DISABLED)),
+                ($s->attributeField !== '') ? self::values($entry, $s->attributeField) : [],
+                $dn);
+            $users[$user->id] = $user;
+            $memberOf[$user->id] = array_map('strtolower', self::values($entry, 'memberof'));
+        }
+        return new Listing($users, $withGroups ? $this->groups($users, $memberOf, $userFilter) : null);
+    }
+
+    /** Attributes entryId() reads */
+    public static function idAttributes(string $configured): array
+    {
+        return ($configured !== '') ? [$configured] : self::ID_ATTRIBUTES;
+    }
+
+    /**
+     * Stable id of an entry: the configured attribute, else objectGUID (as GUID string), entryUUID, nsUniqueId,
+     * ipaUniqueID, else the DN. Also used at login, with an entry of an LdapRecord raw query.
+     */
+    public static function entryId(array $entry, string $configured = ''): string
+    {
+        foreach (self::idAttributes($configured) as $attribute) {
+            $value = self::first($entry, $attribute);
+            if ($value !== null && $value !== '') {
+                return ($attribute === 'objectguid' && strlen($value) === 16) ? self::guid($value) : strtolower($value);
+            }
+        }
+        return 'dn:' . strtolower((string)(self::first($entry, 'dn') ?? ''));
+    }
+
+    /**
+     * @param array<string, DirectoryUser> $users
+     * @param array<string, string[]> $memberOf user id => lowercase group DNs
+     * @return array<string, DirectoryGroup>
+     */
+    private function groups(array $users, array $memberOf, string $userFilter): array
+    {
+        $s = $this->settings;
+        if ($s->groupFilter === '') {
+            throw new DirectoryException('The LDAP group filter is empty');
+        }
+        $userIds = [];
+        foreach ($users as $user) {
+            $userIds[$user->dn] = $user->id;
+        }
+        $attributes = array_merge([$s->groupNameAttribute, 'mail'], self::ID_ATTRIBUTES,
+            ($s->groupMemberMode === 'member') ? [$s->groupMemberAttribute] : []);
+        $groups = [];
+        $groupIds = [];
+        foreach ($this->search(($s->groupBasedn !== '') ? $s->groupBasedn : $s->basedn, self::wrap($s->groupFilter),
+            array_values(array_unique($attributes))) as $entry) {
+            $dn = strtolower((string)$entry['dn']);
+            $mail = self::first($entry, 'mail');
+            $group = new DirectoryGroup(self::entryId($entry), self::first($entry, $s->groupNameAttribute) ?? $dn,
+                ($mail === null) ? null : strtolower($mail));
+            if ($s->groupMemberMode === 'member') {
+                foreach (array_keys($entry) as $key) {
+                    // AD returns large member lists in ranges; a partial list would remove members
+                    if (is_string($key) && str_starts_with($key, $s->groupMemberAttribute . ';range=')) {
+                        throw new DirectoryException('LDAP returned a ranged member list for ' . $dn . ', use the memberOf or AD mode');
+                    }
+                }
+                foreach (self::values($entry, $s->groupMemberAttribute) as $memberDn) {
+                    if (isset($userIds[strtolower($memberDn)])) {
+                        $group->memberIds[] = $userIds[strtolower($memberDn)];
+                    }
+                }
+            }
+            $groups[$group->id] = $group;
+            $groupIds[$dn] = $group->id;
+        }
+        if ($s->groupMemberMode === 'memberof') {
+            foreach ($memberOf as $userId => $dns) {
+                foreach ($dns as $dn) {
+                    if (isset($groupIds[$dn])) {
+                        $groups[$groupIds[$dn]]->memberIds[] = (string)$userId;
+                    }
+                }
+            }
+        } elseif ($s->groupMemberMode === 'ad_chain') {
+            foreach ($groupIds as $dn => $groupId) {
+                $filter = '(&' . $userFilter . '(memberOf:' . self::AD_IN_CHAIN . ':=' . self::escape($dn) . '))';
+                foreach ($this->search($s->basedn, $filter, ['dn']) as $entry) {
+                    $userDn = strtolower((string)$entry['dn']);
+                    if (isset($userIds[$userDn])) {
+                        $groups[$groupId]->memberIds[] = $userIds[$userDn];
+                    }
+                }
+            }
+        }
+        foreach ($groups as $group) {
+            $group->memberIds = array_values(array_unique($group->memberIds));
+        }
+        return $groups;
+    }
+
+    /** Users of the configured filter that have the username attribute */
+    private function userFilter(): string
+    {
+        $filter = ($this->settings->filter !== '') ? self::wrap($this->settings->filter) : '(objectClass=*)';
+        return '(&' . $filter . '(' . $this->settings->usernameField . '=*))';
+    }
+
+    /**
+     * Paged search that fails instead of returning a partial result.
+     *
+     * @return array[] entries as returned by ldap_get_entries()
+     * @throws DirectoryException
+     */
+    protected function search(string $base, string $filter, array $attributes): array
+    {
+        if ($this->connection === null) {
+            throw new DirectoryException('No connection to the LDAP server');
+        }
+        $link = $this->connection->getLdapConnection()->getConnection();
+        $entries = [];
+        $cookie = '';
+        $paged = true;
+        while (true) {
+            $controls = $paged ? [['oid' => LDAP_CONTROL_PAGEDRESULTS, 'iscritical' => true,
+                'value' => ['size' => self::PAGE, 'cookie' => $cookie]]] : [];
+            $result = @ldap_search($link, $base, $filter, $attributes, 0, 0, 0, LDAP_DEREF_NEVER, $controls);
+            if ($result === false) {
+                throw new DirectoryException('LDAP search failed: ' . ldap_error($link));
+            }
+            $responseControls = [];
+            if (!ldap_parse_result($link, $result, $code, $matched, $message, $referrals, $responseControls)) {
+                throw new DirectoryException('LDAP result could not be read: ' . ldap_error($link));
+            }
+            // 12 = unavailableCriticalExtension: no paging on this server, search once without it
+            if ($code === 12 && $paged && $cookie === '') {
+                $paged = false;
+                continue;
+            }
+            if ($code !== 0) {
+                throw new DirectoryException('LDAP search incomplete (code ' . $code . '): ' . (($message !== '') ? $message : ldap_err2str($code)));
+            }
+            $page = ldap_get_entries($link, $result);
+            for ($i = 0; $i < (int)($page['count'] ?? 0); $i++) {
+                $entries[] = $page[$i];
+            }
+            $cookie = $paged ? (string)($responseControls[LDAP_CONTROL_PAGEDRESULTS]['value']['cookie'] ?? '') : '';
+            if ($cookie === '') {
+                return $entries;
+            }
+        }
+    }
+
+    /** First value of an attribute: ldap_get_entries() format or LdapRecord attributes */
+    private static function first(array $entry, string $attribute): ?string
+    {
+        $values = self::values($entry, $attribute);
+        return $values[0] ?? null;
+    }
+
+    /** @return string[] */
+    private static function values(array $entry, string $attribute): array
+    {
+        $raw = $entry[$attribute] ?? $entry[strtolower($attribute)] ?? null;
+        if (!is_array($raw)) {
+            return is_scalar($raw) ? [(string)$raw] : [];
+        }
+        unset($raw['count']);
+        return array_values(array_map('strval', array_filter($raw, 'is_scalar')));
+    }
+
+    private static function wrap(string $filter): string
+    {
+        return str_starts_with($filter, '(') ? $filter : '(' . $filter . ')';
+    }
+
+    /** RFC 4515 value escaping */
+    private static function escape(string $value): string
+    {
+        return str_replace(['\\', '*', '(', ')', "\0"], ['\\5c', '\\2a', '\\28', '\\29', '\\00'], $value);
+    }
+
+    /** Binary objectGUID as the usual GUID string */
+    private static function guid(string $binary): string
+    {
+        $hex = bin2hex($binary);
+        return substr($hex, 6, 2) . substr($hex, 4, 2) . substr($hex, 2, 2) . substr($hex, 0, 2) . '-'
+            . substr($hex, 10, 2) . substr($hex, 8, 2) . '-' . substr($hex, 14, 2) . substr($hex, 12, 2) . '-'
+            . substr($hex, 16, 4) . '-' . substr($hex, 20, 12);
+    }
+}

data/web/inc/lib/mailcow/Identity/Http/KeycloakClient.php

@@ -0,0 +1,92 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Http;
+
+use Mailcow\Identity\Directory\DirectoryException;
+
+/**
+ * Admin REST API of a Keycloak realm with the client_credentials token of the mailcow client
+ * (service account roles: view-users, query-groups). The HTTP call sits in send() so tests can replace it.
+ */
+class KeycloakClient
+{
+    private ?string $token = null;
+
+    public function __construct(private string $serverUrl, private string $realm, private string $clientId,
+        private string $clientSecret, private bool $ignoreSslError = false)
+    {
+    }
+
+    /**
+     * GET below /admin/realms/<realm>, decoded JSON. A rejected token is renewed once.
+     *
+     * @throws DirectoryException
+     */
+    public function get(string $path, array $query = []): array
+    {
+        $url = $this->serverUrl . '/admin/realms/' . rawurlencode($this->realm) . $path . (empty($query) ? '' : '?' . http_build_query($query));
+        for ($attempt = 0; $attempt < 2; $attempt++) {
+            $this->token ??= $this->fetchToken();
+            [$code, $body] = $this->send('GET', $url, ['Authorization: Bearer ' . $this->token, 'Accept: application/json'], null);
+            if ($code === 401 && $attempt === 0) {
+                $this->token = null;
+                continue;
+            }
+            if ($code !== 200) {
+                throw new DirectoryException('Keycloak returned HTTP ' . $code . ' for ' . $path);
+            }
+            $data = json_decode($body, true);
+            if (!is_array($data)) {
+                throw new DirectoryException('Keycloak returned invalid JSON for ' . $path);
+            }
+            return $data;
+        }
+        throw new DirectoryException('Keycloak rejected the access token of the mailcow client');
+    }
+
+    /** @throws DirectoryException */
+    private function fetchToken(): string
+    {
+        [$code, $body] = $this->send('POST', $this->serverUrl . '/realms/' . rawurlencode($this->realm) . '/protocol/openid-connect/token',
+            ['Content-Type: application/x-www-form-urlencoded'],
+            http_build_query(['grant_type' => 'client_credentials', 'client_id' => $this->clientId, 'client_secret' => $this->clientSecret]));
+        $data = json_decode($body, true);
+        if ($code !== 200 || !is_array($data) || empty($data['access_token'])) {
+            throw new DirectoryException('Keycloak token request failed (HTTP ' . $code . ')');
+        }
+        return (string)$data['access_token'];
+    }
+
+    /**
+     * @return array{0: int, 1: string} HTTP status and body
+     * @throws DirectoryException on transport errors
+     */
+    protected function send(string $method, string $url, array $headers, ?string $body): array
+    {
+        $curl = curl_init($url);
+        curl_setopt_array($curl, [
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_CONNECTTIMEOUT => 5,
+            CURLOPT_TIMEOUT => 30,
+            CURLOPT_HTTPHEADER => $headers,
+        ]);
+        if ($method === 'POST') {
+            curl_setopt($curl, CURLOPT_POST, true);
+            curl_setopt($curl, CURLOPT_POSTFIELDS, (string)$body);
+        }
+        if ($this->ignoreSslError) {
+            curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
+            curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
+        }
+        $response = curl_exec($curl);
+        $code = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE);
+        $error = curl_error($curl);
+        curl_close($curl);
+        if ($response === false) {
+            throw new DirectoryException('Keycloak request failed: ' . $error);
+        }
+        return [$code, (string)$response];
+    }
+}

data/web/inc/lib/mailcow/Identity/IdentityModule.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity;
+
+use Closure;
+use LdapRecord\Connection;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\Identity\Controller\IdentityApiController;
+use Mailcow\Identity\Directory\Directory;
+use Mailcow\Identity\Directory\KeycloakDirectory;
+use Mailcow\Identity\Directory\LdapDirectory;
+use Mailcow\Identity\Http\KeycloakClient;
+use Mailcow\Identity\Legacy\LegacyAccounts;
+use Mailcow\Identity\Model\Settings;
+use Mailcow\Identity\Repository\IdentityRepository;
+use Mailcow\Identity\Repository\RunRepository;
+use Mailcow\Identity\Service\LinkService;
+use Mailcow\Identity\Service\StatusService;
+use Mailcow\Identity\Service\SyncPlanner;
+use Mailcow\Identity\Service\SyncService;
+use Mailcow\MailboxGroup\MailboxGroupModule;
+use Mailcow\Sogo\SogoModule;
+use PDO;
+
+/**
+ * Composition root of the identity module (IdP links, sync, offboarding).
+ *
+ *   IdentityModule::create($pdo, Context::system(), identity_provider('get'), fn() => identity_provider('init'))->sync->runDue();  // cron
+ *   IdentityModule::fromGlobals()->api->getLink($username);  // edit page
+ *
+ * Logins use the links through Mailcow\Auth (AuthModule::fromGlobals()->idp).
+ */
+class IdentityModule
+{
+    public function __construct(
+        public readonly Settings $settings,
+        public readonly SyncService $sync,
+        public readonly LinkService $links,
+        public readonly StatusService $status,
+        public readonly IdentityApiController $api
+    ) {
+    }
+
+    /**
+     * @param array $settings identity_provider('get')
+     * @param ?Closure $ldap returns the bound LDAP connection (identity_provider('init')), called when a run starts
+     */
+    public static function create(PDO $pdo, Context $context, array $settings, ?Closure $ldap = null): self
+    {
+        $config = Settings::fromArray($settings);
+        $repo = new IdentityRepository($pdo);
+        $runs = new RunRepository($pdo);
+        $groups = MailboxGroupModule::create($pdo, Context::system());
+        $status = new StatusService($config, $repo, $runs, $context);
+        $directory = fn(): ?Directory => self::directory($config, $ldap);
+        return new self($config,
+            new SyncService($config, $directory, $repo, $runs, new SyncPlanner(), $groups->idp, $groups->templates, new LegacyAccounts(),
+                SogoModule::create()->sogo),
+            new LinkService($repo), $status, new IdentityApiController($status, new ApiResponse()));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo, $iam_settings and $_SESSION; $context overrides the session caller */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        $settings = $GLOBALS['iam_settings'] ?? null;
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []),
+            is_array($settings) ? $settings : identity_provider('get'), fn(): mixed => $GLOBALS['iam_provider'] ?? null);
+    }
+
+    private static function directory(Settings $settings, ?Closure $ldap): ?Directory
+    {
+        if ($settings->provider === Settings::KEYCLOAK) {
+            return new KeycloakDirectory(new KeycloakClient($settings->serverUrl, $settings->realm, $settings->clientId,
+                $settings->clientSecret, $settings->ignoreSslError), $settings);
+        }
+        if ($settings->provider === Settings::LDAP) {
+            $connection = ($ldap === null) ? null : $ldap();
+            return new LdapDirectory(($connection instanceof Connection) ? $connection : null, $settings);
+        }
+        return null;
+    }
+}

data/web/inc/lib/mailcow/Identity/Legacy/LegacyAccounts.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Legacy;
+
+use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
+
+/**
+ * Bridge to the procedural mailbox code for the IdP sync: creating mailboxes, applying templates.
+ */
+class LegacyAccounts extends LegacyMailbox
+{
+    /**
+     * Creates a mailbox of the IdP from a template, like the IdP login does.
+     *
+     * @return ?string null on success, else the message key of the legacy error
+     */
+    public function createMailbox(string $email, string $name, string $authsource, string $template): ?string
+    {
+        return $this->asSystem(fn(): mixed => $this->callCreate($email, $name, $authsource, $template));
+    }
+
+    protected function callCreate(string $email, string $name, string $authsource, string $template): mixed
+    {
+        [$local, $domain] = explode('@', $email, 2);
+        return mailbox('add', 'mailbox_from_template', ['domain' => $domain, 'local_part' => $local, 'name' => $name,
+            'authsource' => $authsource, 'template' => $template]);
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/Account.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * A mailbox with the authsource of the configured IdP and its link (table `identity_link`).
+ */
+class Account
+{
+    /**
+     * @param int $active       mailbox.active: 1 active, 2 login disabled (still receives mail), 0 inactive
+     * @param bool $idpDisabled the sync set active = 2, so it may undo it
+     * @param ?string $missingSince first complete run that did not list the user
+     */
+    public function __construct(
+        public readonly string $username,
+        public readonly string $name,
+        public readonly int $active,
+        public readonly ?string $externalId = null,
+        public readonly bool $idpDisabled = false,
+        public readonly ?string $missingSince = null,
+        public readonly string $attributeHash = ''
+    ) {
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/DirectoryGroup.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * A selected IdP group with the ids of its member users.
+ */
+class DirectoryGroup
+{
+    /**
+     * @param ?string $mail      group address in the IdP, decides the domain of an auto-created group
+     * @param string[] $memberIds ids of the member users (DirectoryUser::$id)
+     */
+    public function __construct(
+        public readonly string $id,
+        public readonly string $name,
+        public readonly ?string $mail = null,
+        public array $memberIds = []
+    ) {
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/DirectoryUser.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * A user as listed by the IdP. `id` is stable across renames (Keycloak user id, LDAP objectGUID/entryUUID/...).
+ */
+class DirectoryUser
+{
+    /**
+     * @param ?string $email   lowercase, null if the user has none
+     * @param string[] $templateValues values of the template attribute (legacy mapping)
+     * @param string $dn       LDAP only, lowercase, used to resolve group members
+     */
+    public function __construct(
+        public readonly string $id,
+        public readonly ?string $email,
+        public readonly string $name,
+        public readonly bool $enabled,
+        public readonly array $templateValues = [],
+        public readonly string $dn = ''
+    ) {
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/Listing.php

@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * A complete listing of the IdP. Directories never return partial listings, they throw DirectoryException,
+ * so a user missing here is really gone (or out of the filter).
+ */
+class Listing
+{
+    /**
+     * @param array<string, DirectoryUser> $users by id
+     * @param ?array<string, DirectoryGroup> $groups by id, null when groups were not listed
+     */
+    public function __construct(public readonly array $users, public readonly ?array $groups = null)
+    {
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/Settings.php

@@ -0,0 +1,118 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * Identity provider settings (table `identity_provider`, key/value, see identity_provider('get')) with defaults.
+ */
+class Settings
+{
+    public const KEYCLOAK = 'keycloak';
+    public const LDAP = 'ldap';
+    public const GENERIC_OIDC = 'generic-oidc';
+    public const MEMBER_MODES = ['memberof', 'member', 'ad_chain'];
+
+    public string $provider = '';
+    public bool $periodicSync = false;
+    public bool $importUsers = false;
+    /** Minutes between scheduled runs */
+    public int $syncInterval = 15;
+    /** Users disabled or removed in the IdP get login disabled (mailbox.active = 2) */
+    public bool $offboarding = false;
+    /** Max. share (percent) of active IdP mailboxes one run may deactivate, at least 5 mailboxes */
+    public int $offboardingLimit = 10;
+    public bool $groupSync = false;
+    public bool $groupAutocreate = true;
+    public string $groupDefaultDomain = '';
+    public string $defaultTemplate = '';
+    /** @var string[] attribute value per mapping, keys aligned with $templates */
+    public array $mappers = [];
+    /** @var string[] */
+    public array $templates = [];
+    public string $serverUrl = '';
+    public string $realm = '';
+    public string $clientId = '';
+    public string $clientSecret = '';
+    public bool $ignoreSslError = false;
+    /** @var string[] Keycloak group paths: '/path', '/path/*' (subtree) or '/*' (all) */
+    public array $groupPaths = [];
+    public string $basedn = '';
+    public string $filter = '';
+    public string $usernameField = 'mail';
+    public string $attributeField = '';
+    /** LDAP attribute with the stable user id, '' = objectGUID, entryUUID, nsUniqueId, ipaUniqueID, else the DN */
+    public string $idAttribute = '';
+    /** LDAP filter matching disabled users (besides AD userAccountControl) */
+    public string $disabledFilter = '';
+    public string $groupBasedn = '';
+    public string $groupFilter = '';
+    public string $groupNameAttribute = 'cn';
+    public string $groupMemberMode = 'memberof';
+    public string $groupMemberAttribute = 'member';
+
+    public static function fromArray(array $s): self
+    {
+        $settings = new self();
+        $settings->provider = (string)($s['authsource'] ?? '');
+        $settings->periodicSync = self::flag($s['periodic_sync'] ?? 0);
+        $settings->importUsers = self::flag($s['import_users'] ?? 0);
+        $settings->syncInterval = max(1, (int)($s['sync_interval'] ?? 15));
+        $settings->offboarding = self::flag($s['offboarding'] ?? 0);
+        $settings->offboardingLimit = min(100, max(0, (int)($s['offboarding_limit'] ?? 10)));
+        $settings->groupSync = self::flag($s['group_sync'] ?? 0);
+        $settings->groupAutocreate = self::flag($s['group_autocreate'] ?? 1);
+        $settings->groupDefaultDomain = strtolower(trim((string)($s['group_default_domain'] ?? '')));
+        $settings->defaultTemplate = (string)($s['default_template'] ?? '');
+        $settings->mappers = array_map('strval', (array)($s['mappers'] ?? []));
+        $settings->templates = array_map('strval', (array)($s['templates'] ?? []));
+        $settings->serverUrl = rtrim((string)($s['server_url'] ?? ''), '/');
+        $settings->realm = (string)($s['realm'] ?? '');
+        $settings->clientId = (string)($s['client_id'] ?? '');
+        $settings->clientSecret = (string)($s['client_secret'] ?? '');
+        $settings->ignoreSslError = self::flag($s['ignore_ssl_error'] ?? 0);
+        $settings->groupPaths = array_values(array_filter(array_map('trim', explode(',', (string)($s['group_paths'] ?? ''))), 'strlen'));
+        $settings->basedn = (string)($s['basedn'] ?? '');
+        $settings->filter = trim((string)($s['filter'] ?? ''));
+        $settings->usernameField = strtolower((string)(($s['username_field'] ?? '') ?: 'mail'));
+        $settings->attributeField = strtolower(trim((string)($s['attribute_field'] ?? '')));
+        $settings->idAttribute = strtolower(trim((string)($s['id_attribute'] ?? '')));
+        $settings->disabledFilter = trim((string)($s['disabled_filter'] ?? ''));
+        $settings->groupBasedn = trim((string)($s['group_basedn'] ?? ''));
+        $settings->groupFilter = trim((string)($s['group_filter'] ?? ''));
+        $settings->groupNameAttribute = strtolower((string)(($s['group_name_attribute'] ?? '') ?: 'cn'));
+        $mode = (string)($s['group_member_mode'] ?? 'memberof');
+        $settings->groupMemberMode = in_array($mode, self::MEMBER_MODES, true) ? $mode : 'memberof';
+        $settings->groupMemberAttribute = strtolower((string)(($s['group_member_attribute'] ?? '') ?: 'member'));
+        return $settings;
+    }
+
+    /** Only Keycloak and LDAP can be listed by the sync */
+    public function canSync(): bool
+    {
+        return in_array($this->provider, [self::KEYCLOAK, self::LDAP], true);
+    }
+
+    public function syncEnabled(): bool
+    {
+        return $this->canSync() && ($this->periodicSync || $this->importUsers || $this->offboarding || $this->groupSync);
+    }
+
+    /** Template of the first mapping (in configured order) matching one of the values, null if none */
+    public function mappedTemplate(array $values): ?string
+    {
+        $values = array_map('strval', array_filter($values, 'is_scalar'));
+        foreach ($this->mappers as $key => $mapper) {
+            if (in_array($mapper, $values, true) && ($this->templates[$key] ?? '') !== '') {
+                return $this->templates[$key];
+            }
+        }
+        return null;
+    }
+
+    private static function flag(mixed $value): bool
+    {
+        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/State.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * What mailcow knows before a sync run, read by SyncService and planned against by SyncPlanner.
+ */
+class State
+{
+    /**
+     * @param array<string, Account> $accounts           mailboxes with the provider as authsource, by username
+     * @param array<string, string> $mailboxes           username => authsource of every mailbox
+     * @param string[] $domains                          mailcow domains
+     * @param array[] $groups                            mailbox groups {id, domain, name, idp_ref, template, priority,
+     *                                                   active, members: username => source}
+     * @param array<string, ?string> $effectiveTemplates username => effective group template
+     * @param array<string, string> $templateHashes      template name => attribute hash
+     */
+    public function __construct(
+        public readonly array $accounts,
+        public readonly array $mailboxes = [],
+        public readonly array $domains = [],
+        public readonly array $groups = [],
+        public readonly array $effectiveTemplates = [],
+        public readonly array $templateHashes = []
+    ) {
+    }
+}

data/web/inc/lib/mailcow/Identity/Model/SyncPlan.php

@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Model;
+
+/**
+ * What one sync run changes. Built by SyncPlanner from a listing and the current state, applied by SyncService.
+ */
+class SyncPlan
+{
+    /** Details per kind kept in the stored summary */
+    private const DETAILS = 50;
+
+    /** @var array<string, string> username => external id (adopted by email or relinked) */
+    public array $links = [];
+    /** @var array[] {email, name, template, id}: mailboxes to create */
+    public array $creates = [];
+    /** @var string[] usernames set to login disabled (active = 2) */
+    public array $deactivate = [];
+    /** @var string[] usernames the sync had disabled and the IdP enabled again (active = 1) */
+    public array $restore = [];
+    /** @var string[] usernames whose idp_disabled marker is dropped, active stays (an admin changed it) */
+    public array $release = [];
+    /** @var string[] usernames not listed for the first time (missing_since is set) */
+    public array $missing = [];
+    /** @var string[] usernames listed again (missing_since is cleared) */
+    public array $found = [];
+    /** @var array[] {username, name, template}: existing mailboxes whose name/template is synced; template = legacy mapping */
+    public array $updates = [];
+    /** @var array<string, array> username => {name, template}: IdP name and legacy mapping of every listed mailbox */
+    public array $profiles = [];
+    /** @var array[] {ref, name, domain, mail}: groups to create for IdP groups */
+    public array $groupCreates = [];
+    /** @var array<int, string> group id => new name */
+    public array $groupRenames = [];
+    /** @var array<string, string[]> group ref => usernames of its IdP members */
+    public array $memberships = [];
+    /** @var int[] linked groups no longer listed; their IdP members are removed */
+    public array $orphanedGroups = [];
+    /** Number of IdP memberships the plan removes (for the guard) */
+    public int $membershipRemovals = 0;
+    /** @var string[] situations an admin has to resolve */
+    public array $conflicts = [];
+    /** Users without email, not in a mailcow domain or without template (not created) */
+    public int $skipped = 0;
+    /** The guard stopped deactivations and membership removals */
+    public bool $blocked = false;
+
+    /** Stored with the run and shown in the UI */
+    public function summary(): array
+    {
+        $summary = ['counts' => [], 'details' => []];
+        foreach (['links' => 'links', 'creates' => 'creates', 'deactivate' => 'deactivate', 'restore' => 'restore',
+            'release' => 'release', 'missing' => 'missing', 'found' => 'found', 'updates' => 'updates', 'group_creates' => 'groupCreates',
+            'group_renames' => 'groupRenames', 'memberships' => 'memberships', 'orphaned_groups' => 'orphanedGroups',
+            'conflicts' => 'conflicts'] as $key => $kind) {
+            $summary['counts'][$key] = count($this->$kind);
+        }
+        $summary['counts']['membership_removals'] = $this->membershipRemovals;
+        $summary['counts']['skipped'] = $this->skipped;
+        $summary['details'] = [
+            'creates' => array_slice(array_column($this->creates, 'email'), 0, self::DETAILS),
+            'deactivate' => array_slice($this->deactivate, 0, self::DETAILS),
+            'restore' => array_slice($this->restore, 0, self::DETAILS),
+            'missing' => array_slice($this->missing, 0, self::DETAILS),
+            'group_creates' => array_slice(array_column($this->groupCreates, 'name'), 0, self::DETAILS),
+            'conflicts' => array_slice($this->conflicts, 0, self::DETAILS),
+        ];
+        $summary['blocked'] = $this->blocked;
+        return $summary;
+    }
+}

data/web/inc/lib/mailcow/Identity/Repository/IdentityRepository.php

@@ -0,0 +1,130 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Repository;
+
+use Mailcow\Core\Repository;
+use Mailcow\Identity\Model\Account;
+use PDO;
+
+/**
+ * SQL for IdP mailboxes and their links (table `identity_link`).
+ */
+class IdentityRepository extends Repository
+{
+    /** @return array<string, Account> mailboxes with the provider as authsource, by username */
+    public function accounts(string $provider): array
+    {
+        $rows = $this->query("SELECT m.`username`, m.`name`, m.`active`,
+            JSON_UNQUOTE(JSON_EXTRACT(m.`attributes`, '$.attribute_hash')) AS `attribute_hash`,
+            l.`external_id`, l.`idp_disabled`, l.`missing_since`
+          FROM `mailbox` m
+            LEFT JOIN `identity_link` l ON l.`username` = m.`username` AND l.`provider` = :p1
+          WHERE m.`authsource` = :p2 AND m.`kind` = ''", ['p1' => $provider, 'p2' => $provider])->fetchAll(PDO::FETCH_ASSOC);
+        $accounts = [];
+        foreach ($rows as $row) {
+            $accounts[(string)$row['username']] = new Account((string)$row['username'], (string)$row['name'], (int)$row['active'],
+                isset($row['external_id']) ? (string)$row['external_id'] : null, (int)($row['idp_disabled'] ?? 0) === 1,
+                isset($row['missing_since']) ? (string)$row['missing_since'] : null, (string)($row['attribute_hash'] ?? ''));
+        }
+        return $accounts;
+    }
+
+    /** @return array{accounts: int, linked: int, idp_disabled: int, missing: int} mailboxes of the provider */
+    public function stats(string $provider): array
+    {
+        $row = $this->query("SELECT COUNT(*) AS `accounts`, COUNT(l.`external_id`) AS `linked`,
+            COALESCE(SUM(l.`idp_disabled`), 0) AS `idp_disabled`, COUNT(l.`missing_since`) AS `missing`
+          FROM `mailbox` m LEFT JOIN `identity_link` l ON l.`username` = m.`username` AND l.`provider` = :p1
+          WHERE m.`authsource` = :p2 AND m.`kind` = ''", ['p1' => $provider, 'p2' => $provider])->fetch(PDO::FETCH_ASSOC);
+        return array_map('intval', $row);
+    }
+
+    /** @return array<string, string> username => authsource of every mailbox (incl. resources) */
+    public function mailboxes(): array
+    {
+        return $this->query("SELECT `username`, `authsource` FROM `mailbox`")->fetchAll(PDO::FETCH_KEY_PAIR);
+    }
+
+    /** @return string[] mailcow domains (no alias domains) */
+    public function domains(): array
+    {
+        return $this->query("SELECT `domain` FROM `domain`")->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** @return array<string, string> template name => hash as stored by mailbox_from_template in attributes.attribute_hash */
+    public function templateHashes(): array
+    {
+        $hashes = [];
+        foreach ($this->query("SELECT `template`, `attributes` FROM `templates` WHERE `type` = 'mailbox'")->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $hashes[(string)$row['template']] = sha1(json_encode($row['attributes']));
+        }
+        return $hashes;
+    }
+
+    /** @return ?array{provider: string, external_id: string, idp_disabled: int, missing_since: ?string} */
+    public function findLink(string $username): ?array
+    {
+        $row = $this->query("SELECT `provider`, `external_id`, `idp_disabled`, `missing_since` FROM `identity_link` WHERE `username` = :username",
+            ['username' => $username])->fetch(PDO::FETCH_ASSOC);
+        return $row ?: null;
+    }
+
+    /** Mailbox linked to the IdP user; links of mailboxes that no longer use the provider do not count */
+    public function usernameOf(string $provider, string $externalId): ?string
+    {
+        $username = $this->query("SELECT l.`username` FROM `identity_link` l
+            JOIN `mailbox` m ON m.`username` = l.`username` AND m.`authsource` = l.`provider`
+          WHERE l.`provider` = :provider AND l.`external_id` = :id", ['provider' => $provider, 'id' => $externalId])->fetchColumn();
+        return ($username === false) ? null : (string)$username;
+    }
+
+    /**
+     * Links the mailbox to the IdP user; an older link of the mailbox is replaced. Callers make sure the IdP user
+     * is not linked to another mailbox of the provider (usernameOf()); stale links of the id are dropped.
+     */
+    public function link(string $username, string $provider, string $externalId): void
+    {
+        $this->transaction(function () use ($username, $provider, $externalId): void {
+            $this->query("DELETE FROM `identity_link` WHERE `provider` = :provider AND `external_id` = :id AND `username` != :username",
+                ['provider' => $provider, 'id' => $externalId, 'username' => $username]);
+            $this->query("INSERT INTO `identity_link` (`username`, `provider`, `external_id`) VALUES (:username, :provider, :id)
+              ON DUPLICATE KEY UPDATE `provider` = VALUES(`provider`), `external_id` = VALUES(`external_id`), `missing_since` = NULL",
+                ['username' => $username, 'provider' => $provider, 'id' => $externalId]);
+        });
+    }
+
+    public function unlink(string $username): void
+    {
+        $this->query("DELETE FROM `identity_link` WHERE `username` = :username", ['username' => $username]);
+    }
+
+    /** Marks the IdP mailbox as not listed (now) or listed again; unlinked mailboxes get a link row without id */
+    public function setMissing(string $username, string $provider, bool $missing): void
+    {
+        $this->query("INSERT INTO `identity_link` (`username`, `provider`, `missing_since`) VALUES (:username, :provider, "
+            . ($missing ? "NOW()" : "NULL") . ") ON DUPLICATE KEY UPDATE `missing_since` = VALUES(`missing_since`)",
+            ['username' => $username, 'provider' => $provider]);
+    }
+
+    /**
+     * Sets mailbox.active like the mailbox edit does (incl. the mailbox's own alias row) and records whether the
+     * IdP sync caused it.
+     */
+    public function setActive(string $username, string $provider, int $active, bool $idpDisabled): void
+    {
+        $this->transaction(function () use ($username, $provider, $active, $idpDisabled): void {
+            $this->query("UPDATE `mailbox` SET `active` = :active WHERE `username` = :username", ['active' => $active, 'username' => $username]);
+            $this->query("UPDATE `alias` SET `active` = :active WHERE `address` = :username", ['active' => $active, 'username' => $username]);
+            $this->setIdpDisabled($username, $provider, $idpDisabled);
+        });
+    }
+
+    public function setIdpDisabled(string $username, string $provider, bool $idpDisabled): void
+    {
+        $this->query("INSERT INTO `identity_link` (`username`, `provider`, `idp_disabled`) VALUES (:username, :provider, :flag)
+          ON DUPLICATE KEY UPDATE `idp_disabled` = VALUES(`idp_disabled`)",
+            ['username' => $username, 'provider' => $provider, 'flag' => $idpDisabled ? 1 : 0]);
+    }
+}

data/web/inc/lib/mailcow/Identity/Repository/RunRepository.php

@@ -0,0 +1,81 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * Sync runs (table `identity_sync_run`): requests from the UI (status pending), the run itself and its summary.
+ */
+class RunRepository extends Repository
+{
+    private const KEEP = 30;
+
+    /** Queues a run for the cron; an identical pending request is reused */
+    public function request(string $provider, string $mode, string $requestedBy): int
+    {
+        $id = $this->query("SELECT `id` FROM `identity_sync_run` WHERE `status` = 'pending' AND `mode` = :mode",
+            ['mode' => $mode])->fetchColumn();
+        if ($id !== false) {
+            return (int)$id;
+        }
+        return $this->insertRow('identity_sync_run', ['provider' => $provider, 'mode' => $mode, 'status' => 'pending',
+            'requested_by' => $requestedBy]);
+    }
+
+    /** @return ?array oldest pending request */
+    public function nextPending(): ?array
+    {
+        $row = $this->query("SELECT * FROM `identity_sync_run` WHERE `status` = 'pending' ORDER BY `id` LIMIT 1")->fetch(PDO::FETCH_ASSOC);
+        return $row ?: null;
+    }
+
+    /** A scheduled run (not requested by anyone) */
+    public function create(string $provider, string $mode): int
+    {
+        return $this->insertRow('identity_sync_run', ['provider' => $provider, 'mode' => $mode, 'status' => 'pending', 'requested_by' => '']);
+    }
+
+    public function start(int $id): void
+    {
+        $this->query("UPDATE `identity_sync_run` SET `status` = 'running', `started` = NOW() WHERE `id` = :id", ['id' => $id]);
+    }
+
+    public function finish(int $id, string $status, array $summary): void
+    {
+        $this->query("UPDATE `identity_sync_run` SET `status` = :status, `finished` = NOW(), `summary` = :summary WHERE `id` = :id",
+            ['status' => $status, 'summary' => json_encode($summary), 'id' => $id]);
+        $this->query("DELETE FROM `identity_sync_run` WHERE `id` NOT IN (SELECT `id` FROM (SELECT `id` FROM `identity_sync_run`
+          ORDER BY `id` DESC LIMIT " . self::KEEP . ") AS `keep`)");
+    }
+
+    /** Seconds since the last scheduled run started, null if there was none */
+    public function secondsSinceScheduled(): ?int
+    {
+        $seconds = $this->query("SELECT TIMESTAMPDIFF(SECOND, MAX(`started`), NOW()) FROM `identity_sync_run`
+          WHERE `requested_by` = '' AND `mode` = 'run'")->fetchColumn();
+        return ($seconds === false || $seconds === null) ? null : (int)$seconds;
+    }
+
+    /** Runs left in status running by a crashed cron are closed, so they do not hide new runs */
+    public function failStale(int $minutes): void
+    {
+        $this->query("UPDATE `identity_sync_run` SET `status` = 'error', `finished` = NOW(),
+            `summary` = JSON_OBJECT('error', 'The sync run did not finish')
+          WHERE `status` = 'running' AND `started` < NOW() - INTERVAL :minutes MINUTE", ['minutes' => $minutes]);
+    }
+
+    /** @return array[] newest first, summary decoded */
+    public function recent(int $limit = 10): array
+    {
+        $rows = $this->query("SELECT * FROM `identity_sync_run` ORDER BY `id` DESC LIMIT " . max(1, $limit))->fetchAll(PDO::FETCH_ASSOC);
+        foreach ($rows as &$row) {
+            $row['id'] = (int)$row['id'];
+            $row['summary'] = ($row['summary'] === null) ? null : json_decode((string)$row['summary'], true);
+        }
+        return $rows;
+    }
+}

data/web/inc/lib/mailcow/Identity/Service/LinkService.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Service;
+
+use Mailcow\Identity\Repository\IdentityRepository;
+
+/**
+ * Links of mailboxes to IdP users, checked at IdP logins (REST, LDAP bind, OIDC). Mailboxes are found by email,
+ * so the IdP user id has to match the link: an address taken over by another IdP user must not open the old
+ * mailbox. Unlinked mailboxes are linked on their first login.
+ */
+class LinkService
+{
+    public function __construct(private IdentityRepository $repo)
+    {
+    }
+
+    /** The IdP user may log in to the mailbox; links the mailbox when it has no link yet */
+    public function verify(string $provider, string $username, string $externalId): bool
+    {
+        if ($externalId === '') {
+            return false;
+        }
+        $link = $this->repo->findLink($username);
+        if ($link !== null && $link['provider'] === $provider && $link['external_id'] !== null) {
+            return hash_equals((string)$link['external_id'], $externalId);
+        }
+        if (!$this->mayCreate($provider, $username, $externalId)) {
+            return false;
+        }
+        $this->repo->link($username, $provider, $externalId);
+        return true;
+    }
+
+    /** The IdP user owns no other mailbox (checked before a mailbox is created on login) */
+    public function mayCreate(string $provider, string $username, string $externalId): bool
+    {
+        $owner = ($externalId === '') ? null : $this->repo->usernameOf($provider, $externalId);
+        return $externalId !== '' && ($owner === null || $owner === $username);
+    }
+
+    /** Links a mailbox created on login */
+    public function link(string $provider, string $username, string $externalId): void
+    {
+        if ($externalId !== '') {
+            $this->repo->link($username, $provider, $externalId);
+        }
+    }
+}

data/web/inc/lib/mailcow/Identity/Service/StatusService.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Service;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Mailcow\Identity\Model\Settings;
+use Mailcow\Identity\Repository\IdentityRepository;
+use Mailcow\Identity\Repository\RunRepository;
+
+/**
+ * Admin use cases of the sync: status, run requests (executed by the cron) and links of mailboxes.
+ * Admins only, read-only API keys may read. Domain admins may read the link of mailboxes they manage.
+ */
+class StatusService
+{
+    public const MODES = ['preview', 'run', 'force'];
+
+    public function __construct(private Settings $settings, private IdentityRepository $repo, private RunRepository $runs,
+        private Context $context)
+    {
+    }
+
+    /** @throws AccessDenied */
+    public function status(): array
+    {
+        $this->context->requireAdmin(false);
+        return [
+            'provider' => $this->settings->provider,
+            'can_sync' => $this->settings->canSync(),
+            'sync_enabled' => $this->settings->syncEnabled(),
+            'counts' => $this->repo->stats($this->settings->provider),
+            'runs' => $this->runs->recent(),
+        ];
+    }
+
+    /**
+     * Queues a run for the cron: preview (dry run), run, or force (run without the change limit).
+     *
+     * @throws UserError
+     */
+    public function requestRun(mixed $mode): int
+    {
+        $this->context->requireAdmin();
+        $mode = Input::text($mode);
+        if (!in_array($mode, self::MODES, true)) {
+            throw new UserError('iam_sync_mode_invalid');
+        }
+        if (!$this->settings->canSync()) {
+            throw new UserError('iam_sync_unavailable');
+        }
+        return $this->runs->request($this->settings->provider, $mode, (string)$this->context->username);
+    }
+
+    /** Link of a mailbox for the mailbox edit page, null if none or not allowed */
+    public function link(string $username): ?array
+    {
+        $username = strtolower(trim($username));
+        if (!($this->context->hasAdminRights() || ($this->context->isDomainAdmin() && $this->context->canAccessMailbox($username)))) {
+            return null;
+        }
+        return $this->repo->findLink($username);
+    }
+
+    /**
+     * Drops the link, the mailbox is linked again by email (next run or login). Resolves conflicts, e.g. when an
+     * address was given to another IdP user.
+     *
+     * @throws AccessDenied
+     */
+    public function unlink(string $username): void
+    {
+        $this->context->requireAdmin();
+        $this->repo->unlink(strtolower(trim($username)));
+    }
+}

data/web/inc/lib/mailcow/Identity/Service/SyncPlanner.php

@@ -0,0 +1,311 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Service;
+
+use Mailcow\Identity\Model\Account;
+use Mailcow\Identity\Model\DirectoryGroup;
+use Mailcow\Identity\Model\DirectoryUser;
+use Mailcow\Identity\Model\Listing;
+use Mailcow\Identity\Model\Settings;
+use Mailcow\Identity\Model\State;
+use Mailcow\Identity\Model\SyncPlan;
+
+/**
+ * Decides what a sync run changes, without side effects.
+ *
+ * Users: linked mailboxes are found by IdP user id, unlinked ones are adopted by email. A mailbox linked to
+ *        another IdP user, or an IdP user whose email moved away from its mailbox, is a conflict; nothing is
+ *        relinked or created for it.
+ * Offboarding: a user disabled in the IdP, or not listed by two consecutive complete runs, gets login disabled
+ *        (active = 2), once per IdP disable. Only those mailboxes are activated again when the IdP enables
+ *        the user; manual changes of active always win.
+ * Groups: IdP groups map to mailbox groups by idp_ref; unknown ones are created in the domain of their mail,
+ *        else in the default group domain. Linked users that are not listed keep their IdP memberships.
+ * Guard: more deactivations or membership removals than max(5, limit % of active IdP mailboxes resp. IdP
+ *        memberships) block the run unless forced.
+ */
+class SyncPlanner
+{
+    private const GUARD_MIN = 5;
+
+    public function plan(Settings $settings, Listing $listing, State $state, bool $force = false): SyncPlan
+    {
+        $plan = new SyncPlan();
+        $owners = [];
+        foreach ($state->accounts as $username => $account) {
+            if ($account->externalId !== null) {
+                $owners[$account->externalId] = (string)$username;
+            }
+        }
+        /** @var array<string, DirectoryUser> $matched username => listed user */
+        $matched = [];
+        /** @var array<string, DirectoryUser> $candidates email => user to create, template still open */
+        $candidates = [];
+        $duplicates = [];
+        foreach ($listing->users as $user) {
+            if (isset($owners[$user->id])) {
+                $username = $owners[$user->id];
+                if ($user->email !== null && $user->email !== $username) {
+                    self::conflict($plan, $username . ' is linked to an IdP user whose email is now ' . $user->email);
+                }
+                $matched[$username] = $user;
+            } elseif ($user->email === null) {
+                $plan->skipped++;
+            } elseif (isset($state->accounts[$user->email])) {
+                $account = $state->accounts[$user->email];
+                if ($account->externalId !== null || isset($plan->links[$user->email])) {
+                    self::conflict($plan, $user->email . ' is linked to another IdP user');
+                    continue;
+                }
+                $plan->links[$user->email] = $user->id;
+                $matched[$user->email] = $user;
+            } elseif (isset($state->mailboxes[$user->email]) || !$settings->importUsers || !$user->enabled
+                || !in_array(substr((string)strrchr($user->email, '@'), 1), $state->domains, true)) {
+                // other authsource, import off, disabled or no mailcow domain
+                $plan->skipped++;
+            } elseif (isset($candidates[$user->email]) || isset($duplicates[$user->email])) {
+                self::conflict($plan, 'Several IdP users have the email ' . $user->email);
+                unset($candidates[$user->email]);
+                $duplicates[$user->email] = true;
+            } else {
+                $candidates[$user->email] = $user;
+            }
+        }
+        $groups = ($settings->groupSync && $listing->groups !== null) ? $listing->groups : null;
+        $this->planCreates($settings, $candidates, $groups ?? [], $state, $plan);
+        $usernames = [];
+        foreach ($matched as $username => $user) {
+            $usernames[$user->id] = (string)$username;
+        }
+        foreach ($plan->creates as $create) {
+            $usernames[$create['id']] = $create['email'];
+        }
+        if ($groups !== null) {
+            $this->planGroups($settings, $groups, $state, $usernames, $matched, $plan);
+        }
+        $this->planStates($settings, $state, $matched, $plan);
+        if ($settings->periodicSync) {
+            $this->planUpdates($settings, $state, $matched, $plan);
+        }
+        $active = count(array_filter($state->accounts, fn(Account $a): bool => $a->active === 1));
+        $memberships = array_sum(array_map(fn(array $g): int => count(self::idpMembers($g)), $state->groups));
+        if (!$force && (count($plan->deactivate) > self::limit($active, $settings->offboardingLimit)
+            || $plan->membershipRemovals > self::limit($memberships, $settings->offboardingLimit))) {
+            $plan->blocked = true;
+        }
+        return $plan;
+    }
+
+    /**
+     * @param array<string, DirectoryGroup> $groups
+     * @param array<string, string> $usernames user id => username of matched and new users
+     */
+    private function planGroups(Settings $settings, array $groups, State $state, array $usernames, array $matched, SyncPlan $plan): void
+    {
+        $linked = [];
+        $names = [];
+        foreach ($state->groups as $group) {
+            if ($group['idp_ref'] !== null) {
+                $linked[(string)$group['idp_ref']] = $group;
+            }
+            $names[$group['domain']][mb_strtolower($group['name'])] = (int)$group['id'];
+        }
+        foreach ($groups as $ref => $directoryGroup) {
+            $ref = (string)$ref;
+            $members = [];
+            foreach ($directoryGroup->memberIds as $id) {
+                if (isset($usernames[$id])) {
+                    $members[] = $usernames[$id];
+                }
+            }
+            $current = $linked[$ref] ?? null;
+            if ($current === null) {
+                if (!$settings->groupAutocreate) {
+                    continue;
+                }
+                $domain = self::groupDomain($settings, $directoryGroup, $state->domains);
+                if ($domain === null) {
+                    self::conflict($plan, 'IdP group ' . $directoryGroup->name . ' has no mailcow domain');
+                    continue;
+                }
+                if (isset($names[$domain][mb_strtolower($directoryGroup->name)])) {
+                    self::conflict($plan, 'A group ' . $directoryGroup->name . ' exists in ' . $domain . ' and is not linked to the IdP group');
+                    continue;
+                }
+                $names[$domain][mb_strtolower($directoryGroup->name)] = 0;
+                $plan->groupCreates[] = ['ref' => $ref, 'name' => $directoryGroup->name, 'domain' => $domain, 'mail' => $directoryGroup->mail];
+                $plan->memberships[$ref] = self::sorted($members);
+                continue;
+            }
+            $idpMembers = self::idpMembers($current);
+            // Linked users missing in the listing keep their memberships until they are listed again
+            foreach ($idpMembers as $username) {
+                if (isset($state->accounts[$username]) && !isset($matched[$username])) {
+                    $members[] = $username;
+                }
+            }
+            $members = self::sorted($members);
+            $removed = array_diff($idpMembers, $members);
+            if (!empty($removed) || !empty(array_diff($members, array_map('strval', array_keys($current['members']))))) {
+                $plan->memberships[$ref] = $members;
+                $plan->membershipRemovals += count($removed);
+            }
+            if ($directoryGroup->name !== $current['name']) {
+                $taken = $names[$current['domain']][mb_strtolower($directoryGroup->name)] ?? null;
+                if ($taken !== null && $taken !== (int)$current['id']) {
+                    self::conflict($plan, 'IdP group ' . $current['name'] . ' was renamed to ' . $directoryGroup->name . ', which exists in ' . $current['domain']);
+                } else {
+                    $plan->groupRenames[(int)$current['id']] = $directoryGroup->name;
+                    $names[$current['domain']][mb_strtolower($directoryGroup->name)] = (int)$current['id'];
+                }
+            }
+        }
+        foreach ($linked as $ref => $group) {
+            if (!isset($groups[$ref])) {
+                $plan->orphanedGroups[] = (int)$group['id'];
+                $plan->membershipRemovals += count(self::idpMembers($group));
+            }
+        }
+    }
+
+    /**
+     * Template of a new mailbox: the template of its highest-priority linked group (as in TemplateService), else
+     * the attribute mapping, else the default template.
+     *
+     * @param array<string, DirectoryUser> $candidates
+     * @param array<string, DirectoryGroup> $groups listed groups, [] without group sync
+     */
+    private function planCreates(Settings $settings, array $candidates, array $groups, State $state, SyncPlan $plan): void
+    {
+        $linked = [];
+        foreach ($state->groups as $group) {
+            if ($group['idp_ref'] !== null && $group['template'] !== null && $group['active'] === 1) {
+                $linked[(string)$group['idp_ref']] = $group;
+            }
+        }
+        $groupsOf = [];
+        foreach ($groups as $ref => $directoryGroup) {
+            if (isset($linked[(string)$ref])) {
+                foreach ($directoryGroup->memberIds as $id) {
+                    $groupsOf[$id][] = $linked[(string)$ref];
+                }
+            }
+        }
+        foreach ($candidates as $email => $user) {
+            $best = null;
+            foreach ($groupsOf[$user->id] ?? [] as $group) {
+                if ($best === null || $group['priority'] > $best['priority'] || ($group['priority'] === $best['priority'] && $group['id'] < $best['id'])) {
+                    $best = $group;
+                }
+            }
+            $template = $best['template'] ?? $settings->mappedTemplate($user->templateValues)
+                ?? (($settings->defaultTemplate !== '') ? $settings->defaultTemplate : null);
+            if ($template === null) {
+                self::conflict($plan, 'No template matches ' . $email);
+                continue;
+            }
+            $plan->creates[] = ['email' => (string)$email, 'name' => self::name($user->name, ''), 'template' => $template, 'id' => $user->id];
+        }
+    }
+
+    /** @param array<string, DirectoryUser> $matched */
+    private function planStates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
+    {
+        foreach ($state->accounts as $username => $account) {
+            $username = (string)$username;
+            $user = $matched[$username] ?? null;
+            if ($user === null && $account->missingSince === null) {
+                $plan->missing[] = $username;
+                continue;
+            }
+            if ($user !== null && $account->missingSince !== null) {
+                $plan->found[] = $username;
+            }
+            if (!$settings->offboarding) {
+                continue;
+            }
+            // Second complete run without the user counts as removed from the IdP
+            $disabled = ($user === null) || !$user->enabled;
+            if ($disabled && !$account->idpDisabled && $account->active === 1) {
+                $plan->deactivate[] = $username;
+            } elseif (!$disabled && $account->idpDisabled) {
+                if ($account->active === 2) {
+                    $plan->restore[] = $username;
+                } else {
+                    $plan->release[] = $username;
+                }
+            }
+        }
+    }
+
+    /**
+     * Name and template of linked mailboxes, like the IdP login does. The planned template is the legacy mapping;
+     * SyncService applies the effective template again after the group changes of the run.
+     *
+     * @param array<string, DirectoryUser> $matched
+     */
+    private function planUpdates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
+    {
+        foreach ($matched as $username => $user) {
+            $account = $state->accounts[$username];
+            $legacy = $settings->mappedTemplate($user->templateValues);
+            $plan->profiles[$username] = ['name' => self::name($user->name, $account->name), 'template' => $legacy];
+            $template = $state->effectiveTemplates[$username] ?? $legacy;
+            if ($template === null) {
+                continue;
+            }
+            if (!isset($state->templateHashes[$template])) {
+                self::conflict($plan, 'Template ' . $template . ' does not exist');
+                continue;
+            }
+            $name = self::name($user->name, $account->name);
+            if ($name !== $account->name || $state->templateHashes[$template] !== $account->attributeHash) {
+                $plan->updates[] = ['username' => (string)$username, 'name' => $name, 'template' => $legacy];
+            }
+        }
+    }
+
+    /** Domain for a new group: the domain of its mail if it is a mailcow domain, else the default group domain */
+    private static function groupDomain(Settings $settings, DirectoryGroup $group, array $domains): ?string
+    {
+        $domain = ($group->mail !== null) ? substr((string)strrchr($group->mail, '@'), 1) : '';
+        if (in_array($domain, $domains, true)) {
+            return $domain;
+        }
+        return in_array($settings->groupDefaultDomain, $domains, true) ? $settings->groupDefaultDomain : null;
+    }
+
+    /** Mailbox name as mailbox_from_template stores it; an empty IdP name keeps the current one */
+    private static function name(string $name, string $current): string
+    {
+        $name = ltrim(rtrim(trim($name), '>'), '<');
+        return ($name === '') ? $current : $name;
+    }
+
+    private static function limit(int $base, int $percent): int
+    {
+        return max(self::GUARD_MIN, (int)ceil($base * $percent / 100));
+    }
+
+    /** @return string[] members of a state group with source idp */
+    private static function idpMembers(array $group): array
+    {
+        return array_map('strval', array_keys(array_filter($group['members'], fn(string $source): bool => $source === 'idp')));
+    }
+
+    private static function sorted(array $usernames): array
+    {
+        $usernames = array_values(array_unique(array_map('strval', $usernames)));
+        sort($usernames);
+        return $usernames;
+    }
+
+    private static function conflict(SyncPlan $plan, string $message): void
+    {
+        if (!in_array($message, $plan->conflicts, true)) {
+            $plan->conflicts[] = $message;
+        }
+    }
+}

data/web/inc/lib/mailcow/Identity/Service/SyncService.php

@@ -0,0 +1,231 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Identity\Service;
+
+use Closure;
+use Mailcow\Core\UserError;
+use Mailcow\Identity\Directory\Directory;
+use Mailcow\Identity\Directory\DirectoryException;
+use Mailcow\Identity\Legacy\LegacyAccounts;
+use Mailcow\Identity\Model\Settings;
+use Mailcow\Identity\Model\State;
+use Mailcow\Identity\Model\SyncPlan;
+use Mailcow\Identity\Repository\IdentityRepository;
+use Mailcow\Identity\Repository\RunRepository;
+use Mailcow\MailboxGroup\Service\IdpSyncService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use Mailcow\Sogo\Service\SogoService;
+use Throwable;
+
+/**
+ * Sync runs of the IdP (cron only): list the directory, plan, apply. A listing error aborts the run before
+ * anything is written; a blocked or preview run writes nothing but the run summary.
+ * The directory is created only when a run is due, so idle cron calls never connect to the IdP.
+ */
+class SyncService
+{
+    /** A run still `running` after this many minutes crashed */
+    private const STALE_MINUTES = 60;
+    /** Errors kept in the run summary */
+    private const DETAILS = 50;
+
+    public function __construct(
+        private Settings $settings,
+        /** @var Closure(): ?Directory */
+        private Closure $directory,
+        private IdentityRepository $repo,
+        private RunRepository $runs,
+        private SyncPlanner $planner,
+        private IdpSyncService $groups,
+        private TemplateService $templates,
+        private LegacyAccounts $legacy,
+        private SogoService $sogo
+    ) {
+    }
+
+    /**
+     * Runs the oldest requested run, else a scheduled run when the sync interval is over.
+     *
+     * @return ?array the finished run (id, mode, status, summary), null if nothing was due
+     */
+    public function runDue(): ?array
+    {
+        $this->runs->failStale(self::STALE_MINUTES);
+        $run = $this->runs->nextPending();
+        if ($run === null) {
+            if (!$this->settings->syncEnabled()) {
+                return null;
+            }
+            $since = $this->runs->secondsSinceScheduled();
+            if ($since !== null && $since < $this->settings->syncInterval * 60) {
+                return null;
+            }
+            $run = ['id' => $this->runs->create($this->settings->provider, 'run'), 'mode' => 'run'];
+        }
+        return $this->execute((int)$run['id'], (string)$run['mode']);
+    }
+
+    /** @return array{id: int, mode: string, status: string, summary: array} */
+    public function execute(int $id, string $mode): array
+    {
+        $this->runs->start($id);
+        try {
+            $directory = $this->settings->canSync() ? ($this->directory)() : null;
+            if ($directory === null) {
+                throw new DirectoryException('The identity provider cannot be synced');
+            }
+            $listing = $directory->listAll($this->settings->groupSync);
+            $plan = $this->planner->plan($this->settings, $listing, $this->state(), $mode === 'force');
+            $summary = $plan->summary();
+            if ($mode === 'preview') {
+                $status = 'ok';
+            } elseif ($plan->blocked) {
+                $status = 'blocked';
+            } else {
+                [$errors, $updated] = $this->apply($plan);
+                $summary['counts']['updates'] = $updated;
+                $summary['counts']['errors'] = count($errors);
+                $summary['errors'] = array_slice($errors, 0, self::DETAILS);
+                $status = 'ok';
+            }
+        } catch (DirectoryException $e) {
+            $status = 'error';
+            $summary = ['error' => $e->getMessage()];
+        } catch (Throwable $e) {
+            $this->runs->finish($id, 'error', ['error' => 'The sync run failed: ' . $e->getMessage(), 'mode' => $mode]);
+            throw $e;
+        }
+        $summary['mode'] = $mode;
+        $this->runs->finish($id, $status, $summary);
+        return ['id' => $id, 'mode' => $mode, 'status' => $status, 'summary' => $summary];
+    }
+
+    private function state(): State
+    {
+        $accounts = $this->repo->accounts($this->settings->provider);
+        return new State($accounts, $this->repo->mailboxes(), $this->repo->domains(),
+            $this->settings->groupSync ? $this->groups->groups() : [],
+            $this->settings->periodicSync ? $this->templates->effective(array_keys($accounts)) : [],
+            $this->repo->templateHashes());
+    }
+
+    /** @return array{0: string[], 1: int} errors of single changes (the rest is still applied), updated mailboxes */
+    private function apply(SyncPlan $plan): array
+    {
+        $provider = $this->settings->provider;
+        $errors = [];
+        foreach ($plan->links as $username => $externalId) {
+            $this->repo->link((string)$username, $provider, $externalId);
+        }
+        foreach ($plan->found as $username) {
+            $this->repo->setMissing($username, $provider, false);
+        }
+        foreach ($plan->missing as $username) {
+            $this->repo->setMissing($username, $provider, true);
+        }
+        foreach ($plan->creates as $create) {
+            $error = $this->legacy->createMailbox($create['email'], $create['name'], $provider, $create['template']);
+            if ($error !== null) {
+                $errors[] = $create['email'] . ': ' . $error;
+                continue;
+            }
+            $this->repo->link($create['email'], $provider, $create['id']);
+        }
+        foreach ($plan->deactivate as $username) {
+            $this->repo->setActive($username, $provider, 2, true);
+        }
+        foreach ($plan->restore as $username) {
+            $this->repo->setActive($username, $provider, 1, false);
+        }
+        foreach ($plan->release as $username) {
+            $this->repo->setIdpDisabled($username, $provider, false);
+        }
+        $this->sogo->refreshStaticView(array_merge($plan->deactivate, $plan->restore));
+        $changed = [];
+        $errors = array_merge($errors, $this->applyGroups($plan, $changed));
+        [$updateErrors, $updated] = $this->applyUpdates($plan, $changed);
+        return [array_merge($errors, $updateErrors), $updated];
+    }
+
+    /**
+     * @param string[] $changed collects the members that joined or left a group
+     * @return string[]
+     */
+    private function applyGroups(SyncPlan $plan, array &$changed): array
+    {
+        $errors = [];
+        foreach ($plan->groupCreates as $create) {
+            try {
+                foreach ($this->groups->createFromIdp($create['ref'], $create['name'], $create['domain'], $create['mail']) as $warning) {
+                    $errors[] = $create['name'] . ': ' . $warning[0];
+                }
+            } catch (UserError $e) {
+                $errors[] = $create['name'] . ': ' . $e->key;
+            }
+        }
+        foreach ($plan->groupRenames as $id => $name) {
+            try {
+                $this->groups->renameFromIdp((int)$id, $name);
+            } catch (UserError $e) {
+                $errors[] = $name . ': ' . $e->key;
+            }
+        }
+        foreach ($plan->memberships as $ref => $usernames) {
+            $group = $this->groups->findByIdpRef((string)$ref);
+            if ($group !== null) {
+                $this->groupResult($group->name, $this->groups->syncGroupMembers((int)$group->id, $usernames), $errors, $changed);
+            }
+        }
+        foreach ($plan->orphanedGroups as $id) {
+            $this->groupResult((string)$id, $this->groups->syncGroupMembers($id, []), $errors, $changed);
+        }
+        return $errors;
+    }
+
+    /**
+     * Name and template updates after the group changes: planned updates and members that joined or left a group
+     * in this run (e.g. the legacy mapping applies again after leaving a template group). The effective template
+     * is resolved and compared again.
+     *
+     * @return array{0: string[], 1: int} errors, updated mailboxes
+     */
+    private function applyUpdates(SyncPlan $plan, array $changed): array
+    {
+        $candidates = array_values(array_unique(array_merge(array_column($plan->updates, 'username'),
+            array_intersect($changed, array_keys($plan->profiles)))));
+        if (empty($candidates)) {
+            return [[], 0];
+        }
+        $errors = [];
+        $updated = 0;
+        $accounts = $this->repo->accounts($this->settings->provider);
+        $hashes = $this->repo->templateHashes();
+        $effective = $this->templates->effective($candidates);
+        foreach ($candidates as $username) {
+            $profile = $plan->profiles[$username] ?? null;
+            $account = $accounts[$username] ?? null;
+            $template = $effective[$username] ?? $profile['template'] ?? null;
+            if ($profile === null || $account === null || $template === null
+                || ($profile['name'] === $account->name && ($hashes[$template] ?? null) === $account->attributeHash)) {
+                continue;
+            }
+            $error = $this->legacy->applyTemplate($username, $profile['name'], $template);
+            if ($error !== null) {
+                $errors[] = $username . ': ' . $error;
+            } else {
+                $updated++;
+            }
+        }
+        return [$errors, $updated];
+    }
+
+    private function groupResult(string $group, array $result, array &$errors, array &$changed): void
+    {
+        foreach ($result['warnings'] as $warning) {
+            $errors[] = $group . ': ' . $warning[0];
+        }
+        array_push($changed, ...$result['added'], ...$result['removed']);
+    }
+}

data/web/inc/lib/mailcow/Imapsync/Service/SourceService.php

@@ -66,7 +66,7 @@ class SourceService
 
     public function canEdit(Source $source): bool
     {
-        if ($this->context->accessAll || $this->context->isAdmin() || $source->createdBy === $this->context->username) {
+        if ($this->context->hasAdminRights() || $source->createdBy === $this->context->username) {
             return true;
         }
         // A domain admin may manage sources created by mailboxes of his domains

data/web/inc/lib/mailcow/MailboxGroup/Controller/MailboxGroupApiController.php

@@ -0,0 +1,173 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Controller;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use Mailcow\MailboxGroup\Model\Group;
+use Mailcow\MailboxGroup\Service\AccountHooks;
+use Mailcow\MailboxGroup\Service\GroupService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php and edit.php (routes `mailbox-group`), plus the hook adapters the legacy
+ * code calls outside a transaction (a database error becomes a `mysql_error` message).
+ * Messages are logged as ['mailbox_group', action, 'group', data].
+ */
+class MailboxGroupApiController
+{
+    public function __construct(private GroupService $groups, private AccountHooks $hooks, private TemplateService $templates,
+        private ApiResponse $response)
+    {
+    }
+
+    /** POST add/mailbox-group */
+    public function addGroup(array $attr): bool
+    {
+        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
+            $group = $this->groups->create($attr);
+            $this->warnings($group->warnings, $log);
+            return $this->response->success(['group_added', htmlspecialchars($group->name)], $log);
+        });
+    }
+
+    /** POST edit/mailbox-group: one message per id */
+    public function editGroups(?array $items, array $attr): bool
+    {
+        $data = array_merge(['id' => $items], $attr);
+        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
+            $this->groups->requireEditor();
+            return $this->batch((array)($data['id'] ?? []), $log, function (int $id) use ($attr, $log): array {
+                $group = $this->groups->update($id, $attr);
+                $this->warnings($group->warnings, $log);
+                return ['group_modified', htmlspecialchars($group->name)];
+            });
+        });
+    }
+
+    /** POST delete/mailbox-group: one message per id */
+    public function deleteGroups(array $items): bool
+    {
+        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
+            $this->groups->requireEditor();
+            return $this->batch($items, $log, function (int $id) use ($log): array {
+                $group = $this->groups->delete($id);
+                $this->warnings($group->warnings, $log);
+                return ['group_removed', htmlspecialchars($group->name)];
+            });
+        });
+    }
+
+    /** GET mailbox-group/all[/<domain,domain>] */
+    public function getGroups(?string $domains = null): array
+    {
+        $list = array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
+        return array_values(array_filter(array_map(fn(int $id): array => $this->getGroup($id), $this->groups->visibleIds($list))));
+    }
+
+    /** GET mailbox-group/<id>, also edit.php */
+    public function getGroup(mixed $id): array
+    {
+        $id = $this->toId($id);
+        $group = ($id === null) ? null : $this->groups->get($id);
+        return ($group === null) ? [] : $this->groups->toArray($group);
+    }
+
+    /** GET mailbox-group/member/<username>: groups of a mailbox (mailbox edit page, user page) */
+    public function getMemberships(string $username): array
+    {
+        return $this->groups->membershipsOf($username);
+    }
+
+    public function mailboxDeleted(string $username): void
+    {
+        $this->hook('mailbox_deleted', $username, fn() => $this->hooks->onMailboxDeleted($username));
+    }
+
+    public function domainDeleted(string $domain): void
+    {
+        $this->hook('domain_deleted', $domain, fn() => $this->hooks->onDomainDeleted($domain));
+    }
+
+    public function addressDeleted(string $address): void
+    {
+        $this->hook('address_deleted', $address, fn() => $this->hooks->onAddressDeleted($address));
+    }
+
+    public function addressRenamed(string $old, string $new): void
+    {
+        $this->hook('address_renamed', [$old, $new], fn() => $this->hooks->onAddressRenamed($old, $new));
+    }
+
+    /** A mailbox template was edited: apply it where it is the effective group template */
+    public function templateChanged(int $templateId): void
+    {
+        $this->hook('template_changed', $templateId, function () use ($templateId): void {
+            $this->warnings($this->templates->onTemplateChanged($templateId), ['mailbox_group', 'edit', 'template', $templateId]);
+        });
+    }
+
+    private function warnings(array $warnings, array $log): void
+    {
+        foreach ($warnings as $msg) {
+            $this->response->warning($msg, $log);
+        }
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     */
+    private function handle(string $action, mixed $data, callable $work): bool
+    {
+        $log = ['mailbox_group', $action, 'group', $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+
+    private function hook(string $action, mixed $data, callable $work): void
+    {
+        try {
+            $work();
+        } catch (PDOException $e) {
+            $this->response->danger(['mysql_error', $e->getMessage()], ['mailbox_group', $action, 'hook', $data]);
+        }
+        $this->response->flush();
+    }
+
+    /** One message per id: the success message returned by $work(id), or the UserError */
+    private function batch(array $ids, array $log, callable $work): bool
+    {
+        foreach ($ids as $id) {
+            try {
+                $this->response->success($work($this->requireId($id)), $log);
+            } catch (UserError $e) {
+                $this->response->error($e, $log);
+            }
+        }
+        return true;
+    }
+
+    /** Integer id from request input ("12", 12); null for anything else */
+    private function toId(mixed $id): ?int
+    {
+        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
+    }
+
+    /** @throws AccessDenied for ids that cannot exist */
+    private function requireId(mixed $id): int
+    {
+        return $this->toId($id) ?? throw new AccessDenied();
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Legacy/LegacyMailbox.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Legacy;
+
+/**
+ * Bridge to the procedural mailbox code. The global calls sit in protected methods so tests can replace them.
+ */
+class LegacyMailbox
+{
+    /**
+     * Applies a mailbox template like the IdP code does: as system caller, with the current name, so only the
+     * template attributes change.
+     *
+     * @return ?string null on success, else the message key of the legacy error
+     */
+    public function applyTemplate(string $username, string $name, string $template): ?string
+    {
+        return $this->asSystem(fn(): mixed => $this->callTemplate($username, $name, $template));
+    }
+
+    /**
+     * Runs a legacy call as system caller (access_all_exception); the caller's session messages are kept.
+     *
+     * @return ?string null unless the call returned false, else the message key of the legacy error
+     */
+    protected function asSystem(callable $call): ?string
+    {
+        $hasReturn = array_key_exists('return', $_SESSION ?? []);
+        $return = $_SESSION['return'] ?? null;
+        $accessAll = $_SESSION['access_all_exception'] ?? null;
+        $_SESSION['return'] = [];
+        $_SESSION['access_all_exception'] = '1';
+        try {
+            if ($call() !== false) {
+                return null;
+            }
+            foreach ($_SESSION['return'] as $message) {
+                if (($message['type'] ?? '') === 'danger') {
+                    return (string)(is_array($message['msg']) ? $message['msg'][0] : $message['msg']);
+                }
+            }
+            return 'generic_server_error';
+        } finally {
+            if ($hasReturn) {
+                $_SESSION['return'] = $return;
+            } else {
+                unset($_SESSION['return']);
+            }
+            $_SESSION['access_all_exception'] = $accessAll ?? '0';
+        }
+    }
+
+    protected function callTemplate(string $username, string $name, string $template): mixed
+    {
+        return mailbox('edit', 'mailbox_from_template', ['username' => $username, 'name' => $name, 'template' => $template]);
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/MailboxGroupModule.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup;
+
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\Dovecot\DovecotModule;
+use Mailcow\MailboxGroup\Controller\MailboxGroupApiController;
+use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+use Mailcow\MailboxGroup\Service\AccountHooks;
+use Mailcow\MailboxGroup\Service\ChangeEffects;
+use Mailcow\MailboxGroup\Service\GroupService;
+use Mailcow\MailboxGroup\Service\IdpSyncService;
+use Mailcow\MailboxGroup\Service\SharingService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use Mailcow\Sogo\SogoModule;
+use PDO;
+
+/**
+ * Composition root of the mailbox group module.
+ *
+ *   process_add_return(MailboxGroupModule::fromGlobals()->api->addGroup($attr));
+ *   MailboxGroupModule::fromGlobals(Context::system())->templates->templateForLogin($username, $legacyTemplate);
+ *   MailboxGroupModule::create($pdo, Context::system())->sharing->reconcile();  // cron
+ */
+class MailboxGroupModule
+{
+    public function __construct(
+        public readonly GroupService $groups,
+        public readonly TemplateService $templates,
+        public readonly AccountHooks $hooks,
+        public readonly IdpSyncService $idp,
+        public readonly SharingService $sharing,
+        public readonly MailboxGroupApiController $api
+    ) {
+    }
+
+    public static function create(PDO $pdo, Context $context): self
+    {
+        $repo = new GroupRepository($pdo);
+        $lookup = new LookupRepository($pdo);
+        $dovecot = DovecotModule::create()->dovecot;
+        $sogo = SogoModule::create()->sogo;
+        $templates = new TemplateService($lookup, new LegacyMailbox());
+        $effects = new ChangeEffects($templates, $sogo, $repo, $dovecot);
+        $sharing = new SharingService($repo, $dovecot, $sogo, $effects);
+        $groups = new GroupService($repo, $lookup, $templates, $effects, $sharing, $context);
+        $hooks = new AccountHooks($repo, $effects);
+        return new self($groups, $templates, $hooks, new IdpSyncService($repo, $lookup, $effects), $sharing,
+            new MailboxGroupApiController($groups, $hooks, $templates, new ApiResponse()));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION; $context overrides the session caller */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Model/Group.php

@@ -0,0 +1,108 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Model;
+
+/**
+ * A mailbox group (table `mailbox_group`) with its members and send-as grants.
+ *
+ * The group tables are the source of truth. GroupRepository::rebuild() derives the rows other services read:
+ * the alias row of the distribution address and the sender_acl rows of the grants, both tagged with `group_id`.
+ * Not to be confused with SOGo resources of kind 'group' (`mailbox.kind`).
+ */
+class Group
+{
+    public const SOURCE_MANUAL = 'manual';
+    public const SOURCE_IDP = 'idp';
+
+    public ?int $id = null;
+    public string $domain = '';
+    public string $name = '';
+    public string $description = '';
+    /** Distribution address, null = group only used for grants and templates */
+    public ?string $address = null;
+    /** Members may send as the address */
+    public int $senderAllowed = 0;
+    public int $sogoVisible = 0;
+    /** Only senders of the domain may write to the address */
+    public int $internal = 0;
+    public ?int $templateId = null;
+    /** Template name, read only */
+    public ?string $template = null;
+    public int $priority = 0;
+    /** IdP group id, set = IdP memberships are synced */
+    public ?string $idpRef = null;
+    public int $active = 1;
+    public ?string $created = null;
+    public ?string $modified = null;
+    /** @var array<string, string> username => source */
+    public array $members = [];
+    /** @var string[] addresses or @domain */
+    public array $sendAs = [];
+    /** @var array[] {owner, rights, subscribe, state, last_error}: members may open these mailboxes (ACL group=mcg_<id>) */
+    public array $imapGrants = [];
+    /** @var array[] {owner, type, folder, access, subscribe, state, last_error}: SOGo folders shared with the members */
+    public array $sogoGrants = [];
+    /** @var array[] warnings of the last write (msg values), not stored */
+    public array $warnings = [];
+
+    /** Builds a group from a `mailbox_group` row (`template_name` optional) */
+    public static function fromRow(array $row): self
+    {
+        $group = new self();
+        $group->id = isset($row['id']) ? (int)$row['id'] : null;
+        $group->domain = (string)($row['domain'] ?? '');
+        $group->name = (string)($row['name'] ?? '');
+        $group->description = (string)($row['description'] ?? '');
+        $group->address = isset($row['address']) ? (string)$row['address'] : null;
+        $group->senderAllowed = (int)($row['sender_allowed'] ?? 0);
+        $group->sogoVisible = (int)($row['sogo_visible'] ?? 0);
+        $group->internal = (int)($row['internal'] ?? 0);
+        $group->templateId = isset($row['template_id']) ? (int)$row['template_id'] : null;
+        $group->template = isset($row['template_name']) ? (string)$row['template_name'] : null;
+        $group->priority = (int)($row['priority'] ?? 0);
+        $group->idpRef = isset($row['idp_ref']) ? (string)$row['idp_ref'] : null;
+        $group->active = (int)($row['active'] ?? 1);
+        $group->created = isset($row['created']) ? (string)$row['created'] : null;
+        $group->modified = isset($row['modified']) ? (string)$row['modified'] : null;
+        return $group;
+    }
+
+    /** @return string[] */
+    public function usernames(): array
+    {
+        return array_keys($this->members);
+    }
+
+    /** API representation (GET mailbox-group) */
+    public function toArray(bool $canEdit): array
+    {
+        $members = [];
+        foreach ($this->members as $username => $source) {
+            $members[] = ['username' => (string)$username, 'source' => $source];
+        }
+        return [
+            'id' => $this->id,
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'description' => $this->description,
+            'address' => $this->address,
+            'sender_allowed' => $this->senderAllowed,
+            'sogo_visible' => $this->sogoVisible,
+            'internal' => $this->internal,
+            'template_id' => $this->templateId,
+            'template' => $this->template,
+            'priority' => $this->priority,
+            'idp_ref' => $this->idpRef,
+            'members' => $members,
+            'send_as' => $this->sendAs,
+            'imap_grants' => $this->imapGrants,
+            'sogo_grants' => $this->sogoGrants,
+            'active' => $this->active,
+            'created' => $this->created,
+            'modified' => $this->modified,
+            'can_edit' => $canEdit,
+        ];
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Repository/GroupRepository.php

@@ -0,0 +1,649 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Repository;
+
+use Mailcow\Core\Context;
+use Mailcow\Core\Repository;
+use Mailcow\MailboxGroup\Model\Group;
+use PDO;
+
+/**
+ * SQL for `mailbox_group`, its members and send-as grants, and the rows derived from them (rebuild()).
+ * Every write rebuilds the derived rows in the same transaction, so readers never see a half-applied group.
+ */
+class GroupRepository extends Repository
+{
+    /** SOGo roles per folder type and access of a SOGo grant */
+    public const SOGO_ROLES = [
+        'Calendar' => [
+            'read' => ['PublicViewer', 'ConfidentialViewer', 'PrivateDAndTViewer'],
+            'write' => ['PublicModifier', 'ConfidentialModifier', 'PrivateDAndTViewer', 'ObjectCreator', 'ObjectEraser'],
+        ],
+        'Contacts' => [
+            'read' => ['ObjectViewer'],
+            'write' => ['ObjectViewer', 'ObjectCreator', 'ObjectEditor', 'ObjectEraser'],
+        ],
+    ];
+
+    /** Grant table, folder and extra "access gone" condition of the subscriptions per kind */
+    private const SUBSCRIPTION_GRANTS = [
+        'imap' => ['table' => 'mailbox_group_imap_grant', 'folder' => "''", 'gone' => " OR g.`state` = 'removing'", 'valid' => " AND g2.`state` != 'removing'"],
+        'sogo' => ['table' => 'mailbox_group_sogo_grant', 'folder' => "CONCAT(g.`folder_type`, '/', g.`folder`)", 'gone' => '', 'valid' => ''],
+    ];
+
+    /** Side effects of the writes since the last takeEffects(): members whose acl_groups changed, sogo_acl changed */
+    private array $effects = ['members' => [], 'sogo_acl' => false];
+    /** Members removed by writeMembers(), rebuild() decides whether Dovecot needs to know */
+    private array $removedMembers = [];
+
+    public function find(int $id): ?Group
+    {
+        return $this->fetch('1=1', ['id' => $id]);
+    }
+
+    /** Like find(), but only when the group is visible to the caller */
+    public function findVisible(int $id, Context $context): ?Group
+    {
+        [$visible, $params] = $this->visibility($context);
+        return $this->fetch($visible, $params + ['id' => $id]);
+    }
+
+    public function findByIdpRef(string $ref): ?Group
+    {
+        $id = $this->query("SELECT `id` FROM `mailbox_group` WHERE `idp_ref` = :ref", ['ref' => $ref])->fetchColumn();
+        return ($id === false) ? null : $this->find((int)$id);
+    }
+
+    /**
+     * @param string[] $domains limit to these domains, [] = all visible
+     * @return int[] ids of the groups visible to the caller, ordered by domain and name
+     */
+    public function visibleIds(Context $context, array $domains = []): array
+    {
+        [$visible, $params] = $this->visibility($context);
+        $sql = "SELECT g.`id` FROM `mailbox_group` g WHERE $visible";
+        if (!empty($domains)) {
+            $names = [];
+            foreach (array_values($domains) as $i => $domain) {
+                $names[] = ':dom' . $i;
+                $params['dom' . $i] = $domain;
+            }
+            $sql .= " AND g.`domain` IN (" . implode(', ', $names) . ")";
+        }
+        return array_map('intval', $this->query($sql . " ORDER BY g.`domain`, g.`name`", $params)->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `mailbox_group` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
+            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
+    }
+
+    /** @return string[] usernames whose SOGo identities changed */
+    public function insert(Group $group): array
+    {
+        return $this->transaction(function () use ($group): array {
+            $group->id = $this->insertRow('mailbox_group', $this->values($group));
+            $this->writeMembers($group);
+            $this->writeSendAs($group);
+            $this->writeGrants($group);
+            return $this->rebuild((int)$group->id);
+        });
+    }
+
+    /** @return string[] usernames whose SOGo identities changed */
+    public function update(Group $group): array
+    {
+        return $this->transaction(function () use ($group): array {
+            $this->updateRow('mailbox_group', (int)$group->id, $this->values($group));
+            $this->writeMembers($group);
+            $this->writeSendAs($group);
+            $this->writeGrants($group);
+            return $this->rebuild((int)$group->id);
+        });
+    }
+
+    /**
+     * Deletes the group; members, grants and the derived alias/sender_acl rows go via FK cascade.
+     *
+     * @return string[] usernames whose SOGo identities changed
+     */
+    public function delete(int $id): array
+    {
+        return $this->transaction(function () use ($id): array {
+            $before = $this->aliasState($id);
+            $this->query("DELETE FROM `mailbox_group` WHERE `id` = :id", ['id' => $id]);
+            $this->cleanupSogoAcl();
+            return $before['members'];
+        });
+    }
+
+    /**
+     * Rewrites the rows derived from the group: the alias row of its address (goto = members, inactive while the
+     * group is inactive or empty, so the address bounces but stays reserved) and one sender_acl row per member
+     * and send-as grant, and the sogo_acl rows of its SOGo grants. Idempotent, joins the caller's transaction.
+     * $sogo = false keeps the sogo_acl rows (mailbox rename: sogo-tool rename-user updates them afterwards).
+     *
+     * @return string[] usernames whose SOGo identities changed
+     */
+    public function rebuild(int $id, bool $sogo = true): array
+    {
+        return $this->transaction(function () use ($id, $sogo): array {
+            $group = $this->query("SELECT * FROM `mailbox_group` WHERE `id` = :id FOR UPDATE", ['id' => $id])->fetch(PDO::FETCH_ASSOC);
+            if (!$group) {
+                return [];
+            }
+            $before = $this->aliasState($id);
+            $members = $this->query("SELECT `username` FROM `mailbox_group_member` WHERE `group_id` = :id ORDER BY `username`",
+                ['id' => $id])->fetchAll(PDO::FETCH_COLUMN);
+            $visible = false;
+            if ($group['address'] === null) {
+                $this->query("DELETE FROM `alias` WHERE `group_id` = :id", ['id' => $id]);
+            } else {
+                $values = [
+                    'address' => $group['address'],
+                    'goto' => empty($members) ? 'null@localhost' : implode(',', $members),
+                    'domain' => $group['domain'],
+                    'public_comment' => $group['description'],
+                    'sender_allowed' => (int)$group['sender_allowed'],
+                    // SOGo identities only make sense when the members may send as the address
+                    'sogo_visible' => ($group['sogo_visible'] == 1 && $group['sender_allowed'] == 1) ? 1 : 0,
+                    'internal' => (int)$group['internal'],
+                    'active' => ($group['active'] == 1 && !empty($members)) ? 1 : 0,
+                ];
+                // Never insert by address: an existing foreign row must not be taken over
+                $aliasId = $this->query("SELECT `id` FROM `alias` WHERE `group_id` = :id", ['id' => $id])->fetchColumn();
+                if ($aliasId === false) {
+                    $this->insertRow('alias', $values + ['group_id' => $id]);
+                } else {
+                    $this->updateRow('alias', (int)$aliasId, $values);
+                }
+                $visible = ($values['active'] === 1 && $values['sogo_visible'] === 1);
+            }
+            $this->query("DELETE FROM `sender_acl` WHERE `group_id` = :id", ['id' => $id]);
+            if ($group['active'] == 1) {
+                $this->query("INSERT INTO `sender_acl` (`logged_in_as`, `send_as`, `external`, `group_id`)
+                  SELECT m.`username`, s.`address`, 0, m.`group_id` FROM `mailbox_group_member` m
+                    JOIN `mailbox_group_send_as` s ON s.`group_id` = m.`group_id`
+                  WHERE m.`group_id` = :id AND m.`username` != s.`address`", ['id' => $id]);
+            }
+            if ($sogo) {
+                $this->rebuildSogoAcl($id, $group['active'] == 1, $members);
+            }
+            // Dovecot caches acl_groups (userdb) of the members, incl. the ones that just left
+            if ($this->query("SELECT 1 FROM `mailbox_group_imap_grant` WHERE `group_id` = :id LIMIT 1", ['id' => $id])->fetchColumn()) {
+                $this->effects['members'] = array_merge($this->effects['members'], $members, $this->removedMembers);
+            }
+            $this->removedMembers = [];
+            $after = $visible ? $members : [];
+            // Same visible address: only joined and left members are affected
+            if ($before['members'] !== [] && $visible && $before['address'] === $group['address']) {
+                return array_values(array_merge(array_diff($before['members'], $after), array_diff($after, $before['members'])));
+            }
+            return array_values(array_unique(array_merge($before['members'], $after)));
+        });
+    }
+
+    /**
+     * Every group as the IdP sync plans with it: {id, domain, name, idp_ref, template, priority, active,
+     * members: username => source}
+     *
+     * @return array[]
+     */
+    public function syncState(): array
+    {
+        $members = [];
+        foreach ($this->query("SELECT `group_id`, `username`, `source` FROM `mailbox_group_member`")->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $members[(int)$row['group_id']][(string)$row['username']] = (string)$row['source'];
+        }
+        $groups = [];
+        foreach ($this->query("SELECT g.`id`, g.`domain`, g.`name`, g.`idp_ref`, t.`template`, g.`priority`, g.`active`
+          FROM `mailbox_group` g LEFT JOIN `templates` t ON t.`id` = g.`template_id` ORDER BY g.`id`")->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $groups[] = [
+                'id' => (int)$row['id'],
+                'domain' => (string)$row['domain'],
+                'name' => (string)$row['name'],
+                'idp_ref' => isset($row['idp_ref']) ? (string)$row['idp_ref'] : null,
+                'template' => isset($row['template']) ? (string)$row['template'] : null,
+                'priority' => (int)$row['priority'],
+                'active' => (int)$row['active'],
+                'members' => $members[(int)$row['id']] ?? [],
+            ];
+        }
+        return $groups;
+    }
+
+    /** @return int[] groups with this member, a send-as grant on this address or a SOGo grant on this mailbox */
+    public function groupIdsOfAccount(string $address): array
+    {
+        return array_map('intval', $this->query("SELECT `group_id` FROM `mailbox_group_member` WHERE `username` = :a1
+          UNION SELECT `group_id` FROM `mailbox_group_send_as` WHERE `address` = :a2
+          UNION SELECT `group_id` FROM `mailbox_group_sogo_grant` WHERE `owner` = :a3",
+            ['a1' => $address, 'a2' => $address, 'a3' => $address])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /** @return int[] groups with send-as grants on addresses of the domain (incl. @domain) */
+    public function groupIdsWithSendAsIn(string $domain): array
+    {
+        return array_map('intval', $this->query("SELECT DISTINCT `group_id` FROM `mailbox_group_send_as`
+          WHERE SUBSTRING_INDEX(`address`, '@', -1) = :domain", ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /**
+     * A mailbox is deleted: its memberships and the grants on its address go, the groups are rebuilt.
+     *
+     * @param int[] $ids groups of the account (groupIdsOfAccount())
+     * @return string[] usernames whose SOGo identities changed
+     */
+    public function removeAccount(string $username, array $ids): array
+    {
+        return $this->transaction(function () use ($username, $ids): array {
+            $this->query("DELETE FROM `mailbox_group_member` WHERE `username` = :username", ['username' => $username]);
+            $this->query("DELETE FROM `mailbox_group_send_as` WHERE `address` = :username", ['username' => $username]);
+            return $this->rebuildAll($ids);
+        });
+    }
+
+    /** An alias was deleted: grants on its address go */
+    public function removeSendAs(string $address, array $ids): array
+    {
+        return $this->transaction(function () use ($address, $ids): array {
+            $this->query("DELETE FROM `mailbox_group_send_as` WHERE `address` = :address", ['address' => $address]);
+            return $this->rebuildAll($ids);
+        });
+    }
+
+    /** A domain or alias domain is deleted: grants on its addresses go */
+    public function removeSendAsIn(string $domain, array $ids): array
+    {
+        return $this->transaction(function () use ($domain, $ids): array {
+            $this->query("DELETE FROM `mailbox_group_send_as` WHERE SUBSTRING_INDEX(`address`, '@', -1) = :domain", ['domain' => $domain]);
+            return $this->rebuildAll($ids);
+        });
+    }
+
+    /**
+     * Account or alias address renamed: members and grants follow. Explicit updates, because the mailbox rename
+     * runs with FOREIGN_KEY_CHECKS = 0; joins the caller's transaction.
+     */
+    public function renameAccount(string $old, string $new, array $ids): array
+    {
+        return $this->transaction(function () use ($old, $new, $ids): array {
+            $this->query("UPDATE `mailbox_group_member` SET `username` = :new WHERE `username` = :old", ['new' => $new, 'old' => $old]);
+            $this->query("UPDATE `mailbox_group_send_as` SET `address` = :new WHERE `address` = :old", ['new' => $new, 'old' => $old]);
+            return $this->rebuildAll($ids, false);
+        });
+    }
+
+    /** @return int[] groups with SOGo grants */
+    public function groupIdsWithSogoGrants(): array
+    {
+        return array_map('intval', $this->query("SELECT DISTINCT `group_id` FROM `mailbox_group_sogo_grant`")->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /**
+     * IMAP grants Dovecot has to apply: pending, failed and removing ones, with $all also applied ones (new
+     * top-level folders); of one group or all.
+     *
+     * @return array[] rows of mailbox_group_imap_grant
+     */
+    public function imapGrantsToApply(?int $groupId, bool $all): array
+    {
+        $where = $all ? '1=1' : "`state` != 'applied'";
+        $params = [];
+        if ($groupId !== null) {
+            $where .= " AND `group_id` = :id";
+            $params['id'] = $groupId;
+        }
+        return $this->query("SELECT * FROM `mailbox_group_imap_grant` WHERE $where ORDER BY `id`", $params)->fetchAll(PDO::FETCH_ASSOC);
+    }
+
+    public function setImapState(int $grantId, string $state, ?string $error): void
+    {
+        $this->query("UPDATE `mailbox_group_imap_grant` SET `state` = :state, `last_error` = :error WHERE `id` = :id",
+            ['state' => $state, 'error' => $error, 'id' => $grantId]);
+    }
+
+    public function deleteImapGrant(int $grantId): void
+    {
+        $this->query("DELETE FROM `mailbox_group_imap_grant` WHERE `id` = :id", ['id' => $grantId]);
+    }
+
+    /**
+     * Deletes the sogo_acl rows of grants that are gone, e.g. by FK cascades of groups, owners and domains.
+     * Joins the caller's transaction.
+     */
+    public function cleanupSogoAcl(): void
+    {
+        $this->transaction(function (): void {
+            $orphans = $this->query("SELECT l.`acl_id` FROM `mailbox_group_sogo_acl` l
+                LEFT JOIN `mailbox_group_sogo_grant` g ON g.`id` = l.`grant_id`
+              WHERE g.`id` IS NULL")->fetchAll(PDO::FETCH_COLUMN);
+            foreach ($orphans as $aclId) {
+                $this->deleteSogoAcl((int)$aclId);
+            }
+        });
+    }
+
+    /**
+     * Members to subscribe to grants with `subscribe` (kind imap or sogo): applied grants of active groups, members
+     * without a subscription by mailcow yet (a member that removes it keeps it removed).
+     *
+     * @return array[] {grant_id, owner, folder, usernames}
+     */
+    public function missingSubscriptions(string $kind): array
+    {
+        $config = self::SUBSCRIPTION_GRANTS[$kind];
+        $grants = [];
+        foreach ($this->query("SELECT g.`id`, g.`owner`, {$config['folder']} AS `folder`, m.`username`
+            FROM `{$config['table']}` g
+              JOIN `mailbox_group` mg ON mg.`id` = g.`group_id` AND mg.`active` = 1
+              JOIN `mailbox_group_member` m ON m.`group_id` = g.`group_id` AND m.`username` != g.`owner`
+              LEFT JOIN `mailbox_group_subscription` s ON s.`kind` = :kind AND s.`grant_id` = g.`id` AND s.`username` = m.`username`
+            WHERE g.`subscribe` = 1 AND g.`state` = 'applied' AND s.`id` IS NULL
+            ORDER BY g.`id`, m.`username`", ['kind' => $kind])->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $grants[$row['id']] ??= ['grant_id' => (int)$row['id'], 'owner' => (string)$row['owner'], 'folder' => (string)$row['folder'], 'usernames' => []];
+            $grants[$row['id']]['usernames'][] = (string)$row['username'];
+        }
+        return array_values($grants);
+    }
+
+    /**
+     * Subscriptions by mailcow whose member lost the access: grant, membership or group gone or inactive.
+     * `covered`: another group still grants the member the same folder, so the subscription stays.
+     *
+     * @return array[] rows of mailbox_group_subscription with member_exists, owner_exists, covered
+     */
+    public function staleSubscriptions(string $kind): array
+    {
+        $config = self::SUBSCRIPTION_GRANTS[$kind];
+        return $this->query("SELECT s.`id`, s.`username`, s.`owner`, s.`folder`,
+            EXISTS(SELECT 1 FROM `mailbox` WHERE `username` = s.`username`) AS `member_exists`,
+            EXISTS(SELECT 1 FROM `mailbox` WHERE `username` = s.`owner`) AS `owner_exists`,
+            EXISTS(SELECT 1 FROM `mailbox_group_subscription` s2
+                JOIN `{$config['table']}` g2 ON g2.`id` = s2.`grant_id`{$config['valid']}
+                JOIN `mailbox_group` mg2 ON mg2.`id` = g2.`group_id` AND mg2.`active` = 1
+                JOIN `mailbox_group_member` m2 ON m2.`group_id` = g2.`group_id` AND m2.`username` = s2.`username`
+              WHERE s2.`kind` = s.`kind` AND s2.`id` != s.`id` AND s2.`username` = s.`username` AND s2.`owner` = s.`owner`
+                AND s2.`folder` = s.`folder`) AS `covered`
+          FROM `mailbox_group_subscription` s
+            LEFT JOIN `{$config['table']}` g ON g.`id` = s.`grant_id`
+            LEFT JOIN `mailbox_group` mg ON mg.`id` = g.`group_id` AND mg.`active` = 1
+            LEFT JOIN `mailbox_group_member` m ON m.`group_id` = g.`group_id` AND m.`username` = s.`username`
+          WHERE s.`kind` = :kind AND (g.`id` IS NULL OR mg.`id` IS NULL OR m.`username` IS NULL{$config['gone']})
+          ORDER BY s.`id`", ['kind' => $kind])->fetchAll(PDO::FETCH_ASSOC);
+    }
+
+    public function addSubscriptions(string $kind, int $grantId, string $owner, string $folder, array $usernames): void
+    {
+        foreach ($usernames as $username) {
+            $this->query("INSERT IGNORE INTO `mailbox_group_subscription` (`kind`, `grant_id`, `username`, `owner`, `folder`)
+              VALUES (:kind, :grant, :username, :owner, :folder)",
+                ['kind' => $kind, 'grant' => $grantId, 'username' => $username, 'owner' => $owner, 'folder' => $folder]);
+        }
+    }
+
+    public function deleteSubscription(int $id): void
+    {
+        $this->query("DELETE FROM `mailbox_group_subscription` WHERE `id` = :id", ['id' => $id]);
+    }
+
+    /**
+     * Owners of pending SOGo grants on their personal folder that SOGo knows (active mailboxes), of one group or all.
+     * Their folders do not exist because they never logged in to SOGo, e.g. shared mailboxes.
+     *
+     * @return string[]
+     */
+    public function pendingSogoOwners(?int $groupId): array
+    {
+        $params = [];
+        $where = '';
+        if ($groupId !== null) {
+            $where = " AND g.`group_id` = :id";
+            $params['id'] = $groupId;
+        }
+        return $this->query("SELECT DISTINCT g.`owner` FROM `mailbox_group_sogo_grant` g
+            JOIN `mailbox` mb ON mb.`username` = g.`owner` AND mb.`active` = '1' AND mb.`kind` = ''
+          WHERE g.`state` = 'pending' AND g.`folder` = 'personal'$where ORDER BY g.`owner`", $params)->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /**
+     * Side effects of the writes since the last call, for ChangeEffects after the commit.
+     *
+     * @return array{members: string[], sogo_acl: bool}
+     */
+    public function takeEffects(): array
+    {
+        $effects = $this->effects;
+        $effects['members'] = array_values(array_unique($effects['members']));
+        $this->effects = ['members' => [], 'sogo_acl' => false];
+        return $effects;
+    }
+
+    /** @return string[] */
+    private function rebuildAll(array $ids, bool $sogo = true): array
+    {
+        $users = [];
+        foreach (array_unique($ids) as $id) {
+            $users = array_merge($users, $this->rebuild((int)$id, $sogo));
+        }
+        return array_values(array_unique($users));
+    }
+
+    /**
+     * sogo_acl rows of the group's SOGo grants: one row per member (except the owner) and role while the group is
+     * active. Only rows of the ledger are ever deleted, so shares made in SOGo stay. A grant on a folder that does
+     * not exist yet (the owner never opened SOGo) stays pending.
+     */
+    private function rebuildSogoAcl(int $id, bool $active, array $members): void
+    {
+        $this->cleanupSogoAcl();
+        foreach ($this->query("SELECT * FROM `mailbox_group_sogo_grant` WHERE `group_id` = :id", ['id' => $id])->fetchAll(PDO::FETCH_ASSOC) as $grant) {
+            $object = '/' . $grant['owner'] . '/' . $grant['folder_type'] . '/' . $grant['folder'];
+            $folderId = $this->query("SELECT `c_folder_id` FROM `sogo_folder_info` WHERE `c_path` = :path",
+                ['path' => '/Users' . $object])->fetchColumn();
+            $wanted = [];
+            if ($folderId !== false && $active) {
+                foreach ($members as $username) {
+                    foreach (($username === $grant['owner']) ? [] : self::SOGO_ROLES[$grant['folder_type']][$grant['access']] as $role) {
+                        $wanted[$username . '|' . $role] = [$username, $role];
+                    }
+                }
+            }
+            $rows = $this->query("SELECT l.`acl_id`, l.`c_uid`, l.`c_role`, a.`c_folder_id`, a.`c_object` FROM `mailbox_group_sogo_acl` l
+                LEFT JOIN `sogo_acl` a ON a.`id` = l.`acl_id`
+              WHERE l.`grant_id` = :id", ['id' => $grant['id']])->fetchAll(PDO::FETCH_ASSOC);
+            foreach ($rows as $row) {
+                $key = $row['c_uid'] . '|' . $row['c_role'];
+                if (isset($wanted[$key]) && (string)$row['c_folder_id'] === (string)$folderId && $row['c_object'] === $object) {
+                    unset($wanted[$key]);
+                    continue;
+                }
+                $this->deleteSogoAcl((int)$row['acl_id']);
+            }
+            foreach ($wanted as [$username, $role]) {
+                $aclId = $this->insertRow('sogo_acl', ['c_folder_id' => $folderId, 'c_object' => $object, 'c_uid' => $username, 'c_role' => $role]);
+                $this->query("INSERT INTO `mailbox_group_sogo_acl` (`acl_id`, `grant_id`, `c_uid`, `c_role`) VALUES (:acl, :grant, :uid, :role)",
+                    ['acl' => $aclId, 'grant' => $grant['id'], 'uid' => $username, 'role' => $role]);
+                $this->effects['sogo_acl'] = true;
+            }
+            $state = ($folderId === false) ? 'pending' : 'applied';
+            if ($grant['state'] !== $state) {
+                $this->query("UPDATE `mailbox_group_sogo_grant` SET `state` = :state, `last_error` = :error WHERE `id` = :id",
+                    ['state' => $state, 'error' => ($folderId === false) ? 'folder_missing' : null, 'id' => $grant['id']]);
+            }
+        }
+    }
+
+    private function deleteSogoAcl(int $aclId): void
+    {
+        $this->query("DELETE FROM `sogo_acl` WHERE `id` = :id", ['id' => $aclId]);
+        $this->query("DELETE FROM `mailbox_group_sogo_acl` WHERE `acl_id` = :id", ['id' => $aclId]);
+        $this->effects['sogo_acl'] = true;
+    }
+
+    private function fetch(string $where, array $params): ?Group
+    {
+        $row = $this->query("SELECT g.*, t.`template` AS `template_name` FROM `mailbox_group` g
+          LEFT JOIN `templates` t ON t.`id` = g.`template_id`
+          WHERE g.`id` = :id AND $where", $params)->fetch(PDO::FETCH_ASSOC);
+        if (!$row) {
+            return null;
+        }
+        $group = Group::fromRow($row);
+        $id = ['id' => $group->id];
+        $group->members = $this->query("SELECT `username`, `source` FROM `mailbox_group_member` WHERE `group_id` = :id ORDER BY `username`",
+            $id)->fetchAll(PDO::FETCH_KEY_PAIR);
+        $group->sendAs = $this->query("SELECT `address` FROM `mailbox_group_send_as` WHERE `group_id` = :id ORDER BY `address`",
+            $id)->fetchAll(PDO::FETCH_COLUMN);
+        $group->imapGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
+            $this->query("SELECT `owner`, `rights`, `subscribe`, `state`, `last_error` FROM `mailbox_group_imap_grant`
+              WHERE `group_id` = :id AND `state` != 'removing' ORDER BY `owner`", $id)->fetchAll(PDO::FETCH_ASSOC));
+        $group->sogoGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
+            $this->query("SELECT `owner`, `folder_type` AS `type`, `folder`, `access`, `subscribe`, `state`, `last_error`
+              FROM `mailbox_group_sogo_grant` WHERE `group_id` = :id ORDER BY `owner`, `folder_type`, `folder`", $id)->fetchAll(PDO::FETCH_ASSOC));
+        return $group;
+    }
+
+    /** Members (username => source) as in the model; changed sources are updated, created stays */
+    private function writeMembers(Group $group): void
+    {
+        $id = (int)$group->id;
+        $stored = $this->query("SELECT `username`, `source` FROM `mailbox_group_member` WHERE `group_id` = :id", ['id' => $id])
+            ->fetchAll(PDO::FETCH_KEY_PAIR);
+        foreach (array_diff_key($stored, $group->members) as $username => $source) {
+            $this->query("DELETE FROM `mailbox_group_member` WHERE `group_id` = :id AND `username` = :username",
+                ['id' => $id, 'username' => $username]);
+            $this->removedMembers[] = (string)$username;
+        }
+        foreach ($group->members as $username => $source) {
+            if (!isset($stored[$username])) {
+                $this->query("INSERT INTO `mailbox_group_member` (`group_id`, `username`, `source`) VALUES (:id, :username, :source)",
+                    ['id' => $id, 'username' => $username, 'source' => $source]);
+            } elseif ($stored[$username] !== $source) {
+                $this->query("UPDATE `mailbox_group_member` SET `source` = :source WHERE `group_id` = :id AND `username` = :username",
+                    ['source' => $source, 'id' => $id, 'username' => $username]);
+            }
+        }
+    }
+
+    /**
+     * IMAP grants: new ones are pending, removed ones are set to removing until Dovecot dropped the ACL
+     * (SharingService). SOGo grants are replaced; rebuild() maintains their sogo_acl rows.
+     */
+    private function writeGrants(Group $group): void
+    {
+        $id = (int)$group->id;
+        $stored = [];
+        foreach ($this->query("SELECT `id`, `owner`, `rights`, `subscribe`, `state` FROM `mailbox_group_imap_grant` WHERE `group_id` = :id",
+            ['id' => $id])->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $stored[(string)$row['owner']] = $row;
+        }
+        $wanted = [];
+        foreach ($group->imapGrants as $grant) {
+            $wanted[(string)$grant['owner']] = $grant;
+        }
+        foreach ($stored as $owner => $row) {
+            if (!isset($wanted[$owner]) && $row['state'] !== 'removing') {
+                $this->setImapState((int)$row['id'], 'removing', null);
+            }
+        }
+        foreach ($wanted as $owner => $grant) {
+            $row = $stored[$owner] ?? null;
+            $subscribe = (int)($grant['subscribe'] ?? 0);
+            if ($row === null) {
+                $this->insertRow('mailbox_group_imap_grant', ['group_id' => $id, 'owner' => $owner, 'rights' => $grant['rights'],
+                    'subscribe' => $subscribe, 'state' => 'pending']);
+                continue;
+            }
+            if ($row['rights'] !== $grant['rights'] || $row['state'] === 'removing') {
+                $this->query("UPDATE `mailbox_group_imap_grant` SET `rights` = :rights, `state` = 'pending', `last_error` = NULL WHERE `id` = :id",
+                    ['rights' => $grant['rights'], 'id' => $row['id']]);
+            }
+            if ((int)$row['subscribe'] !== $subscribe) {
+                $this->query("UPDATE `mailbox_group_imap_grant` SET `subscribe` = :subscribe WHERE `id` = :id",
+                    ['subscribe' => $subscribe, 'id' => $row['id']]);
+            }
+        }
+        $stored = [];
+        foreach ($this->query("SELECT `id`, `owner`, `folder_type`, `folder`, `access`, `subscribe` FROM `mailbox_group_sogo_grant` WHERE `group_id` = :id",
+            ['id' => $id])->fetchAll(PDO::FETCH_ASSOC) as $row) {
+            $stored[$row['owner'] . '|' . $row['folder_type'] . '|' . $row['folder']] = $row;
+        }
+        $wanted = [];
+        foreach ($group->sogoGrants as $grant) {
+            $wanted[$grant['owner'] . '|' . $grant['type'] . '|' . $grant['folder']] = $grant;
+        }
+        foreach (array_diff_key($stored, $wanted) as $row) {
+            $this->query("DELETE FROM `mailbox_group_sogo_grant` WHERE `id` = :id", ['id' => $row['id']]);
+        }
+        foreach ($wanted as $key => $grant) {
+            $subscribe = (int)($grant['subscribe'] ?? 0);
+            if (!isset($stored[$key])) {
+                $this->insertRow('mailbox_group_sogo_grant', ['group_id' => $id, 'owner' => $grant['owner'], 'folder_type' => $grant['type'],
+                    'folder' => $grant['folder'], 'access' => $grant['access'], 'subscribe' => $subscribe, 'state' => 'pending']);
+            } elseif ($stored[$key]['access'] !== $grant['access'] || (int)$stored[$key]['subscribe'] !== $subscribe) {
+                $this->query("UPDATE `mailbox_group_sogo_grant` SET `access` = :access, `subscribe` = :subscribe WHERE `id` = :id",
+                    ['access' => $grant['access'], 'subscribe' => $subscribe, 'id' => $stored[$key]['id']]);
+            }
+        }
+    }
+
+    private function writeSendAs(Group $group): void
+    {
+        $id = ['id' => $group->id];
+        $this->query("DELETE FROM `mailbox_group_send_as` WHERE `group_id` = :id", $id);
+        foreach ($group->sendAs as $address) {
+            $this->query("INSERT INTO `mailbox_group_send_as` (`group_id`, `address`) VALUES (:id, :address)", $id + ['address' => $address]);
+        }
+    }
+
+    /**
+     * The group's alias row as SOGo sees it: visible = active and sogo_visible (see the grouped_mail_aliases view).
+     *
+     * @return array{address: ?string, members: string[]} members only when visible
+     */
+    private function aliasState(int $id): array
+    {
+        $row = $this->query("SELECT `address`, `goto`, `active`, `sogo_visible` FROM `alias` WHERE `group_id` = :id",
+            ['id' => $id])->fetch(PDO::FETCH_ASSOC);
+        $visible = $row && $row['active'] == 1 && $row['sogo_visible'] == 1;
+        return ['address' => $row['address'] ?? null, 'members' => $visible ? explode(',', (string)$row['goto']) : []];
+    }
+
+    /**
+     * WHERE fragment over `mailbox_group g` that limits rows to what the caller may see, and its parameters.
+     *
+     * @return array{0: string, 1: array}
+     */
+    private function visibility(Context $context): array
+    {
+        if ($context->hasAdminRights()) {
+            return ['1=1', []];
+        }
+        if ($context->isDomainAdmin()) {
+            return ["g.`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
+                ['vis_da' => $context->username]];
+        }
+        return ['0=1', []];
+    }
+
+    private function values(Group $group): array
+    {
+        return [
+            'domain' => $group->domain,
+            'name' => $group->name,
+            'description' => $group->description,
+            'address' => $group->address,
+            'sender_allowed' => $group->senderAllowed,
+            'sogo_visible' => $group->sogoVisible,
+            'internal' => $group->internal,
+            'template_id' => $group->templateId,
+            'priority' => $group->priority,
+            'idp_ref' => $group->idpRef,
+            'active' => $group->active,
+        ];
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Repository/LookupRepository.php

@@ -0,0 +1,148 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * Read-only queries on tables of the legacy code (domains, mailboxes, aliases, templates) and the group
+ * memberships the template logic needs.
+ */
+class LookupRepository extends Repository
+{
+    /** A primary domain (groups never belong to alias domains) */
+    public function isDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+    }
+
+    public function isDomainOrAliasDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :d1 UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :d2",
+            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
+    }
+
+    /** An alias or mailbox address (mailboxes have an alias row of their own) */
+    public function aliasExists(string $address): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address", ['address' => $address])->fetchColumn();
+    }
+
+    /** Used by an alias other than the group's own row, by a mailbox or resource, or as spam alias */
+    public function addressTaken(string $address, ?int $groupId): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :a1 AND (`group_id` IS NULL OR `group_id` != :id)
+          UNION SELECT 1 FROM `mailbox` WHERE `username` = :a2
+          UNION SELECT 1 FROM `spamalias` WHERE `address` = :a3",
+            ['a1' => $address, 'id' => $groupId ?? 0, 'a2' => $address, 'a3' => $address])->fetchColumn();
+    }
+
+    /** Aliases the domain may still create (same count as mailbox('get', 'domain_details')) */
+    public function aliasesLeft(string $domain): int
+    {
+        $max = $this->query("SELECT `aliases` FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+        $used = $this->query("SELECT COUNT(`address`) FROM `alias`
+          WHERE (`domain` = :d1 OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :d2))
+            AND `address` NOT IN (SELECT `username` FROM `mailbox`)", ['d1' => $domain, 'd2' => $domain])->fetchColumn();
+        return (int)$max - (int)$used;
+    }
+
+    /**
+     * @param string[] $usernames
+     * @return string[] the given usernames that are regular mailboxes (no resources)
+     */
+    public function regularMailboxes(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in('u', $usernames);
+        return $this->query("SELECT `username` FROM `mailbox` WHERE `kind` = '' AND `username` IN ($in)", $params)->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** Name of a mailbox template, null if there is none with this id */
+    public function templateName(int $id): ?string
+    {
+        $name = $this->query("SELECT `template` FROM `templates` WHERE `id` = :id AND `type` = 'mailbox'", ['id' => $id])->fetchColumn();
+        return ($name === false) ? null : (string)$name;
+    }
+
+    /** @return array<string, string> username => full name */
+    public function mailboxNames(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in('u', $usernames);
+        return $this->query("SELECT `username`, `name` FROM `mailbox` WHERE `username` IN ($in)", $params)->fetchAll(PDO::FETCH_KEY_PAIR);
+    }
+
+    /**
+     * Template of the active group with the highest priority (lowest id on ties) per member.
+     *
+     * @return array<string, ?string> username => template name, null without group template
+     */
+    public function effectiveTemplates(array $usernames): array
+    {
+        $result = array_fill_keys(array_map('strval', $usernames), null);
+        if (empty($usernames)) {
+            return $result;
+        }
+        [$in, $params] = self::in('u', $usernames);
+        $rows = $this->query("SELECT m.`username`, t.`template` FROM `mailbox_group_member` m
+          JOIN `mailbox_group` g ON g.`id` = m.`group_id` AND g.`active` = '1'
+          JOIN `templates` t ON t.`id` = g.`template_id` AND t.`type` = 'mailbox'
+          WHERE m.`username` IN ($in)
+          ORDER BY m.`username`, g.`priority` DESC, g.`id`", $params)->fetchAll(PDO::FETCH_ASSOC);
+        foreach ($rows as $row) {
+            $result[$row['username']] ??= $row['template'];
+        }
+        return $result;
+    }
+
+    /** @return string[] members of active groups bound to the template */
+    public function membersWithTemplate(int $templateId): array
+    {
+        return $this->query("SELECT DISTINCT m.`username` FROM `mailbox_group_member` m
+          JOIN `mailbox_group` g ON g.`id` = m.`group_id` AND g.`active` = '1'
+          WHERE g.`template_id` = :id", ['id' => $templateId])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** Groups of a mailbox with its membership source */
+    public function membershipsOf(string $username): array
+    {
+        return $this->query("SELECT g.`id`, g.`domain`, g.`name`, g.`address`, g.`sender_allowed`, g.`active`, g.`priority`,
+            t.`template`, m.`source` FROM `mailbox_group_member` m
+          JOIN `mailbox_group` g ON g.`id` = m.`group_id`
+          LEFT JOIN `templates` t ON t.`id` = g.`template_id`
+          WHERE m.`username` = :username ORDER BY g.`domain`, g.`name`", ['username' => $username])->fetchAll(PDO::FETCH_ASSOC);
+    }
+
+    /** @return string[] addresses a mailbox may send as through its active groups */
+    public function sendAsOf(string $username): array
+    {
+        return $this->query("SELECT `send_as` FROM `sender_acl` WHERE `logged_in_as` = :u1 AND `group_id` IS NOT NULL
+          UNION SELECT g.`address` FROM `mailbox_group_member` m JOIN `mailbox_group` g ON g.`id` = m.`group_id`
+          WHERE m.`username` = :u2 AND g.`active` = '1' AND g.`sender_allowed` = '1' AND g.`address` IS NOT NULL
+          ORDER BY 1", ['u1' => $username, 'u2' => $username])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /**
+     * IN list with one placeholder per value (native prepares allow a placeholder only once).
+     *
+     * @return array{0: string, 1: array}
+     */
+    private static function in(string $prefix, array $values): array
+    {
+        $names = [];
+        $params = [];
+        foreach (array_values($values) as $i => $value) {
+            $names[] = ':' . $prefix . $i;
+            $params[$prefix . $i] = (string)$value;
+        }
+        return [implode(', ', $names), $params];
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/AccountHooks.php

@@ -0,0 +1,70 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+
+/**
+ * Keeps groups consistent with account changes made by the legacy code (functions.mailbox.inc.php).
+ * No permission checks: the legacy caller already checked them.
+ */
+class AccountHooks
+{
+    public function __construct(private GroupRepository $repo, private ChangeEffects $effects)
+    {
+    }
+
+    /** Before a mailbox is deleted: its memberships and the grants on its address go */
+    public function onMailboxDeleted(string $username): void
+    {
+        $ids = $this->repo->groupIdsOfAccount($username);
+        if (!empty($ids)) {
+            // The deleted mailbox itself needs no SOGo refresh
+            $this->effects->apply(array_diff($this->repo->removeAccount($username, $ids), [$username]), []);
+        }
+    }
+
+    /**
+     * Inside the legacy mailbox rename transaction (FOREIGN_KEY_CHECKS = 0): members and grants follow.
+     * Must not catch, so the caller can roll back. The legacy rename refreshes SOGo itself.
+     */
+    public function onMailboxRenamed(string $old, string $new): void
+    {
+        $ids = $this->repo->groupIdsOfAccount($old);
+        if (!empty($ids)) {
+            $this->repo->renameAccount($old, $new, $ids);
+        }
+    }
+
+    /**
+     * After a domain or alias domain was deleted: grants on its addresses go. Its groups went via FK cascade,
+     * the sogo_acl rows of their SOGo grants go here.
+     */
+    public function onDomainDeleted(string $domain): void
+    {
+        $ids = $this->repo->groupIdsWithSendAsIn($domain);
+        $users = empty($ids) ? [] : $this->repo->removeSendAsIn($domain, $ids);
+        $this->repo->cleanupSogoAcl();
+        $this->effects->apply($users, []);
+    }
+
+    /** An alias was deleted: grants on its address go */
+    public function onAddressDeleted(string $address): void
+    {
+        $ids = $this->repo->groupIdsOfAccount($address);
+        if (!empty($ids)) {
+            $this->effects->apply($this->repo->removeSendAs($address, $ids), []);
+        }
+    }
+
+    /** An alias address changed: grants follow */
+    public function onAddressRenamed(string $old, string $new): void
+    {
+        $ids = $this->repo->groupIdsOfAccount($old);
+        if (!empty($ids) && $old !== $new) {
+            $this->effects->apply($this->repo->renameAccount($old, $new, $ids), []);
+        }
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/ChangeEffects.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\Dovecot\Service\DovecotService;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\Sogo\Service\SogoService;
+
+/**
+ * What follows a committed group write: SOGo identities of the affected members, group templates and the caches
+ * of SOGo (ACLs) and Dovecot (acl_groups). Runs after the commit; failures become warnings and never roll the
+ * write back.
+ */
+class ChangeEffects
+{
+    public function __construct(private TemplateService $templates, private SogoService $sogo, private GroupRepository $repo,
+        private DovecotService $dovecot)
+    {
+    }
+
+    /** Effective templates of the mailboxes a write may touch, taken before the write */
+    public function snapshot(array $usernames): array
+    {
+        return $this->templates->effective($usernames);
+    }
+
+    /** @return array[] warnings (msg values) */
+    public function apply(array $sogoUsers, array $snapshot): array
+    {
+        $this->sogo->refreshStaticView($sogoUsers);
+        $effects = $this->repo->takeEffects();
+        if ($effects['sogo_acl']) {
+            $this->sogo->flushCache();
+        }
+        $this->dovecot->flushAuthCache($effects['members']);
+        return $this->templates->applyChanged($snapshot);
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/GroupService.php

@@ -0,0 +1,485 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Mailcow\MailboxGroup\Model\Group;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+
+/**
+ * Use cases and rules for mailbox groups.
+ *
+ * Read: admins all groups (also read-only API keys); domain admins with the `mailbox_groups` ACL the groups of
+ *       their domains; users only their own memberships (membershipsOf()).
+ * Edit: admins; domain admins with the ACL for groups of their domains, as long as every send-as grant of the
+ *       group is inside their domains. Template, priority and IdP link are set by admins only.
+ * Members: regular mailboxes the editor may access. IdP members and members outside the editor's scope are kept
+ *       when the member list is replaced; IdP members can only be removed by the IdP sync.
+ */
+class GroupService
+{
+    /** Postfix expands at most this many recipients per address (virtual_alias_expansion_limit) */
+    private const MAX_RECIPIENTS = 1000;
+
+    public function __construct(private GroupRepository $repo, private LookupRepository $lookup, private TemplateService $templates,
+        private ChangeEffects $effects, private SharingService $sharing, private Context $context)
+    {
+    }
+
+    public function canRead(): bool
+    {
+        return $this->context->hasAdminRights()
+            || ($this->context->isDomainAdmin() && $this->context->hasAcl('mailbox_groups'));
+    }
+
+    /** @throws AccessDenied unless the caller may edit groups at all */
+    public function requireEditor(): void
+    {
+        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
+            throw new AccessDenied();
+        }
+    }
+
+    public function get(int $id): ?Group
+    {
+        return $this->canRead() ? $this->repo->findVisible($id, $this->context) : null;
+    }
+
+    /** @return int[] */
+    public function visibleIds(array $domains = []): array
+    {
+        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
+    }
+
+    public function canEdit(Group $group): bool
+    {
+        if ($this->context->isReadOnlyApi() || !$this->canRead()) {
+            return false;
+        }
+        if ($this->context->hasAdminRights()) {
+            return true;
+        }
+        if (!$this->context->canAccessDomain($group->domain)) {
+            return false;
+        }
+        // Grants on foreign domains or mailboxes make the group read-only for a domain admin
+        foreach ($group->sendAs as $address) {
+            if (!$this->context->canAccessDomain(self::domainOf($address))) {
+                return false;
+            }
+        }
+        foreach (array_merge($group->imapGrants, $group->sogoGrants) as $grant) {
+            if (!$this->context->canAccessMailbox((string)$grant['owner'])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public function toArray(Group $group): array
+    {
+        return $group->toArray($this->canEdit($group));
+    }
+
+    /** @throws UserError */
+    public function create(array $input): Group
+    {
+        $this->requireEditor();
+        $group = new Group();
+        $group->domain = $this->domainFor($input['domain'] ?? null);
+        $this->apply($group, $input, null);
+        $snapshot = $this->effects->snapshot($group->usernames());
+        $sogo = $this->repo->insert($group);
+        $group->warnings = array_merge($this->sizeWarnings($group), $this->effects->apply($sogo, $snapshot),
+            $this->sharing->applyImap((int)$group->id), $this->sharing->applySogo((int)$group->id));
+        return $group;
+    }
+
+    /**
+     * Updates a group; attributes missing in $input keep their value. `reapply_template` (admin) applies the
+     * effective templates to all members again.
+     *
+     * @throws UserError
+     */
+    public function update(int $id, array $input): Group
+    {
+        $this->requireEditor();
+        $current = $this->editable($id);
+        $group = clone $current;
+        $this->apply($group, $input, $current);
+        $snapshot = $this->effects->snapshot(array_merge($current->usernames(), $group->usernames()));
+        $sogo = $this->repo->update($group);
+        $group->warnings = array_merge($this->sizeWarnings($group), $this->effects->apply($sogo, $snapshot),
+            $this->sharing->applyImap((int)$group->id), $this->sharing->applySogo((int)$group->id));
+        if ($this->context->hasAdminRights() && self::flag($input['reapply_template'] ?? 0) === 1) {
+            $group->warnings = array_merge($group->warnings, $this->templates->applyTo($group->usernames()));
+        }
+        return $group;
+    }
+
+    /**
+     * Deletes a group and returns it. Members keep their settings; another group's template may now apply.
+     *
+     * @throws UserError
+     */
+    public function delete(int $id): Group
+    {
+        $this->requireEditor();
+        $group = $this->editable($id);
+        $snapshot = $this->effects->snapshot($group->usernames());
+        $this->sharing->removeImap($group);
+        $sogo = $this->repo->delete($id);
+        $group->warnings = $this->effects->apply($sogo, $snapshot);
+        return $group;
+    }
+
+    /**
+     * Groups of a mailbox, its effective group template and the addresses it may send as through groups.
+     * For the mailbox itself and for admins/domain admins with access to it.
+     */
+    public function membershipsOf(string $username): array
+    {
+        $username = strtolower(trim($username));
+        $own = ($this->context->isUser() && $this->context->username === $username);
+        if (!$own && !($this->canRead() && $this->context->canAccessMailbox($username))) {
+            return [];
+        }
+        return [
+            'groups' => $this->lookup->membershipsOf($username),
+            'template' => $this->templates->effective([$username])[$username] ?? null,
+            'send_as' => $this->lookup->sendAsOf($username),
+        ];
+    }
+
+    /** @throws AccessDenied when the caller cannot see or edit the group */
+    private function editable(int $id): Group
+    {
+        $group = $this->get($id);
+        if ($group === null || !$this->canEdit($group)) {
+            throw new AccessDenied();
+        }
+        return $group;
+    }
+
+    /** @throws UserError domain_not_found, access_denied */
+    private function domainFor(mixed $value): string
+    {
+        $domain = Input::text($value);
+        $domain = ($domain === null || $domain === '') ? '' : (string)idn_to_ascii(strtolower($domain), 0, INTL_IDNA_VARIANT_UTS46);
+        if ($domain === '' || !$this->lookup->isDomain($domain)) {
+            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
+        }
+        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
+            throw new AccessDenied();
+        }
+        return $domain;
+    }
+
+    /**
+     * Add/edit rules shared by create() and update(): attributes missing in $input keep the value of $group,
+     * a new group starts with the model defaults. Everything is validated before anything is written.
+     *
+     * @throws UserError
+     */
+    private function apply(Group $group, array $input, ?Group $current): void
+    {
+        $name = isset($input['name']) ? Input::text($input['name']) : $group->name;
+        if ($name === null || $name === '' || mb_strlen($name) > 100) {
+            throw new UserError('group_name_invalid');
+        }
+        $description = isset($input['description']) ? Input::text($input['description']) : $group->description;
+        if ($description === null || mb_strlen($description) > 255) {
+            throw new UserError('group_description_invalid');
+        }
+        if ($this->repo->nameExists($group->domain, $name, $group->id)) {
+            throw new UserError('object_exists', [htmlspecialchars($name)]);
+        }
+        $group->name = $name;
+        $group->description = $description;
+        foreach (['sender_allowed' => 'senderAllowed', 'sogo_visible' => 'sogoVisible', 'internal' => 'internal', 'active' => 'active'] as $key => $property) {
+            if (isset($input[$key])) {
+                $group->$property = self::flag($input[$key]);
+            }
+        }
+        $this->applyAddress($group, $input, $current);
+        $this->applyMembers($group, $input);
+        $this->applySendAs($group, $input);
+        $this->applyGrants($group, $input);
+        if ($this->context->hasAdminRights()) {
+            $this->applyAdminFields($group, $input);
+        }
+    }
+
+    /** @throws UserError */
+    private function applyAddress(Group $group, array $input, ?Group $current): void
+    {
+        if (array_key_exists('address', $input)) {
+            $value = Input::text($input['address']);
+        } elseif (array_key_exists('local_part', $input)) {
+            $local = Input::text($input['local_part']);
+            $value = ($local === null || $local === '') ? $local : $local . '@' . $group->domain;
+        } else {
+            return;
+        }
+        if ($value === '') {
+            $group->address = null;
+            return;
+        }
+        $address = ($value === null) ? null : self::normalizeAddress($value);
+        if ($address === null || self::domainOf($address) !== $group->domain) {
+            throw new UserError('group_address_invalid', [htmlspecialchars((string)$value)]);
+        }
+        $group->address = $address;
+        if ($address === $current?->address) {
+            return;
+        }
+        if ($this->lookup->addressTaken($address, $group->id)) {
+            throw new UserError('is_alias_or_mailbox', [htmlspecialchars($address)]);
+        }
+        // A group with an address counts like an alias of the domain
+        if ($current?->address === null && $this->lookup->aliasesLeft($group->domain) < 1) {
+            throw new UserError('max_alias_exceeded');
+        }
+    }
+
+    /**
+     * `members` replaces the manual members the caller may access, `add_members` / `remove_members` change them.
+     *
+     * @throws UserError group_member_invalid, group_member_idp, access_denied
+     */
+    private function applyMembers(Group $group, array $input): void
+    {
+        $members = $group->members;
+        $add = [];
+        if (array_key_exists('members', $input)) {
+            $add = self::cleanList($input['members']);
+            foreach ($members as $username => $source) {
+                if ($source === Group::SOURCE_MANUAL && !in_array((string)$username, $add, true) && $this->context->canAccessMailbox((string)$username)) {
+                    unset($members[$username]);
+                }
+            }
+        }
+        $add = array_merge($add, self::cleanList($input['add_members'] ?? []));
+        foreach (self::cleanList($input['remove_members'] ?? []) as $username) {
+            if (!isset($members[$username])) {
+                continue;
+            }
+            if ($members[$username] === Group::SOURCE_IDP) {
+                throw new UserError('group_member_idp', [htmlspecialchars($username)]);
+            }
+            if (!$this->context->canAccessMailbox($username)) {
+                throw new AccessDenied();
+            }
+            unset($members[$username]);
+        }
+        $new = array_values(array_diff(array_unique($add), array_map('strval', array_keys($members))));
+        $regular = $this->lookup->regularMailboxes($new);
+        foreach ($new as $username) {
+            if (!in_array($username, $regular, true)) {
+                throw new UserError('group_member_invalid', [htmlspecialchars($username)]);
+            }
+            if (!$this->context->canAccessMailbox($username)) {
+                throw new AccessDenied();
+            }
+            $members[$username] = Group::SOURCE_MANUAL;
+        }
+        ksort($members);
+        $group->members = $members;
+    }
+
+    /**
+     * Addresses (mailboxes or aliases) and @domain the members may send as; the editor needs access to their domains.
+     *
+     * @throws UserError group_send_as_invalid, access_denied
+     */
+    private function applySendAs(Group $group, array $input): void
+    {
+        if (!array_key_exists('send_as', $input)) {
+            return;
+        }
+        $sendAs = [];
+        foreach (self::cleanList($input['send_as']) as $value) {
+            if (str_starts_with($value, '@')) {
+                $domain = (string)idn_to_ascii(substr($value, 1), 0, INTL_IDNA_VARIANT_UTS46);
+                $address = '@' . $domain;
+                $valid = ($domain !== '' && $this->lookup->isDomainOrAliasDomain($domain));
+            } else {
+                $address = self::normalizeAddress($value);
+                // The group's own address is covered by sender_allowed
+                $valid = ($address !== null && $address !== $group->address && $this->lookup->aliasExists($address));
+            }
+            if (!$valid) {
+                throw new UserError('group_send_as_invalid', [htmlspecialchars($value)]);
+            }
+            if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain(self::domainOf((string)$address))) {
+                throw new AccessDenied();
+            }
+            $sendAs[] = (string)$address;
+        }
+        $sendAs = array_values(array_unique($sendAs));
+        sort($sendAs);
+        $group->sendAs = $sendAs;
+    }
+
+    /**
+     * `imap_grants` [{owner, rights, subscribe}] and `sogo_grants` [{owner, type, folder, access, subscribe}] replace
+     * the grants; the editor needs access to every owner mailbox.
+     *
+     * @throws UserError group_grant_invalid, access_denied
+     */
+    private function applyGrants(Group $group, array $input): void
+    {
+        if (array_key_exists('imap_grants', $input)) {
+            $grants = [];
+            foreach (self::grantList($input['imap_grants']) as $grant) {
+                $owner = $this->grantOwner($grant);
+                $rights = Input::text($grant['rights'] ?? 'read');
+                if (!isset(SharingService::IMAP_RIGHTS[$rights])) {
+                    throw new UserError('group_grant_invalid', [htmlspecialchars($owner)]);
+                }
+                $grants[$owner] = ['owner' => $owner, 'rights' => $rights, 'subscribe' => self::flag($grant['subscribe'] ?? 0)];
+            }
+            ksort($grants);
+            $group->imapGrants = array_values($grants);
+        }
+        if (array_key_exists('sogo_grants', $input)) {
+            $grants = [];
+            foreach (self::grantList($input['sogo_grants']) as $grant) {
+                $owner = $this->grantOwner($grant);
+                $type = Input::text($grant['type'] ?? 'Calendar');
+                $folder = Input::text($grant['folder'] ?? '') ?: 'personal';
+                $access = Input::text($grant['access'] ?? 'read');
+                if (!isset(GroupRepository::SOGO_ROLES[$type][$access]) || !preg_match('/^[A-Za-z0-9._-]{1,100}$/', $folder)) {
+                    throw new UserError('group_grant_invalid', [htmlspecialchars($owner)]);
+                }
+                $grants[$owner . '|' . $type . '|' . $folder] = ['owner' => $owner, 'type' => $type, 'folder' => $folder, 'access' => $access,
+                    'subscribe' => self::flag($grant['subscribe'] ?? 0)];
+            }
+            ksort($grants);
+            $group->sogoGrants = array_values($grants);
+        }
+    }
+
+    /** @throws UserError group_grant_invalid, access_denied */
+    private function grantOwner(array $grant): string
+    {
+        $owner = strtolower((string)Input::text($grant['owner'] ?? ''));
+        if ($owner === '' || empty($this->lookup->regularMailboxes([$owner]))) {
+            throw new UserError('group_grant_invalid', [htmlspecialchars($owner)]);
+        }
+        if (!$this->context->canAccessMailbox($owner)) {
+            throw new AccessDenied();
+        }
+        return $owner;
+    }
+
+    /**
+     * Grant objects of a JSON body or of a form field (JSON string)
+     *
+     * @throws UserError group_grant_invalid
+     */
+    private static function grantList(mixed $value): array
+    {
+        if (is_string($value)) {
+            $value = ($value === '') ? [] : json_decode($value, true);
+        }
+        if (!is_array($value)) {
+            throw new UserError('group_grant_invalid', ['']);
+        }
+        foreach ($value as $grant) {
+            if (!is_array($grant)) {
+                throw new UserError('group_grant_invalid', ['']);
+            }
+        }
+        return array_values($value);
+    }
+
+    /** @throws UserError template_id_invalid, group_priority_invalid */
+    private function applyAdminFields(Group $group, array $input): void
+    {
+        if (array_key_exists('template_id', $input)) {
+            $id = $input['template_id'];
+            if ($id === null || $id === '' || $id === '0' || $id === 0) {
+                $group->templateId = null;
+                $group->template = null;
+            } else {
+                $templateId = Input::intInRange($id, 1, PHP_INT_MAX);
+                $name = ($templateId === null) ? null : $this->lookup->templateName($templateId);
+                if ($name === null) {
+                    throw new UserError('template_id_invalid', [htmlspecialchars((string)Input::text($id))]);
+                }
+                $group->templateId = $templateId;
+                $group->template = $name;
+            }
+        }
+        if (isset($input['priority'])) {
+            $priority = Input::intInRange($input['priority'], -1000000, 1000000);
+            if ($priority === null) {
+                throw new UserError('group_priority_invalid');
+            }
+            $group->priority = $priority;
+        }
+        if (array_key_exists('idp_ref', $input)) {
+            $ref = Input::text($input['idp_ref']);
+            if ($ref === null || mb_strlen($ref) > 255) {
+                throw new AccessDenied();
+            }
+            $group->idpRef = ($ref === '') ? null : $ref;
+        }
+    }
+
+    /** @return array[] */
+    private function sizeWarnings(Group $group): array
+    {
+        if ($group->address !== null && count($group->members) > self::MAX_RECIPIENTS) {
+            return [['group_too_many_members', htmlspecialchars($group->name), self::MAX_RECIPIENTS]];
+        }
+        return [];
+    }
+
+    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
+    private static function flag(mixed $value): int
+    {
+        if (is_array($value)) {
+            $value = end($value);
+        }
+        return filter_var($value, FILTER_VALIDATE_BOOLEAN) ? 1 : 0;
+    }
+
+    /** @return string[] trimmed, lowercase, unique entries of an array or of a string separated by spaces, commas, semicolons or newlines */
+    private static function cleanList(mixed $value): array
+    {
+        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', is_scalar($value) ? (string)$value : '');
+        $clean = [];
+        foreach ($values as $item) {
+            if (is_scalar($item) && trim((string)$item) !== '') {
+                $clean[] = strtolower(trim((string)$item));
+            }
+        }
+        return array_values(array_unique($clean));
+    }
+
+    /** Lowercase address with ASCII (IDN) domain, null if invalid */
+    private static function normalizeAddress(string $value): ?string
+    {
+        $value = strtolower(trim($value));
+        $at = strrpos($value, '@');
+        if ($at === false || $at === 0) {
+            return null;
+        }
+        $domain = idn_to_ascii(substr($value, $at + 1), 0, INTL_IDNA_VARIANT_UTS46);
+        $address = substr($value, 0, $at) . '@' . $domain;
+        return ($domain !== false && filter_var($address, FILTER_VALIDATE_EMAIL)) ? $address : null;
+    }
+
+    private static function domainOf(string $address): string
+    {
+        return substr($address, strrpos($address, '@') + 1);
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/IdpSyncService.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\Core\UserError;
+use Mailcow\MailboxGroup\Model\Group;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+
+/**
+ * Entry points for the IdP sync: system caller, no permission checks, never routed in json_api.
+ * Only memberships with source 'idp' are managed here; manual memberships stay.
+ */
+class IdpSyncService
+{
+    public function __construct(private GroupRepository $repo, private LookupRepository $lookup, private ChangeEffects $effects)
+    {
+    }
+
+    public function findByIdpRef(string $ref): ?Group
+    {
+        return $this->repo->findByIdpRef($ref);
+    }
+
+    /** @return array[] every group with its members, see GroupRepository::syncState() */
+    public function groups(): array
+    {
+        return $this->repo->syncState();
+    }
+
+    /**
+     * Creates the group of an IdP group, members follow with syncGroupMembers(). The group mail becomes the
+     * distribution address when it is in the group's domain and free.
+     *
+     * @return array[] warnings (msg values)
+     * @throws UserError
+     */
+    public function createFromIdp(string $ref, string $name, string $domain, ?string $mail): array
+    {
+        $name = $this->validName($domain, $name, null);
+        if (!$this->lookup->isDomain($domain)) {
+            throw new UserError('domain_not_found', [htmlspecialchars($domain)]);
+        }
+        $group = new Group();
+        $group->domain = $domain;
+        $group->name = $name;
+        $group->idpRef = $ref;
+        $warnings = [];
+        if ($mail !== null && filter_var($mail, FILTER_VALIDATE_EMAIL) && str_ends_with($mail, '@' . $domain)) {
+            if ($this->lookup->addressTaken($mail, null) || $this->lookup->aliasesLeft($domain) < 1) {
+                $warnings[] = ['group_idp_address_skipped', htmlspecialchars($mail), htmlspecialchars($name)];
+            } else {
+                $group->address = $mail;
+            }
+        }
+        $this->repo->insert($group);
+        return $warnings;
+    }
+
+    /** @throws UserError */
+    public function renameFromIdp(int $groupId, string $name): void
+    {
+        $group = $this->repo->find($groupId);
+        if ($group === null) {
+            return;
+        }
+        $group->name = $this->validName($group->domain, $name, $groupId);
+        $this->repo->update($group);
+    }
+
+    /**
+     * Sets the IdP members of a group. Usernames that are no regular mailbox are skipped; a manual member that
+     * the IdP also lists stays manual.
+     *
+     * @return array{added: string[], removed: string[], skipped: string[], warnings: array[]}
+     */
+    public function syncGroupMembers(int $groupId, array $usernames): array
+    {
+        $result = ['added' => [], 'removed' => [], 'skipped' => [], 'warnings' => []];
+        $group = $this->repo->find($groupId);
+        if ($group === null) {
+            return $result;
+        }
+        $wanted = array_values(array_unique(array_map(fn(mixed $u): string => strtolower(trim((string)$u)), $usernames)));
+        $regular = $this->lookup->regularMailboxes($wanted);
+        $result['skipped'] = array_values(array_diff($wanted, $regular));
+        $members = $group->members;
+        foreach ($members as $username => $source) {
+            if ($source === Group::SOURCE_IDP && !in_array((string)$username, $regular, true)) {
+                unset($members[$username]);
+                $result['removed'][] = (string)$username;
+            }
+        }
+        foreach ($regular as $username) {
+            if (!isset($members[$username])) {
+                $members[$username] = Group::SOURCE_IDP;
+                $result['added'][] = $username;
+            }
+        }
+        if (empty($result['added']) && empty($result['removed'])) {
+            return $result;
+        }
+        ksort($members);
+        $snapshot = $this->effects->snapshot(array_merge($group->usernames(), array_keys($members)));
+        $group->members = $members;
+        $result['warnings'] = $this->effects->apply($this->repo->update($group), $snapshot);
+        return $result;
+    }
+
+    /** @throws UserError */
+    private function validName(string $domain, string $name, ?int $groupId): string
+    {
+        $name = trim($name);
+        if ($name === '' || mb_strlen($name) > 100) {
+            throw new UserError('group_name_invalid');
+        }
+        if ($this->repo->nameExists($domain, $name, $groupId)) {
+            throw new UserError('object_exists', [htmlspecialchars($name)]);
+        }
+        return $name;
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/SharingService.php

@@ -0,0 +1,169 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\Dovecot\Service\DovecotService;
+use Mailcow\MailboxGroup\Model\Group;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\Sogo\Service\SogoService;
+
+/**
+ * Mailbox access of group members (shared mailboxes, calendars, address books).
+ *
+ * IMAP: a grant is the Dovecot ACL group=mcg_<id> on every folder of the owner; members get the group through
+ * the userdb field acl_groups, so membership changes need no ACL change. Dovecot is reached through dockerapi
+ * after the commit; subfolders created later inherit the ACL, new top-level folders and failed grants are
+ * handled by reconcile() (cron).
+ * SOGo: sogo_acl rows per member, maintained by GroupRepository::rebuild(); missing personal folders of owners that
+ * never logged in to SOGo (shared mailboxes) are created with sogo-tool init-user.
+ * Grants with `subscribe` are subscribed for each member once by syncSubscriptions(); a member that removes the
+ * subscription keeps it removed, members that lose the access are unsubscribed.
+ */
+class SharingService
+{
+    /** Dovecot rights per grant level */
+    public const IMAP_RIGHTS = [
+        'read' => ['lookup', 'read', 'write-seen'],
+        'write' => ['lookup', 'read', 'write-seen', 'write', 'write-deleted', 'insert', 'post', 'expunge', 'create'],
+        'full' => ['lookup', 'read', 'write-seen', 'write', 'write-deleted', 'insert', 'post', 'expunge', 'create', 'delete'],
+    ];
+
+    public function __construct(private GroupRepository $repo, private DovecotService $dovecot, private SogoService $sogo,
+        private ChangeEffects $effects)
+    {
+    }
+
+    /** Group name in the userdb field acl_groups (dovecot docker-entrypoint.sh) */
+    public static function aclGroup(int $groupId): string
+    {
+        return 'mcg_' . $groupId;
+    }
+
+    /**
+     * Applies the IMAP grants Dovecot does not have yet (pending, failed, removing); $all also applies the applied
+     * ones again. Of one group or all.
+     *
+     * @return array[] warnings (msg values)
+     */
+    public function applyImap(?int $groupId, bool $all = false): array
+    {
+        $warnings = [];
+        foreach ($this->repo->imapGrantsToApply($groupId, $all) as $grant) {
+            $removing = ($grant['state'] === 'removing');
+            $error = $this->dovecot->setAclAll((string)$grant['owner'], 'group=' . self::aclGroup((int)$grant['group_id']),
+                $removing ? [] : self::IMAP_RIGHTS[$grant['rights']]);
+            if ($error === null && $removing) {
+                $this->repo->deleteImapGrant((int)$grant['id']);
+                continue;
+            }
+            if ($error === null) {
+                if ($grant['state'] !== 'applied' || $grant['last_error'] !== null) {
+                    $this->repo->setImapState((int)$grant['id'], 'applied', null);
+                }
+                continue;
+            }
+            $this->repo->setImapState((int)$grant['id'], $removing ? 'removing' : 'failed', $error);
+            $warnings[] = ['group_imap_grant_failed', htmlspecialchars((string)$grant['owner'])];
+        }
+        return $warnings;
+    }
+
+    /** After a group write: creates missing SOGo folders of its grant owners and applies the pending grants to them */
+    public function applySogo(int $groupId): array
+    {
+        $warnings = $this->initSogoFolders($groupId);
+        // null: no grant waits for a folder, warnings: the folders could not be created
+        if ($warnings === null || $warnings !== []) {
+            return $warnings ?? [];
+        }
+        return $this->effects->apply($this->repo->rebuild($groupId), []);
+    }
+
+    /** Before a group is deleted: its Dovecot ACLs go (best effort, the ACL of a deleted group grants nothing) */
+    public function removeImap(Group $group): void
+    {
+        foreach ($group->imapGrants as $grant) {
+            $this->dovecot->setAclAll((string)$grant['owner'], 'group=' . self::aclGroup((int)$group->id), []);
+        }
+    }
+
+    /**
+     * Cron: applies all IMAP grants again (new top-level folders, failed grants), rebuilds the SOGo grants
+     * (owners that opened SOGo meanwhile, rows of removed grants) and syncs the SOGo subscriptions.
+     *
+     * @return array[] warnings (msg values)
+     */
+    public function reconcile(): array
+    {
+        $warnings = array_merge($this->applyImap(null, true), $this->initSogoFolders(null) ?? []);
+        $users = [];
+        foreach ($this->repo->groupIdsWithSogoGrants() as $id) {
+            $users = array_merge($users, $this->repo->rebuild($id));
+        }
+        $this->repo->cleanupSogoAcl();
+        $warnings = array_merge($warnings, $this->effects->apply(array_values(array_unique($users)), []));
+        return array_merge($warnings, $this->syncSubscriptions());
+    }
+
+    /**
+     * Unsubscribes members that lost the access to a folder mailcow subscribed them to, then subscribes the members
+     * of grants with `subscribe` that have no subscription by mailcow yet. Cheap without pending changes, the cron
+     * runs it every minute.
+     *
+     * @return array[] warnings (msg values)
+     */
+    public function syncSubscriptions(): array
+    {
+        $warnings = [];
+        $sogoChanged = false;
+        foreach (['imap', 'sogo'] as $kind) {
+            foreach ($this->repo->staleSubscriptions($kind) as $row) {
+                // A deleted member has no subscriptions left, a deleted owner no folders; another group may still grant it
+                if ($row['member_exists'] && $row['owner_exists'] && !$row['covered']) {
+                    if ($this->subscribe($kind, (string)$row['owner'], (string)$row['folder'], [(string)$row['username']], true) !== null) {
+                        $warnings[] = ['group_subscription_failed', htmlspecialchars((string)$row['username'])];
+                        continue;
+                    }
+                    $sogoChanged = $sogoChanged || ($kind === 'sogo');
+                }
+                $this->repo->deleteSubscription((int)$row['id']);
+            }
+            foreach ($this->repo->missingSubscriptions($kind) as $grant) {
+                if ($this->subscribe($kind, $grant['owner'], $grant['folder'], $grant['usernames']) !== null) {
+                    $warnings[] = ['group_subscription_failed', htmlspecialchars(implode(', ', $grant['usernames']))];
+                    continue;
+                }
+                $this->repo->addSubscriptions($kind, $grant['grant_id'], $grant['owner'], $grant['folder'], $grant['usernames']);
+                $sogoChanged = $sogoChanged || ($kind === 'sogo');
+            }
+        }
+        if ($sogoChanged) {
+            $this->sogo->flushCache();
+        }
+        return $warnings;
+    }
+
+    /** @return ?string null on success, else the error */
+    private function subscribe(string $kind, string $owner, string $folder, array $usernames, bool $unsubscribe = false): ?string
+    {
+        return ($kind === 'imap') ? $this->dovecot->subscribeShared($owner, $usernames, $unsubscribe)
+            : $this->sogo->subscribe($owner, $folder, $usernames, $unsubscribe);
+    }
+
+    /**
+     * Creates the personal SOGo folders of grant owners that never logged in to SOGo, so their pending grants can
+     * be applied.
+     *
+     * @return ?array[] warnings, null if no grant was waiting for a folder
+     */
+    private function initSogoFolders(?int $groupId): ?array
+    {
+        $owners = $this->repo->pendingSogoOwners($groupId);
+        if (empty($owners)) {
+            return null;
+        }
+        return ($this->sogo->initUsers($owners) === null) ? [] : [['group_sogo_init_failed', htmlspecialchars(implode(', ', $owners))]];
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Service/TemplateService.php

@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailboxGroup\Service;
+
+use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+
+/**
+ * Group templates: the effective template of a mailbox is the one of its active group with the highest
+ * priority. It is applied only when it changes, so manual settings of a mailbox stay until then.
+ * A mailbox that leaves all groups keeps its settings.
+ */
+class TemplateService
+{
+    public function __construct(private LookupRepository $lookup, private LegacyMailbox $legacy)
+    {
+    }
+
+    /** @return array<string, ?string> username => effective group template */
+    public function effective(array $usernames): array
+    {
+        return $this->lookup->effectiveTemplates(array_values(array_unique($usernames)));
+    }
+
+    /**
+     * Applies the effective template to the mailboxes whose effective template differs from $before
+     * (a snapshot of effective() taken before the change).
+     *
+     * @return array[] warnings (msg values)
+     */
+    public function applyChanged(array $before): array
+    {
+        $changed = [];
+        foreach ($this->effective(array_keys($before)) as $username => $template) {
+            if ($template !== null && $template !== $before[$username]) {
+                $changed[$username] = $template;
+            }
+        }
+        return $this->apply($changed);
+    }
+
+    /** Applies the effective templates again, e.g. after manual changes (admin action) */
+    public function applyTo(array $usernames): array
+    {
+        return $this->apply(array_filter($this->effective($usernames), fn(?string $template): bool => $template !== null));
+    }
+
+    /** A template was edited: apply it to the mailboxes it is the effective template of */
+    public function onTemplateChanged(int $templateId): array
+    {
+        $name = $this->lookup->templateName($templateId);
+        if ($name === null) {
+            return [];
+        }
+        return $this->apply(array_filter($this->effective($this->lookup->membersWithTemplate($templateId)),
+            fn(?string $template): bool => $template === $name));
+    }
+
+    /** Template for IdP logins and syncs: the effective group template, else the legacy attribute mapping */
+    public function templateForLogin(string $username, ?string $legacy): ?string
+    {
+        return $this->effective([$username])[$username] ?? $legacy;
+    }
+
+    /**
+     * @param array<string, string> $templates username => template
+     * @return array[] warnings (msg values)
+     */
+    private function apply(array $templates): array
+    {
+        $warnings = [];
+        $names = $this->lookup->mailboxNames(array_keys($templates));
+        foreach ($templates as $username => $template) {
+            $error = $this->legacy->applyTemplate((string)$username, (string)($names[$username] ?? ''), $template);
+            if ($error !== null) {
+                $warnings[] = ['group_template_failed', htmlspecialchars($template), htmlspecialchars((string)$username)];
+            }
+        }
+        return $warnings;
+    }
+}

data/web/inc/lib/mailcow/README.md

@@ -30,6 +30,15 @@ Core/                     shared by all modules
 
 Example: `Imapsync/` (sync jobs, sync sources, OAuth). Use it as the template for new modules.
 `ApiKey/` (API keys incl. the service keys of mailcow.conf) is a small module with the same layout.
+`MailboxGroup/` (mailbox groups) keeps its own tables as source of truth and maintains rows the legacy code and
+Postfix read (`alias`, `sender_acl`, tagged with `group_id`); calls into procedural code go through `Legacy/`.
+Shares become Dovecot ACLs on `group=mcg_<id>` (userdb `acl_groups`) and `sogo_acl` rows tracked in a ledger.
+`Dovecot/` (doveadm through dockerapi: ACLs, subscriptions, auth cache) and `Sogo/` (user view, cache, sogo-tool
+through dockerapi) are used by other modules instead of calling `docker()` or the SOGo functions themselves.
+`Identity/` (IdP links, sync runs, offboarding) lists Keycloak/LDAP through `Directory/` adapters, decides in the
+side-effect free `SyncPlanner` and is run by the `identity-sync.php` cron.
+`Auth/` (logins) holds for now the IdP login steps the legacy login flows call (link checks fail closed, group
+template before the attribute mapping); the auth logic of the function files moves here step by step.
 
 ## Layers and rules
 

data/web/inc/lib/mailcow/Sogo/Service/SogoService.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Sogo\Service;
+
+/**
+ * SOGo: its user view, its cache and sogo-tool in the sogo container through dockerapi. The legacy calls sit in
+ * protected methods so tests can replace them.
+ */
+class SogoService
+{
+    /** More changed mailboxes than this: one full view rebuild instead of single updates */
+    private const SINGLE_UPDATES = 25;
+
+    /** Refreshes the SOGo identities (visible aliases, send-as addresses) of the given mailboxes */
+    public function refreshStaticView(array $usernames): void
+    {
+        $usernames = array_values(array_unique($usernames));
+        if (empty($usernames)) {
+            return;
+        }
+        if (count($usernames) > self::SINGLE_UPDATES) {
+            $this->updateStaticView(null);
+            return;
+        }
+        foreach ($usernames as $username) {
+            $this->updateStaticView($username);
+        }
+    }
+
+    /** SOGo caches ACLs and user settings in memcached */
+    public function flushCache(): void
+    {
+        $this->flushMemcached();
+    }
+
+    /**
+     * Subscribes the users to a folder of the owner (sogo-tool manage-acl), or unsubscribes them.
+     *
+     * @param string $folder Calendar/<id> or Contacts/<id>
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function subscribe(string $owner, string $folder, array $usernames, bool $unsubscribe = false): ?string
+    {
+        return self::error($this->exec(['cmd' => 'sogo', 'task' => 'subscribe', 'owner' => $owner, 'folder' => $folder,
+            'users' => array_values($usernames), 'unsubscribe' => $unsubscribe]));
+    }
+
+    /**
+     * Creates the personal folders of users that never logged in to SOGo (sogo-tool init-user).
+     *
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function initUsers(array $usernames): ?string
+    {
+        return self::error($this->exec(['cmd' => 'sogo', 'task' => 'init_user', 'users' => array_values($usernames)]));
+    }
+
+    protected function updateStaticView(?string $username): void
+    {
+        update_sogo_static_view($username);
+    }
+
+    protected function flushMemcached(): void
+    {
+        flush_memcached();
+    }
+
+    protected function exec(array $fields): mixed
+    {
+        return docker('post', 'sogo-mailcow', 'exec', $fields);
+    }
+
+    private static function error(mixed $response): ?string
+    {
+        $result = json_decode((string)$response, true);
+        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
+    }
+}

data/web/inc/lib/mailcow/Sogo/SogoModule.php

@@ -0,0 +1,22 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Sogo;
+
+use Mailcow\Sogo\Service\SogoService;
+
+/**
+ * SOGo (user view, cache, sogo-tool through dockerapi). Modules use it instead of the legacy SOGo functions.
+ */
+class SogoModule
+{
+    public function __construct(public readonly SogoService $sogo)
+    {
+    }
+
+    public static function create(): self
+    {
+        return new self(new SogoService());
+    }
+}

data/web/inc/lib/vendor/autoload.php

@@ -14,10 +14,7 @@ if (PHP_VERSION_ID < 50600) {
             echo $err;
         }
     }
-    trigger_error(
-        $err,
-        E_USER_ERROR
-    );
+    throw new RuntimeException($err);
 }
 
 require_once __DIR__ . '/composer/autoload_real.php';

data/web/inc/lib/vendor/bin/minifycss

@@ -12,6 +12,7 @@
 
 namespace Composer;
 
+$GLOBALS['_composer_bin_dir'] = __DIR__;
 $GLOBALS['_composer_autoload_path'] = __DIR__ . '/..'.'/autoload.php';
 
 if (PHP_VERSION_ID < 80000) {
@@ -23,18 +24,17 @@ if (PHP_VERSION_ID < 80000) {
         {
             private $handle;
             private $position;
+            private $realpath;
 
             public function stream_open($path, $mode, $options, &$opened_path)
             {
-                // get rid of composer-bin-proxy:// prefix for __FILE__ & __DIR__ resolution
-                $opened_path = substr($path, 21);
-                $opened_path = realpath($opened_path) ?: $opened_path;
-                $this->handle = fopen($opened_path, $mode);
+                // get rid of phpvfscomposer:// prefix for __FILE__ & __DIR__ resolution
+                $opened_path = substr($path, 17);
+                $this->realpath = realpath($opened_path) ?: $opened_path;
+                $opened_path = $this->realpath;
+                $this->handle = fopen($this->realpath, $mode);
                 $this->position = 0;
 
-                // remove all traces of this stream wrapper once it has been used
-                stream_wrapper_unregister('composer-bin-proxy');
-
                 return (bool) $this->handle;
             }
 
@@ -66,6 +66,16 @@ if (PHP_VERSION_ID < 80000) {
                 return $operation ? flock($this->handle, $operation) : true;
             }
 
+            public function stream_seek($offset, $whence)
+            {
+                if (0 === fseek($this->handle, $offset, $whence)) {
+                    $this->position = ftell($this->handle);
+                    return true;
+                }
+
+                return false;
+            }
+
             public function stream_tell()
             {
                 return $this->position;
@@ -78,20 +88,32 @@ if (PHP_VERSION_ID < 80000) {
 
             public function stream_stat()
             {
-                return fstat($this->handle);
+                return array();
             }
 
             public function stream_set_option($option, $arg1, $arg2)
             {
                 return true;
             }
+
+            public function url_stat($path, $flags)
+            {
+                $path = substr($path, 17);
+                if (file_exists($path)) {
+                    return stat($path);
+                }
+
+                return false;
+            }
         }
     }
 
-    if (function_exists('stream_wrapper_register') && stream_wrapper_register('composer-bin-proxy', 'Composer\BinProxyWrapper')) {
-        include("composer-bin-proxy://" . __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifycss');
-        exit(0);
+    if (
+        (function_exists('stream_get_wrappers') && in_array('phpvfscomposer', stream_get_wrappers(), true))
+        || (function_exists('stream_wrapper_register') && stream_wrapper_register('phpvfscomposer', 'Composer\BinProxyWrapper'))
+    ) {
+        return include("phpvfscomposer://" . __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifycss');
     }
 }
 
-include __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifycss';
+return include __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifycss';

data/web/inc/lib/vendor/bin/minifyjs

@@ -12,6 +12,7 @@
 
 namespace Composer;
 
+$GLOBALS['_composer_bin_dir'] = __DIR__;
 $GLOBALS['_composer_autoload_path'] = __DIR__ . '/..'.'/autoload.php';
 
 if (PHP_VERSION_ID < 80000) {
@@ -23,18 +24,17 @@ if (PHP_VERSION_ID < 80000) {
         {
             private $handle;
             private $position;
+            private $realpath;
 
             public function stream_open($path, $mode, $options, &$opened_path)
             {
-                // get rid of composer-bin-proxy:// prefix for __FILE__ & __DIR__ resolution
-                $opened_path = substr($path, 21);
-                $opened_path = realpath($opened_path) ?: $opened_path;
-                $this->handle = fopen($opened_path, $mode);
+                // get rid of phpvfscomposer:// prefix for __FILE__ & __DIR__ resolution
+                $opened_path = substr($path, 17);
+                $this->realpath = realpath($opened_path) ?: $opened_path;
+                $opened_path = $this->realpath;
+                $this->handle = fopen($this->realpath, $mode);
                 $this->position = 0;
 
-                // remove all traces of this stream wrapper once it has been used
-                stream_wrapper_unregister('composer-bin-proxy');
-
                 return (bool) $this->handle;
             }
 
@@ -66,6 +66,16 @@ if (PHP_VERSION_ID < 80000) {
                 return $operation ? flock($this->handle, $operation) : true;
             }
 
+            public function stream_seek($offset, $whence)
+            {
+                if (0 === fseek($this->handle, $offset, $whence)) {
+                    $this->position = ftell($this->handle);
+                    return true;
+                }
+
+                return false;
+            }
+
             public function stream_tell()
             {
                 return $this->position;
@@ -78,20 +88,32 @@ if (PHP_VERSION_ID < 80000) {
 
             public function stream_stat()
             {
-                return fstat($this->handle);
+                return array();
             }
 
             public function stream_set_option($option, $arg1, $arg2)
             {
                 return true;
             }
+
+            public function url_stat($path, $flags)
+            {
+                $path = substr($path, 17);
+                if (file_exists($path)) {
+                    return stat($path);
+                }
+
+                return false;
+            }
         }
     }
 
-    if (function_exists('stream_wrapper_register') && stream_wrapper_register('composer-bin-proxy', 'Composer\BinProxyWrapper')) {
-        include("composer-bin-proxy://" . __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifyjs');
-        exit(0);
+    if (
+        (function_exists('stream_get_wrappers') && in_array('phpvfscomposer', stream_get_wrappers(), true))
+        || (function_exists('stream_wrapper_register') && stream_wrapper_register('phpvfscomposer', 'Composer\BinProxyWrapper'))
+    ) {
+        return include("phpvfscomposer://" . __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifyjs');
     }
 }
 
-include __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifyjs';
+return include __DIR__ . '/..'.'/matthiasmullie/minify/bin/minifyjs';

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/.github/workflows/tests.yml

@@ -0,0 +1,74 @@
+name: Test Suite
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
+      mongodb:
+        image: mongo
+        ports:
+          - 27017:27017
+      myriadb:
+        image: mariadb
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3808:3808
+          - 3306:3306
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: oauth2_server_php
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: --health-cmd="pg_isready -h localhost" --health-interval=10s --health-timeout=5s --health-retries=5
+    strategy:
+      matrix:
+        php: [ 7.2, 7.3, 7.4, "8.0", 8.1, 8.2 ]
+    name: "PHP ${{ matrix.php }} Unit Test"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: codecov/codecov-action@v1
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php }}
+          extensions: mongodb, mbstring, intl, redis, pdo_mysql
+      - name: Install composer dependencies
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: composer install
+      - name: Run PHPUnit
+        run: vendor/bin/phpunit -v
+  phpstan:
+    name: "PHPStan"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: 8.1
+      - name: Install composer dependencies
+        uses: nick-invision/retry@v1
+        with:
+          timeout_minutes: 10
+          max_attempts: 3
+          command: composer install
+      - name: Run PHPStan
+        run: |
+          composer require phpstan/phpstan
+          vendor/bin/phpstan analyse --level=0 src/

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/README.md

@@ -1,8 +1,6 @@
 oauth2-server-php
 =================
 
-[![Build Status](https://travis-ci.org/bshaffer/oauth2-server-php.svg?branch=master)](https://travis-ci.org/bshaffer/oauth2-server-php)
-
-[![Total Downloads](https://poser.pugx.org/bshaffer/oauth2-server-php/downloads.png)](https://packagist.org/packages/bshaffer/oauth2-server-php)
+[![Total Downloads](https://poser.pugx.org/bshaffer/oauth2-server-php/downloads)](https://packagist.org/packages/bshaffer/oauth2-server-php)
 
 View the [complete documentation](https://bshaffer.github.io/oauth2-server-php-docs/)

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/composer.json

@@ -16,21 +16,21 @@
         "psr-0": { "OAuth2": "src/" }
     },
     "require":{
-        "php":">=5.3.9"
+        "php":">=7.2"
     },
     "require-dev": {
-        "phpunit/phpunit": "^4.0",
-        "aws/aws-sdk-php": "~2.8",
-        "firebase/php-jwt": "~2.2",
-        "predis/predis": "dev-master",
+        "phpunit/phpunit": "^7.5||^8.0",
+        "aws/aws-sdk-php": "^2.8",
+        "firebase/php-jwt": "^6.4",
+        "predis/predis": "^1.1",
         "thobbs/phpcassa": "dev-master",
-        "mongodb/mongodb": "^1.1"
+        "yoast/phpunit-polyfills": "^1.0"
     },
     "suggest": {
         "predis/predis": "Required to use Redis storage",
         "thobbs/phpcassa": "Required to use Cassandra storage",
         "aws/aws-sdk-php": "~2.8 is required to use DynamoDB storage",
-        "firebase/php-jwt": "~2.2 is required to use JWT features",
+        "firebase/php-jwt": "~v6.4 is required to use JWT features",
         "mongodb/mongodb": "^1.1 is required to use MongoDB storage"
     }
 }

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Autoloader.php

@@ -39,7 +39,7 @@ class Autoloader
      * Handles autoloading of classes.
      *
      * @param string $class - A class name.
-     * @return boolean      - Returns true if the class has been loaded
+     * @return void
      */
     public function autoload($class)
     {

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/ClientAssertionType/HttpBasic.php

@@ -112,7 +112,7 @@ class HttpBasic implements ClientAssertionTypeInterface
      *
      * @ingroup oauth2_section_2
      */
-    public function getClientCredentials(RequestInterface $request, ResponseInterface $response = null)
+    public function getClientCredentials(RequestInterface $request, ?ResponseInterface $response = null)
     {
         if (!is_null($request->headers('PHP_AUTH_USER')) && !is_null($request->headers('PHP_AUTH_PW'))) {
             return array('client_id' => $request->headers('PHP_AUTH_USER'), 'client_secret' => $request->headers('PHP_AUTH_PW'));

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Controller/AuthorizeController.php

@@ -78,7 +78,7 @@ class AuthorizeController implements AuthorizeControllerInterface
      *     );
      * @endcode
      */
-    public function __construct(ClientInterface $clientStorage, array $responseTypes = array(), array $config = array(), ScopeInterface $scopeUtil = null)
+    public function __construct(ClientInterface $clientStorage, array $responseTypes = array(), array $config = array(), ?ScopeInterface $scopeUtil = null)
     {
         $this->clientStorage = $clientStorage;
         $this->responseTypes = $responseTypes;
@@ -87,6 +87,7 @@ class AuthorizeController implements AuthorizeControllerInterface
             'enforce_state'  => true,
             'require_exact_redirect_uri' => true,
             'redirect_status_code' => 302,
+            'enforce_pkce' => false,
         ), $config);
 
         if (is_null($scopeUtil)) {

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Controller/ResourceController.php

@@ -47,7 +47,7 @@ class ResourceController implements ResourceControllerInterface
      * @param array                $config
      * @param ScopeInterface       $scopeUtil
      */
-    public function __construct(TokenTypeInterface $tokenType, AccessTokenInterface $tokenStorage, $config = array(), ScopeInterface $scopeUtil = null)
+    public function __construct(TokenTypeInterface $tokenType, AccessTokenInterface $tokenStorage, $config = array(), ?ScopeInterface $scopeUtil = null)
     {
         $this->tokenType = $tokenType;
         $this->tokenStorage = $tokenStorage;

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Controller/ResourceControllerInterface.php

@@ -13,7 +13,7 @@ use OAuth2\ResponseInterface;
  * @code
  *     if (!$resourceController->verifyResourceRequest(OAuth2\Request::createFromGlobals(), $response = new OAuth2\Response())) {
  *         $response->send(); // authorization failed
- *         exit();
+ *         die();
  *     }
  *     return json_encode($resource); // valid token!  Send the stuff!
  * @endcode

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Controller/TokenController.php

@@ -54,7 +54,7 @@ class TokenController implements TokenControllerInterface
      * @param ScopeInterface               $scopeUtil
      * @throws InvalidArgumentException
      */
-    public function __construct(AccessTokenInterface $accessToken, ClientInterface $clientStorage, array $grantTypes = array(), ClientAssertionTypeInterface $clientAssertionType = null, ScopeInterface $scopeUtil = null)
+    public function __construct(AccessTokenInterface $accessToken, ClientInterface $clientStorage, array $grantTypes = array(), ?ClientAssertionTypeInterface $clientAssertionType = null, ?ScopeInterface $scopeUtil = null)
     {
         if (is_null($clientAssertionType)) {
             foreach ($grantTypes as $grantType) {

data/web/inc/lib/vendor/bshaffer/oauth2-server-php/src/OAuth2/Encryption/FirebaseJwt.php

@@ -2,6 +2,9 @@
 
 namespace OAuth2\Encryption;
 
+use Firebase\JWT\JWT;
+use Firebase\JWT\Key;
+
 /**
  * Bridge file to use the firebase/php-jwt package for JWT encoding and decoding.
  * @author Francis Chuang <francis.chuang@gmail.com>
@@ -10,26 +13,36 @@ class FirebaseJwt implements EncryptionInterface
 {
     public function __construct()
     {
-        if (!class_exists('\JWT')) {
+        if (!class_exists(JWT::class)) {
             throw new \ErrorException('firebase/php-jwt must be installed to use this feature. You can do this by running "composer require firebase/php-jwt"');
         }
     }
 
     public function encode($payload, $key, $alg = 'HS256', $keyId = null)
     {
-        return \JWT::encode($payload, $key, $alg, $keyId);
+        return JWT::encode($payload, $key, $alg, $keyId);
     }
 
     public function decode($jwt, $key = null, $allowedAlgorithms = null)
     {
         try {
-
             //Maintain BC: Do not verify if no algorithms are passed in.
             if (!$allowedAlgorithms) {
-                $key = null;
+                $tks = \explode('.', $jwt);
+                if (\count($tks) === 3) {
+                    [$headb64] = $tks;
+                    $headerRaw = JWT::urlsafeB64Decode($headb64);
+                    if (($header = JWT::jsonDecode($headerRaw))) {
+                        $key = new Key($key, $header->alg);
+                    }
+                }
+            } elseif(is_array($allowedAlgorithms)) {
+                $key = new Key($key, $allowedAlgorithms[0]);
+            } else {
+                $key = new Key($key, $allowedAlgorithms);
             }
 
-            return (array)\JWT::decode($jwt, $key, $allowedAlgorithms);
+            return (array) JWT::decode($jwt, $key);
         } catch (\Exception $e) {
             return false;
         }
@@ -37,11 +50,11 @@ class FirebaseJwt implements EncryptionInterface
 
     public function urlSafeB64Encode($data)
     {
-        return \JWT::urlsafeB64Encode($data);
+        return JWT::urlsafeB64Encode($data);
     }
 
     public function urlSafeB64Decode($b64)
     {
-        return \JWT::urlsafeB64Decode($b64);
+        return JWT::urlsafeB64Decode($b64);
     }
 }