[Rspamd] Update to 4.2.3

[Web] Update composer libraries

.github/workflows/close_old_issues_and_prs.yml

@@ -14,7 +14,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Mark/Close Stale Issues and Pull Requests 🗑️
-        uses: actions/stale@v10.2.0
+        uses: actions/stale@v11.0.0
         with:
           repo-token: ${{ secrets.STALE_ACTION_PAT }}
           days-before-stale: 60

.github/workflows/image_builds.yml

@@ -27,7 +27,7 @@ jobs:
           - "watchdog-mailcow"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Setup Docker
         run: |
           curl -sSL https://get.docker.com/ | CHANNEL=stable sudo sh

.github/workflows/pr_to_nightly.yml

@@ -8,11 +8,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Run the Action
-        uses: devops-infra/action-pull-request@v1.0.2
+        uses: devops-infra/action-pull-request@v1.4.0
         with:
           github_token: ${{ secrets.PRTONIGHTLY_ACTION_PAT }}
           title: Automatic PR to nightly from ${{ github.event.repository.updated_at}}

.github/workflows/rebuild_backup_image.yml

@@ -13,13 +13,13 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to GHCR
         if: github.event_name != 'pull_request'
@@ -30,7 +30,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.github/workflows/update_postscreen_access_list.yml

@@ -15,7 +15,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
     - name: Checkout
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
 
     - name: Generate postscreen_access.cidr
       run: |

.gitignore

@@ -64,6 +64,8 @@ data/web/templates/cache/*
 !data/web/templates/cache/.gitkeep
 data/web/.well-known/acme-challenge
 data/web/css/build/0081-custom-mailcow.css
+data/web/css/themes/*/
+!data/web/css/themes/servercow/
 data/web/inc/vars.local.inc.php
 data/web/inc/app_info.inc.php
 data/web/nextcloud*/
@@ -76,4 +78,6 @@ refresh_images.sh
 update_diffs/
 create_cold_standby.sh
 !data/conf/nginx/mailcow_auth.conf
-data/conf/postfix/postfix-tlspol
+data/conf/postfix/postfix-tlspol
+tests/php/vendor/
+tests/php/.phpunit.cache/

README.md

@@ -23,10 +23,15 @@ A big thank you to everyone supporting us on GitHub Sponsors—your contribution
   <a href="https://www.maehdros.com/" target=_blank><img
     src="https://avatars.githubusercontent.com/u/173894712" height="58"
   /></a>
+  <a href="https://garske-systems.de/" target=_blank><img
+    src="https://garske-systems.de/img/logo_min.png" height="58"
+  /></a>
 
 ### 50$/Month Sponsors
   <a href="https://github.com/vnukhr" target=_blank><img
     src="https://avatars.githubusercontent.com/u/7805987?s=52&v=4" height="58"
+  /></a><a href="https://github.com/humiico" target=_blank><img
+    src="https://avatars.githubusercontent.com/u/110134705?s=52&v=4" height="58"
   /></a>
 
 ## Info, documentation and support

_modules/scripts/new_options.sh

@@ -64,7 +64,7 @@ adapt_new_options() {
 
   sed -i --follow-symlinks '$a\' mailcow.conf
   for option in ${CONFIG_ARRAY[@]}; do
-    if grep -q "${option}" mailcow.conf; then
+    if grep -q "^#\?${option}=" mailcow.conf; then
       continue
     fi
 
@@ -111,16 +111,16 @@ adapt_new_options() {
             echo "SQL_PORT=127.0.0.1:13306" >> mailcow.conf
             ;;
         API_KEY)
-            echo '# Create or override API key for web UI' >> mailcow.conf
+            echo '# Read-write service API key, shown read-only in the web UI' >> mailcow.conf
             echo "#API_KEY=" >> mailcow.conf
             ;;
         API_KEY_READ_ONLY)
-            echo '# Create or override read-only API key for web UI' >> mailcow.conf
+            echo '# Read-only service API key, shown read-only in the web UI' >> mailcow.conf
             echo "#API_KEY_READ_ONLY=" >> mailcow.conf
             ;;
         API_ALLOW_FROM)
-            echo '# Must be set for API_KEY to be active' >> mailcow.conf
-            echo '# IPs only, no networks (networks can be set via UI)' >> mailcow.conf
+            echo '# Must be set for API_KEY and API_KEY_READ_ONLY to be active' >> mailcow.conf
+            echo '# Comma separated list of IPs/networks' >> mailcow.conf
             echo "#API_ALLOW_FROM=" >> mailcow.conf
             ;;
         SNAT_TO_SOURCE)
@@ -302,7 +302,7 @@ adapt_new_options() {
             ;;
         ACME_DNS_PROVIDER)
             echo '# DNS provider for DNS-01 challenge (e.g. dns_cf, dns_azure, dns_gd, etc.)' >> mailcow.conf
-            echo '# See the dns-101 provider documentation for more information.' >> mailcow.conf
+            echo '# See the dns-01 provider documentation for more information.' >> mailcow.conf
             echo 'ACME_DNS_PROVIDER=dns_xxx' >> mailcow.conf
             ;;
         ACME_ACCOUNT_EMAIL)

data/Dockerfiles/acme/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/acme/acme.sh

@@ -249,14 +249,45 @@ while true; do
   done <<< "${SQL_DOMAINS}"
 
   if [[ ${ONLY_MAILCOW_HOSTNAME} != "y" ]]; then
+  # Fetch all domains with an active MTA-STS policy once.
+  unset MTA_STS_ACTIVE_DOMAINS
+  declare -A MTA_STS_ACTIVE_DOMAINS
+  if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
+    SQL_MTA_STS_DOMAINS=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM mta_sts WHERE active = 1" -Bs)
+    if [[ $? -eq 0 ]]; then
+      while read mta_sts_domain; do
+        if [[ -z "${mta_sts_domain}" ]]; then
+          # ignore empty lines
+          continue
+        fi
+        MTA_STS_ACTIVE_DOMAINS["${mta_sts_domain}"]=1
+      done <<< "${SQL_MTA_STS_DOMAINS}"
+    fi
+  fi
   for SQL_DOMAIN in "${SQL_DOMAIN_ARR[@]}"; do
     unset VALIDATED_CONFIG_DOMAINS_SUBDOMAINS
     declare -a VALIDATED_CONFIG_DOMAINS_SUBDOMAINS
     for SUBDOMAIN in "${ADDITIONAL_WC_ARR[@]}"; do
-      if [[  "${SUBDOMAIN}.${SQL_DOMAIN}" != "${MAILCOW_HOSTNAME}" ]]; then
-        if check_domain "${SUBDOMAIN}.${SQL_DOMAIN}"; then
-          VALIDATED_CONFIG_DOMAINS_SUBDOMAINS+=("${SUBDOMAIN}.${SQL_DOMAIN}")
-        fi
+      FULL_SUBDOMAIN="${SUBDOMAIN}.${SQL_DOMAIN}"
+
+      # Skip mta-sts subdomain unless MTA-STS is enabled (active) for this domain
+      if [[ "${SUBDOMAIN}" == "mta-sts" && -z "${MTA_STS_ACTIVE_DOMAINS[${SQL_DOMAIN}]}" ]]; then
+        log_f "MTA-STS is not enabled for ${SQL_DOMAIN} - skipping mta-sts subdomain certificate"
+        continue
+      fi
+
+      # Skip if subdomain matches MAILCOW_HOSTNAME
+      if [[ "${FULL_SUBDOMAIN}" == "${MAILCOW_HOSTNAME}" ]]; then
+        continue
+      fi
+      # Skip if subdomain is covered by a wildcard in ADDITIONAL_SAN
+      if is_covered_by_wildcard "${FULL_SUBDOMAIN}"; then
+        log_f "Subdomain '${FULL_SUBDOMAIN}' is covered by wildcard - skipping explicit subdomain"
+        continue
+      fi
+      # Validate and add subdomain
+      if check_domain "${FULL_SUBDOMAIN}"; then
+        VALIDATED_CONFIG_DOMAINS_SUBDOMAINS+=("${FULL_SUBDOMAIN}")
       fi
     done
     VALIDATED_CONFIG_DOMAINS+=("${VALIDATED_CONFIG_DOMAINS_SUBDOMAINS[*]}")
@@ -273,7 +304,10 @@ while true; do
         fi
         # Only add mta-sts subdomain for alias domains
         if [[ "mta-sts.${alias_domain}" != "${MAILCOW_HOSTNAME}" ]]; then
-          if check_domain "mta-sts.${alias_domain}"; then
+          # Skip if mta-sts subdomain is covered by a wildcard
+          if is_covered_by_wildcard "mta-sts.${alias_domain}"; then
+            log_f "Alias domain mta-sts subdomain 'mta-sts.${alias_domain}' is covered by wildcard - skipping"
+          elif check_domain "mta-sts.${alias_domain}"; then
             VALIDATED_CONFIG_DOMAINS+=("mta-sts.${alias_domain}")
           fi
         fi
@@ -308,13 +342,31 @@ while true; do
   done
   fi
 
+  # Check if MAILCOW_HOSTNAME is covered by a wildcard in ADDITIONAL_SAN
+  MAILCOW_HOSTNAME_COVERED=0
+  if [[ ! -z ${VALIDATED_MAILCOW_HOSTNAME} ]]; then
+    if is_covered_by_wildcard "${VALIDATED_MAILCOW_HOSTNAME}"; then
+      MAILCOW_PARENT_DOMAIN=$(echo ${VALIDATED_MAILCOW_HOSTNAME} | cut -d. -f2-)
+      log_f "MAILCOW_HOSTNAME '${VALIDATED_MAILCOW_HOSTNAME}' is covered by wildcard '*.${MAILCOW_PARENT_DOMAIN}' - skipping explicit hostname"
+      MAILCOW_HOSTNAME_COVERED=1
+    fi
+  fi
+
   # Unique domains for server certificate
   if [[ ${ENABLE_SSL_SNI} == "y" ]]; then
     # create certificate for server name and fqdn SANs only
-    SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    if [[ ${MAILCOW_HOSTNAME_COVERED} == "1" ]]; then
+      SERVER_SAN_VALIDATED=($(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    else
+      SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    fi
   else
     # create certificate for all domains, including all subdomains from other domains [*]
-    SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    if [[ ${MAILCOW_HOSTNAME_COVERED} == "1" ]]; then
+      SERVER_SAN_VALIDATED=($(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    else
+      SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    fi
   fi
   if [[ ! -z ${SERVER_SAN_VALIDATED[*]} ]]; then
     CERT_NAME=${SERVER_SAN_VALIDATED[0]}

data/Dockerfiles/acme/functions.sh

@@ -135,3 +135,32 @@ verify_challenge_path(){
     return 1
   fi
 }
+
+# Check if a domain is covered by a wildcard (*.example.com) in ADDITIONAL_SAN
+# Usage: is_covered_by_wildcard "subdomain.example.com"
+# Returns: 0 if covered, 1 if not covered
+# Note: Only returns 0 (covered) when DNS-01 challenge is enabled,
+#       as wildcards cannot be validated with HTTP-01 challenge
+is_covered_by_wildcard() {
+  local DOMAIN=$1
+
+  # Only skip if DNS challenge is enabled (wildcards require DNS-01)
+  if [[ ${ACME_DNS_CHALLENGE} != "y" ]]; then
+    return 1
+  fi
+
+  # Return early if no ADDITIONAL_SAN is set
+  if [[ -z ${ADDITIONAL_SAN} ]]; then
+    return 1
+  fi
+
+  # Extract parent domain (e.g., mail.example.com -> example.com)
+  local PARENT_DOMAIN=$(echo ${DOMAIN} | cut -d. -f2-)
+
+  # Check if ADDITIONAL_SAN contains a wildcard for this parent domain
+  if [[ "${ADDITIONAL_SAN}" == *"*.${PARENT_DOMAIN}"* ]]; then
+    return 0  # Covered by wildcard
+  fi
+
+  return 1  # Not covered
+}

data/Dockerfiles/acme/load-dns-config.sh

@@ -7,7 +7,7 @@ else
   __dns_loader_standalone=0
 fi
 
-CONFIG_PATH="${ACME_DNS_CONFIG_FILE:-/etc/acme/dns-101.conf}"
+CONFIG_PATH="${ACME_DNS_CONFIG_FILE:-/etc/acme/dns-01.conf}"
 
 if [[ ! -f "${CONFIG_PATH}" ]]; then
   if [[ $__dns_loader_standalone -eq 1 ]]; then

data/Dockerfiles/acme/obtain-certificate-dns.sh

@@ -12,7 +12,7 @@ CERT_DOMAINS=(${DOMAINS[@]})
 CERT_DOMAIN=${CERT_DOMAINS[0]}
 ACME_BASE=/var/lib/acme
 
-# Load optional DNS provider secrets from /etc/acme/dns-101.conf
+# Load optional DNS provider secrets from /etc/acme/dns-01.conf
 if [[ -f /srv/load-dns-config.sh ]]; then
   source /srv/load-dns-config.sh
   if declare -F log_f >/dev/null; then

data/Dockerfiles/clamd/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.21 AS builder
+FROM alpine:3.24 AS builder
 
 WORKDIR /src
-ENV CLAMD_VERSION=1.4.2
+ENV CLAMD_VERSION=1.4.6
 
 RUN apk upgrade --no-cache \
   && apk add --update --no-cache \
@@ -68,7 +68,7 @@ RUN wget -P /src https://www.clamav.net/downloads/production/clamav-${CLAMD_VERS
   "/clamav/etc/clamav/clamav-milter.conf.sample" > "/clamav/etc/clamav/clamav-milter.conf" || exit 1
 
 
-FROM alpine:3.21
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -412,6 +412,63 @@ class DockerApi:
 
         sogo_return = container.exec_run(["/bin/bash", "-c", f"sogo-tool rename-user '{old_username}' '{new_username}'"], user='sogo')
         return self.exec_run_handler('generic', sogo_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: subscribe
+  # Subscribes users to a folder of the owner, or unsubscribes them; values are passed as environment and
+  # arguments, never through the shell
+  def container_post__exec__sogo__subscribe(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      folder = str(request_json.get('folder', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or not re.match(r'^(Calendar|Contacts)/[A-Za-z0-9._-]+$', folder):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner, folder and users required' }, indent=4), media_type="application/json")
+      script = 'for user in "$@"; do sogo-tool manage-acl "$SOGO_ACTION" "$SOGO_OWNER" "$SOGO_FOLDER" "$user" || exit 1; done'
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe"] + users, user='sogo',
+        environment={ 'SOGO_ACTION': 'unsubscribe' if request_json.get('unsubscribe') else 'subscribe', 'SOGO_OWNER': owner, 'SOGO_FOLDER': folder })
+      return self.exec_run_handler('generic', subscribe_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: init_user
+  # Creates the personal folders of users that never logged in to SOGo (e.g. shared mailboxes)
+  def container_post__exec__sogo__init_user(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not users:
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
+      init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
+      return self.exec_run_handler('generic', init_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
+  # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
+  def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or any(c in owner for c in '/*%'):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner and users required' }, indent=4), media_type="application/json")
+      if request_json.get('unsubscribe'):
+        # Without access the user no longer sees the folders, their names come from the owner
+        script = ('set -o pipefail; for user in "$@"; do doveadm mailbox list -u "$SUB_OWNER" | while IFS= read -r folder; do '
+          'case "$folder" in Shared|Shared/*) continue;; esac; doveadm mailbox unsubscribe -u "$user" "Shared/$SUB_OWNER/$folder" || exit 1; '
+          'done || exit 1; done')
+      else:
+        # No visible folder means no access yet (ACL, auth cache): fail, so the subscription is retried
+        script = ('for user in "$@"; do folders=$(doveadm mailbox list -u "$user" "Shared/$SUB_OWNER/*") && [ -n "$folders" ] || exit 1; '
+          'while IFS= read -r folder; do doveadm mailbox subscribe -u "$user" "$folder" || exit 1; done <<< "$folders"; done')
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe_shared"] + users, environment={ 'SUB_OWNER': owner })
+      return self.exec_run_handler('generic', subscribe_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: get_acl
   def container_post__exec__doveadm__get_acl(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -439,7 +496,8 @@ class DockerApi:
           if len(acls) >= 2:
             for acl in acls[1:]:
               user_id, rights = acl.split(maxsplit=1)
-              user_id = user_id.split('=')[1]
+              # user=<name> as plain name, other identifiers (group=...) keep their type
+              user_id = user_id.split('=', 1)[1] if user_id.startswith('user=') else user_id
               mailbox_seen.append(mailbox)
               formatted_acls.append({ 'user': id, 'id': user_id, 'mailbox': mailbox, 'rights': rights.split() })
         elif "Shared" in shared_folder and "/" in shared_folder:
@@ -476,7 +534,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' 'user={id}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -510,8 +568,50 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' 'user={id}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
+  # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
+  # environment, never through the shell
+  def container_post__exec__doveadm__set_acl_all(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      user = request_json.get('user', '')
+      id = self.acl_identifier(request_json.get('id', ''))
+      rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
+      if not user or id in ('user=', 'group='):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
+      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
+      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
+        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
+      acl_return = container.exec_run(["/bin/bash", "-c", script],
+        environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
+      return self.exec_run_handler('generic', acl_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: auth_cache_flush
+  # Drops cached userdb results (e.g. acl_groups) of the given users, all users if none are given
+  def container_post__exec__doveadm__auth_cache_flush(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
+      return self.exec_run_handler('generic', flush_return)
+
+  # Rights doveadm acl set accepts from the API
+  acl_rights = ["admin", "create", "delete", "expunge", "insert", "lookup", "post", "read", "write", "write-deleted", "write-seen"]
+
+  # ACL identifier for doveadm: "user=<name>" for a plain name, "user=..." / "group=..." as given
+  def acl_identifier(self, id):
+    if id.startswith('user=') or id.startswith('group='):
+      return id
+    return 'user=' + id
 
 
   # Collect host stats

data/Dockerfiles/dovecot/Dockerfile

@@ -1,12 +1,48 @@
-FROM alpine:3.22
+# Alpine >= 3.22 only ships Dovecot 2.4, Dovecot 2.3 is built from the Alpine 3.21 packaging
+FROM alpine:3.24 AS builder
+
+# Last commit of the aports 3.21-stable branch
+ARG APORTS_COMMIT=55126a11a4b9dcd14de3068e3348e9951d144ca6
+
+RUN apk add --no-cache alpine-sdk git \
+  && git init -q /aports \
+  && cd /aports \
+  && git remote add origin https://github.com/alpinelinux/aports.git \
+  && git sparse-checkout set main/dovecot community/dovecot-fts-flatcurve \
+  && git fetch -q --depth 1 --filter=blob:none origin ${APORTS_COMMIT} \
+  && git checkout -q FETCH_HEAD \
+  && abuild-keygen -a -n \
+  && cp /root/.abuild/*.rsa.pub /etc/apk/keys/ \
+  && apk update \
+  && cd /aports/main/dovecot \
+  # newer gettext does not overwrite the shipped config.rpath without -f
+  && sed -i 's/autoreconf -vi/autoreconf -fvi/' APKBUILD \
+  && abuild -F -r \
+  # flatcurve must be built against the Dovecot 2.3 headers
+  && apk add --no-cache /root/packages/main/*/dovecot-2*.apk /root/packages/main/*/dovecot-dev-*.apk \
+  && cd /aports/community/dovecot-fts-flatcurve \
+  && abuild -F -r \
+  && mkdir /packages \
+  && cp /root/.abuild/*.rsa.pub /packages/ \
+  && for pkg in dovecot dovecot-dev dovecot-lmtpd dovecot-lua dovecot-ldap dovecot-mysql dovecot-sql \
+    dovecot-submissiond dovecot-pigeonhole-plugin dovecot-pop3d dovecot-fts-flatcurve; do \
+    cp /root/packages/*/*/${pkg}-[0-9]*.apk /packages/; \
+  done
+
+
+FROM alpine:3.24
 
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.19
+ARG GOSU_VERSION=1.20
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
+# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, this covers Perl DBI, client.cnf covers Dovecot
+ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
+
+COPY --from=builder /packages /tmp/packages
 
 # Add groups and users before installing Dovecot to not break compatibility
 RUN addgroup -g 5000 vmail \
@@ -18,6 +54,7 @@ RUN addgroup -g 5000 vmail \
   && adduser -D -u 5000 -G vmail -h /var/vmail vmail \
   && adduser -D -G dovecot -u 401 -h /dev/null -s /sbin/nologin dovecot \
   && adduser -D -G dovenull -u 402 -h /dev/null -s /sbin/nologin dovenull \
+  && cp /tmp/packages/*.rsa.pub /etc/apk/keys/ \
   && apk add --no-cache --update \
   bash \
   bind-tools \
@@ -68,7 +105,6 @@ RUN addgroup -g 5000 vmail \
   perl-package-stash-xs \
   perl-par-packer \
   perl-parse-recdescent \
-  perl-lockfile-simple \
   libproc2 \
   perl-readonly \
   perl-regexp-common \
@@ -99,28 +135,21 @@ RUN addgroup -g 5000 vmail \
   supervisor \
   tzdata \
   wget \
-  dovecot \
-  dovecot-dev \
-  dovecot-lmtpd \
-  dovecot-lua \
-  dovecot-ldap \
-  dovecot-mysql \
-  dovecot-sql \
-  dovecot-submissiond \
-  dovecot-pigeonhole-plugin \
-  dovecot-pop3d \
-  dovecot-fts-flatcurve \
+  /tmp/packages/*.apk \
+  && rm -rf /tmp/packages \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
   && chmod +x /usr/local/bin/gosu \
   && gosu nobody true
 
+# Dovecot's MySQL driver reads [client] from /etc/my.cnf, Connector/C ignores the !includedir there
+COPY client.cnf /etc/my.cnf
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
-COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
+COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/client.cnf

@@ -0,0 +1,3 @@
+[client]
+ssl = false
+ssl-verify-server-cert = false

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -44,109 +44,91 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
 else
   QUOTA_TABLE=quota2replica
 fi
-
-cat <<EOF > /etc/dovecot/conf.d/12-mysql.conf
-# Autogenerated by mailcow - DO NOT TOUCH!
-mysql /var/run/mysqld/mysqld.sock {
-  dbname=${DBNAME}
-  user=${DBUSER}
-  password=${DBPASS}
-
-  ssl = no
-}
-EOF
-
-
 cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-quota.conf
 # Autogenerated by mailcow
-dict_map priv/quota/storage {
-  sql_table = ${QUOTA_TABLE}
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+map {
+  pattern = priv/quota/storage
+  table = ${QUOTA_TABLE}
   username_field = username
-  value_field bytes {
-  }
+  value_field = bytes
 }
-
-dict_map priv/quota/messages {
-  sql_table = ${QUOTA_TABLE}
+map {
+  pattern = priv/quota/messages
+  table = ${QUOTA_TABLE}
   username_field = username
-  value_field messages {
-  }
+  value_field = messages
 }
 EOF
 
 # Create dict used for sieve pre and postfilters
 cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
 # Autogenerated by mailcow
-
-dict_map priv/sieve/name/\$script_name {
-  sql_table = sieve_before
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+map {
+  pattern = priv/sieve/name/\$script_name
+  table = sieve_before
   username_field = username
-  value_field id {
-  }
-
-  # The script name field in the table to query
-  key_field script_name {
-    value = \$script_name
+  value_field = id
+  fields {
+    script_name = \$script_name
   }
 }
-
-dict_map priv/sieve/data/\$id {
-  sql_table = sieve_before
+map {
+  pattern = priv/sieve/data/\$id
+  table = sieve_before
   username_field = username
-  value_field script_data {
-  }
-  key_field id {
-    value = \$id
+  value_field = script_data
+  fields {
+    id = \$id
   }
 }
 EOF
 
 cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf
 # Autogenerated by mailcow
-
-dict_map priv/sieve/name/\$script_name {
-  sql_table = sieve_after
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+map {
+  pattern = priv/sieve/name/\$script_name
+  table = sieve_after
   username_field = username
-  value_field id {
-  }
-  key_field script_name {
-    value = \$script_name
+  value_field = id
+  fields {
+    script_name = \$script_name
   }
 }
-
-dict_map priv/sieve/data/\$id {
-  sql_table = sieve_after
+map {
+  pattern = priv/sieve/data/\$id
+  table = sieve_after
   username_field = username
-  value_field script_data {
-  }
-  key_field id {
-    value = \$id
+  value_field = script_data
+  fields {
+    id = \$id
   }
 }
 EOF
 
-if [[ "${ACL_ANYONE}" == "allow" ]]; then
-  echo -n "yes" > /etc/dovecot/acl_anyone
-else
-  echo -n "no" > /etc/dovecot/acl_anyone
-fi
+echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
 if [[ "${SKIP_FTS}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
 echo -e "\e[33mDetecting SKIP_FTS=y... not enabling Flatcurve (FTS) then...\e[0m"
-echo -n 'quota quota_clone acl mail_crypt mail_crypt_acl mail_log mail_compress notify lazy_expunge' > /etc/dovecot/mail_plugins
-echo -n 'quota quota_clone imap_quota imap_acl acl imap_sieve mail_crypt mail_crypt_acl mail_compress notify mail_log' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota quota_clone sieve acl mail_crypt mail_crypt_acl mail_compress notify' > /etc/dovecot/mail_plugins_lmtp
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 else
 echo -e "\e[32mDetecting SKIP_FTS=n... enabling Flatcurve (FTS)\e[0m"
-echo -n 'quota quota_clone acl mail_crypt mail_crypt_acl mail_log mail_compress notify fts fts_flatcurve lazy_expunge' > /etc/dovecot/mail_plugins
-echo -n 'quota quota_clone imap_quota imap_acl acl imap_sieve mail_crypt mail_crypt_acl mail_compress notify mail_log fts fts_flatcurve' > /etc/dovecot/mail_plugins_imap
-echo -n 'quota quota_clone sieve acl mail_crypt mail_crypt_acl mail_compress fts fts_flatcurve notify' > /etc/dovecot/mail_plugins_lmtp
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 fi
 chmod 644 /etc/dovecot/mail_plugins /etc/dovecot/mail_plugins_imap /etc/dovecot/mail_plugins_lmtp /templates/quarantine.tpl
 
 cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 # Autogenerated by mailcow
-query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%{user | domain }}/%{user | username }/Maildir:VOLATILEDIR=/var/volatile/%{user}:INDEX=/var/vmail_index/%{user}') AS mail, '%{protocol}' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%{user}' AND (active = '1' OR active = '2')
+driver = mysql
+connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
+# acl_groups: active mailbox groups of the user, IMAP grants are ACLs on group=mcg_<id>
+user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule, (SELECT GROUP_CONCAT(CONCAT('mcg_', gm.group_id)) FROM mailbox_group_member gm JOIN mailbox_group g ON g.id = gm.group_id AND g.active = 1 WHERE gm.username = mailbox.username) AS acl_groups FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
 iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
 EOF
 
@@ -177,8 +159,8 @@ for cert_dir in /etc/ssl/mail/*/ ; do
   domains=($(cat ${cert_dir}domains))
   for domain in ${domains[@]}; do
     echo 'local_name '${domain}' {' >> /etc/dovecot/sni.conf;
-    echo '  ssl_server_cert_file = '${cert_dir}'cert.pem' >> /etc/dovecot/sni.conf;
-    echo '  ssl_server_key_file = '${cert_dir}'key.pem' >> /etc/dovecot/sni.conf;
+    echo '  ssl_cert = <'${cert_dir}'cert.pem' >> /etc/dovecot/sni.conf;
+    echo '  ssl_key = <'${cert_dir}'key.pem' >> /etc/dovecot/sni.conf;
     echo '}' >> /etc/dovecot/sni.conf;
   done
 done
@@ -202,13 +184,11 @@ else
 fi
 cat <<EOF > /etc/dovecot/shared_namespace.conf
 # Autogenerated by mailcow
-namespace shared {
+namespace {
     type = shared
     separator = /
-    prefix = Shared/\$user/
-    mail_driver = maildir
-    mail_path = %{owner_home}${MAILDIR_SUB_SHARED}
-    mail_index_private_path = ~${MAILDIR_SUB_SHARED}/Shared/%{owner_user}
+    prefix = Shared/%%u/
+    location = maildir:%%h${MAILDIR_SUB_SHARED}:INDEX=~${MAILDIR_SUB_SHARED}/Shared/%%u
     subscriptions = no
     list = children
 }
@@ -218,7 +198,7 @@ EOF
 cat <<EOF > /etc/dovecot/sogo_trusted_ip.conf
 # Autogenerated by mailcow
 remote ${IPV4_NETWORK}.248 {
-  auth_allow_cleartext = yes
+  disable_plaintext_auth = no
 }
 EOF
 
@@ -227,13 +207,9 @@ RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
 echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
 cat <<EOF > /etc/dovecot/sogo-sso.conf
 # Autogenerated by mailcow
-passdb static {
-  fields {
-    allow_real_nets=${IPV4_NETWORK}.248/32
-  }
-
-  password={plain}${RAND_PASS}
-  
+passdb {
+  driver = static
+  args = allow_nets=${IPV4_NETWORK}.248/32 password={plain}${RAND_PASS}
 }
 EOF
 
@@ -261,9 +237,9 @@ fi
 if [[ "${SKIP_FTS}" =~ ^([nN][oO]|[nN])+$ ]]; then
   echo -e "\e[94mConfiguring FTS Settings...\e[0m"
   echo -e "\e[94mSetting FTS Memory Limit (per process) to ${FTS_HEAP} MB\e[0m"
-  sed -i "s/vsz_limit\s*=\s*[0-9]*\s*MB*/vsz_limit=${FTS_HEAP} MB/" /etc/dovecot/conf.d/35-fts.conf
+  sed -i "s/vsz_limit\s*=\s*[0-9]*\s*MB*/vsz_limit=${FTS_HEAP} MB/" /etc/dovecot/conf.d/fts.conf
   echo -e "\e[94mSetting FTS Process Limit to ${FTS_PROCS}\e[0m"
-  sed -i "s/process_limit\s*=\s*[0-9]*/process_limit=${FTS_PROCS}/" /etc/dovecot/conf.d/35-fts.conf
+  sed -i "s/process_limit\s*=\s*[0-9]*/process_limit=${FTS_PROCS}/" /etc/dovecot/conf.d/fts.conf
 fi
 
 # 401 is user dovecot
@@ -275,16 +251,16 @@ else
 	chown 401 /mail_crypt/ecprivkey.pem /mail_crypt/ecpubkey.pem
 fi
 
-# # Fix OpenSSL 3.X TLS1.0, 1.1 support (https://community.mailcow.email/d/4062-hi-all/20)
-# if grep -qE 'ssl_min_protocol\s*=\s*(TLSv1|TLSv1\.1)\s*$' /etc/dovecot/dovecot.conf /etc/dovecot/extra.conf; then
-#     sed -i '/\[openssl_init\]/a ssl_conf = ssl_configuration' /etc/ssl/openssl.cnf
+# Fix OpenSSL 3.X TLS1.0, 1.1 support (https://community.mailcow.email/d/4062-hi-all/20)
+if grep -qE 'ssl_min_protocol\s*=\s*(TLSv1|TLSv1\.1)\s*$' /etc/dovecot/dovecot.conf /etc/dovecot/extra.conf; then
+    sed -i '/\[openssl_init\]/a ssl_conf = ssl_configuration' /etc/ssl/openssl.cnf
 
-#     echo "[ssl_configuration]" >> /etc/ssl/openssl.cnf
-#     echo "system_default = tls_system_default" >> /etc/ssl/openssl.cnf
-#     echo "[tls_system_default]" >> /etc/ssl/openssl.cnf
-#     echo "MinProtocol = TLSv1" >> /etc/ssl/openssl.cnf
-#     echo "CipherString = DEFAULT@SECLEVEL=0" >> /etc/ssl/openssl.cnf
-# fi
+    echo "[ssl_configuration]" >> /etc/ssl/openssl.cnf
+    echo "system_default = tls_system_default" >> /etc/ssl/openssl.cnf
+    echo "[tls_system_default]" >> /etc/ssl/openssl.cnf
+    echo "MinProtocol = TLSv1" >> /etc/ssl/openssl.cnf
+    echo "CipherString = DEFAULT@SECLEVEL=0" >> /etc/ssl/openssl.cnf
+fi
 
 # Compile sieve scripts
 sievec /var/vmail/sieve/global_sieve_before.sieve
@@ -304,7 +280,7 @@ chmod g+rw /dev/console
 chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_runner.pl \
+  /usr/local/bin/imapsync_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -329,6 +305,15 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 # Envsubst maildir_gc
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+done
+echo "DB schema is ${DBV_NOW}"
+
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do
   if [[ ! -z $(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"${DBNAME}\" AND TABLE_NAME = 'versions'") ]]; then

data/Dockerfiles/dovecot/imapsync_runner.pl

@@ -1,196 +0,0 @@
-#!/usr/bin/perl
-
-use DBI;
-use LockFile::Simple qw(lock trylock unlock);
-use Proc::ProcessTable;
-use Data::Dumper qw(Dumper);
-use IPC::Run 'run';
-use File::Temp;
-use Try::Tiny;
-use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT);
-
-sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
-my $t = Proc::ProcessTable->new;
-my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table};
-if ($imapsync_running ge 1)
-{
-  print "imapsync is active, exiting...";
-  exit;
-}
-
-sub qqw($) {
-  my @params = ();
-  my @values = split(/(?=--)/, $_[0]);
-  foreach my $val (@values) {
-    my @tmpparam = split(/ /, $val, 2);
-    foreach my $tmpval (@tmpparam) {
-        if ($tmpval ne '') {
-          push @params, $tmpval;
-        }
-    }
-  }
-  foreach my $val (@params) {
-    $val=trim($val);
-  }
-  return @params;
-}
-
-$run_dir="/tmp";
-$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
-$lock_file = $run_dir . "/imapsync_busy";
-$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
-$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
-  mysql_auto_reconnect => 1,
-  mysql_enable_utf8mb4 => 1
-});
-$dbh->do("UPDATE imapsync SET is_running = 0");
-
-sub sig_handler {
-  # Send die to force exception in "run"
-  die "sig_handler received signal, preparing to exit...\n";
-};
-
-open my $file, '<', "/etc/sogo/sieve.creds"; 
-my $creds = <$file>; 
-close $file;
-my ($master_user, $master_pass) = split /:/, $creds;
-my $sth = $dbh->prepare("SELECT id,
-  user1,
-  user2,
-  host1,
-  authmech1,
-  password1,
-  exclude,
-  port1,
-  enc1,
-  delete2duplicates,
-  maxage,
-  subfolder2,
-  delete1,
-  delete2,
-  automap,
-  skipcrossduplicates,
-  maxbytespersecond,
-  custom_params,
-  subscribeall,
-  timeout1,
-  timeout2,
-  dry
-    FROM imapsync
-      WHERE active = 1
-        AND is_running = 0
-        AND (
-          UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
-          OR
-          last_run IS NULL)
-  ORDER BY last_run");
-
-$sth->execute();
-my $row;
-
-while ($row = $sth->fetchrow_arrayref()) {
-
-  $id                  = @$row[0];
-  $user1               = @$row[1];
-  $user2               = @$row[2];
-  $host1               = @$row[3];
-  $authmech1           = @$row[4];
-  $password1           = @$row[5];
-  $exclude             = @$row[6];
-  $port1               = @$row[7];
-  $enc1                = @$row[8];
-  $delete2duplicates   = @$row[9];
-  $maxage              = @$row[10];
-  $subfolder2          = @$row[11];
-  $delete1             = @$row[12];
-  $delete2             = @$row[13];
-  $automap             = @$row[14];
-  $skipcrossduplicates = @$row[15];
-  $maxbytespersecond   = @$row[16];
-  $custom_params       = @$row[17];
-  $subscribeall        = @$row[18];
-  $timeout1            = @$row[19];
-  $timeout2            = @$row[20];
-  $dry                 = @$row[21];
-
-  if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
-
-  my $template = $run_dir . '/imapsync.XXXXXXX';
-  my $passfile1 = File::Temp->new(TEMPLATE => $template);
-  my $passfile2 = File::Temp->new(TEMPLATE => $template);
-  
-  binmode( $passfile1, ":utf8" );
-  
-  print $passfile1 "$password1\n";
-  print $passfile2 trim($master_pass) . "\n";
-
-  my @custom_params_a = qqw($custom_params);
-  my $custom_params_ref = \@custom_params_a;
-
-  my $generated_cmds = [ "/usr/local/bin/imapsync",
-  "--tmpdir", "/tmp",
-  "--nofoldersizes",
-  "--addheader",
-  ($timeout1 le "0" ? () : ('--timeout1', $timeout1)),
-  ($timeout2 le "0" ? () : ('--timeout2', $timeout2)),
-  ($exclude eq "" ? () : ("--exclude", $exclude)),
-  ($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)),
-  ($maxage eq "0" ? () : ('--maxage', $maxage)),
-  ($maxbytespersecond eq "0" ? () : ('--maxbytespersecond', $maxbytespersecond)),
-  ($delete2duplicates ne "1" ? () : ('--delete2duplicates')),
-  ($subscribeall  ne "1" ? () : ('--subscribeall')),
-  ($delete1 ne "1" ? () : ('--delete')),
-  ($delete2 ne "1" ? () : ('--delete2')),
-  ($automap ne "1" ? () : ('--automap')),
-  ($skipcrossduplicates ne "1" ? () : ('--skipcrossduplicates')),
-  (!defined($enc1) ? () : ($enc1)),
-  "--host1", $host1,
-  "--user1", $user1,
-  "--passfile1", $passfile1->filename,
-  "--port1", $port1,
-  "--host2", "localhost",
-  "--user2", $user2 . '*' . trim($master_user),
-  "--passfile2", $passfile2->filename,
-  ($dry eq "1" ? ('--dry') : ()),
-  '--no-modulesversion',
-  '--noreleasecheck'];
-
-  try {
-    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
-    $is_running->bind_param( 1, ${id} );
-    $is_running->execute();
-
-    run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
-
-    # check exit code and status
-    ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
-
-    $success = 0;
-    if (defined $exit_code && $exit_code == 0) {
-      $success = 1;
-    }
-
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
-    $update->bind_param( 1, ${stdout} );
-    $update->bind_param( 2, ${success} );
-    $update->bind_param( 3, ${exit_status} );
-    $update->bind_param( 4, ${id} );
-    $update->execute();
-  } catch {
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
-    $update->execute();
-  } finally {
-    $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
-    $update->execute();
-  };
-
-
-}
-
-$sth->finish();
-$dbh->disconnect();
-
-$lockmgr->unlock($lock_file);

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -0,0 +1,331 @@
+#!/usr/bin/python3
+
+import fcntl
+import json
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import MySQLdb
+import MySQLdb.cursors
+
+run_dir = '/tmp'
+# Max size of imapsync output stored in returned_text (the tail holds the summary)
+max_output = 16 * 1024 * 1024
+lock_file = run_dir + '/imapsync_busy.lock'
+# An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
+token_min_valid = 900
+# Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
+token_sync_interval = 60
+
+# Next due job: higher prio first, never-run jobs first, then oldest run
+next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
+  i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
+  i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
+  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
+    FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
+      WHERE i.active = 1
+        AND s.active = 1
+        AND i.is_running = 0
+        AND (UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(i.last_run) > i.mins_interval * 60 OR i.last_run IS NULL)
+        {skip}
+  ORDER BY i.prio DESC, i.last_run IS NOT NULL, i.last_run ASC, i.id ASC
+  LIMIT 1"""
+# Atomic claim, a job is only started when this flips is_running 0 -> 1
+claim_sql = """UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL
+  WHERE id = %s AND is_running = 0 AND active = 1"""
+# Access token: client_credentials share the source token, authorization_code use the token of
+# source + local mailbox (user2) + remote user (user1). issued/error describe the last refresh.
+source_token_sql = """SELECT oauth_access_token AS access_token, oauth_token_expires AS token_expires,
+  UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued, oauth_last_refresh_error AS error
+  FROM imapsync_source WHERE id = %s"""
+user_token_sql = """SELECT access_token, token_expires, UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued,
+  last_refresh_error AS error FROM imapsync_source_oauth_token
+  WHERE source_id = %s AND owner = %s AND username = %s"""
+
+conn = None
+# Popen => job of running workers
+running = {}
+# Jobs released without a run (token not ready), skipped for the rest of this runner round
+deferred = set()
+stopping = False
+
+def num(value):
+  try:
+    return int(value)
+  except (TypeError, ValueError):
+    return 0
+
+def unlink(files):
+  for path in files:
+    try:
+      os.unlink(path)
+    except OSError:
+      pass
+
+def imapsync_running():
+  for pid in filter(str.isdigit, os.listdir('/proc')):
+    try:
+      with open('/proc/' + pid + '/cmdline', 'rb') as f:
+        cmdline = f.read().replace(b'\0', b' ')
+    except OSError:
+      continue
+    if re.search(rb'imapsync\s', cmdline, re.I):
+      return True
+  return False
+
+def connect():
+  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
+    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
+
+# Reconnect when the server dropped the connection (syncs can outlast wait_timeout)
+def query(sql, args=None):
+  global conn
+  try:
+    conn.ping()
+  except MySQLdb.Error:
+    conn = connect()
+  cur = conn.cursor(MySQLdb.cursors.DictCursor)
+  cur.execute(sql, args)
+  return cur
+
+# Store a result and release the job
+def finish_job(job_id, text, success, exit_status):
+  try:
+    query("UPDATE imapsync SET returned_text = %s, success = %s, exit_status = %s, last_run = NOW(), is_running = 0 WHERE id = %s",
+      (text, success, exit_status, job_id))
+  except MySQLdb.Error:
+    query("UPDATE imapsync SET success = 0, last_run = NOW(), is_running = 0 WHERE id = %s", (job_id,))
+
+# Release a job without a run; last_run stays, so the next runner round retries it
+def defer_job(job_id, text, exit_status):
+  query("UPDATE imapsync SET returned_text = %s, success = 0, exit_status = %s, is_running = 0 WHERE id = %s",
+    (text, exit_status, job_id))
+  deferred.add(job_id)
+
+def oauth_token(j):
+  if j['oauth_flow'] == 'authorization_code':
+    return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
+  return query(source_token_sql, (j['source_id'],)).fetchone() or {}
+
+# Valid for token_min_valid, or half the lifetime of a short-lived token (known when its last refresh succeeded)
+def token_ready(row):
+  token, expires, issued = row.get('access_token'), row.get('token_expires'), row.get('issued')
+  if not token or expires is None:
+    return False
+  need = token_min_valid
+  if issued and not row.get('error') and expires > int(issued):
+    need = min(need, (expires - int(issued)) // 2)
+  return expires - time.time() >= need
+
+# Write a refreshed token over the token file of a running job (atomic, same path)
+def sync_token_file(oauth):
+  token = oauth_token(oauth['job']).get('access_token')
+  if not token or token == oauth['token']:
+    return
+  fd, tmp = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
+  try:
+    with os.fdopen(fd, 'w', encoding='utf-8') as f:
+      f.write(token + '\n')
+    os.replace(tmp, oauth['file'])
+    oauth['token'] = token
+  except OSError:
+    unlink([tmp])
+
+# Max concurrent imapsync processes and global per-process bandwidth cap in bytes/s (0 = off),
+# re-read before every claim round so changes apply to long runs
+def read_settings():
+  settings = {r['name']: r['value'] for r in query("SELECT name, value FROM imapsync_settings")}
+  return max(num(settings.get('max_parallel')), 1), max(num(settings.get('max_bytes_per_second')), 0)
+
+# Build temp files and argv for a claimed job. Returns None when the job was finished without running imapsync.
+def prepare_job(j, global_bw, files):
+  def tmpfile(content):
+    fd, path = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
+    files.append(path)
+    with os.fdopen(fd, 'w', encoding='utf-8') as f:
+      f.write(content)
+    return path
+
+  auth_type = j['auth_type'] or 'PLAIN'
+  oauth = None
+  if auth_type == 'XOAUTH2':
+    row = oauth_token(j)
+    if not token_ready(row):
+      defer_job(j['id'], "OAuth access token for the configured source is missing or expires too soon; refresh cron will retry.",
+        'OAUTH_TOKEN_NOT_READY')
+      return None
+    oauth = {'job': j, 'token': row['access_token'], 'file': tmpfile(row['access_token'] + '\n')}
+    auth_args = ['--authmech1', 'XOAUTH2', '--oauthaccesstoken1', oauth['file']]
+  else:
+    auth_args = ['--passfile1', tmpfile(j['password1'] + '\n')]
+    # imapsync auto-picks PLAIN, only pass --authmech1 for LOGIN/CRAM-MD5
+    if auth_type != 'PLAIN':
+      auth_args += ['--authmech1', auth_type]
+
+  # Custom params: JSON array of {o,v}, validated by the backend on write. One argv token per pair,
+  # "--opt" or "--opt=value" (value glued to its option, no shell)
+  custom_params = []
+  try:
+    pairs = json.loads(j['custom_params'] or '[]')
+  except ValueError:
+    pairs = []
+  for p in pairs if isinstance(pairs, list) else []:
+    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
+      continue
+    value = '' if p.get('v') is None else str(p['v'])
+    custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
+
+  # Effective per-process bandwidth cap (bytes/s): per-job value, capped by the global limit
+  eff_bw = num(j['maxbytespersecond'])
+  if global_bw > 0 and (eff_bw <= 0 or eff_bw > global_bw):
+    eff_bw = global_bw
+
+  # --nolog: the output is stored in returned_text, imapsync's own log dir would be the (read-only) cwd
+  cmd = ['/usr/local/bin/imapsync', '--tmpdir', '/tmp', '--nolog', '--nofoldersizes', '--addheader']
+  if num(j['timeout1']) > 0:
+    cmd += ['--timeout1', str(j['timeout1'])]
+  if num(j['timeout2']) > 0:
+    cmd += ['--timeout2', str(j['timeout2'])]
+  if j['exclude']:
+    cmd += ['--exclude', j['exclude']]
+  if j['subfolder2']:
+    cmd += ['--subfolder2', j['subfolder2']]
+  if num(j['maxage']) != 0:
+    cmd += ['--maxage', str(j['maxage'])]
+  if eff_bw > 0:
+    cmd += ['--maxbytespersecond', str(eff_bw)]
+  for col, opt in (('delete2duplicates', '--delete2duplicates'), ('subscribeall', '--subscribeall'),
+      ('delete1', '--delete1'), ('delete2', '--delete2'), ('automap', '--automap'),
+      ('skipcrossduplicates', '--skipcrossduplicates')):
+    if j[col] == 1:
+      cmd.append(opt)
+  enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
+  if enc1:
+    cmd.append(enc1)
+  cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
+  cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
+  if j['dry'] == 1:
+    cmd.append('--dry')
+  cmd += ['--no-modulesversion', '--noreleasecheck'] + custom_params
+
+  return {'id': j['id'], 'files': files, 'outfile': tmpfile(''), 'cmd': cmd, 'oauth': oauth}
+
+# Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
+def start_job(job):
+  with open(job['outfile'], 'wb') as out:
+    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
+
+# Store the result of a finished worker and remove its temp files
+def reap_job(job):
+  stdout = b''
+  try:
+    with open(job['outfile'], 'rb') as f:
+      size = os.fstat(f.fileno()).st_size
+      if size > max_output:
+        f.seek(size - max_output)
+        stdout = b'[output truncated, last %d bytes]\n' % max_output
+      stdout += f.read()
+  except OSError:
+    pass
+  unlink(job['files'])
+  text = stdout.decode('utf-8', 'replace')
+  # Last match: the echoed command line at the top may contain user controlled values
+  m = re.findall(r'Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)', text)
+  exit_code, exit_status = m[-1] if m else (None, None)
+  if exit_code is None:
+    text = 'Could not start or finish imapsync\n' + text
+  finish_job(job['id'], text, 1 if exit_code is not None and int(exit_code) == 0 else 0, exit_status)
+
+# Stop claiming new jobs, pass the signal on to the running imapsync processes
+def stop(signum, frame):
+  global stopping
+  stopping = True
+  for proc in list(running):
+    proc.send_signal(signal.SIGTERM)
+
+# Released by the kernel when the process ends, a crashed run leaves no stale lock
+lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+try:
+  fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+except OSError:
+  sys.exit("can't lock " + lock_file)
+
+if imapsync_running():
+  print('imapsync is active, exiting...')
+  sys.exit(0)
+
+for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
+  signal.signal(sig, stop)
+
+conn = connect()
+# Stale flags from crashed runs, safe only here: under the lock and with no imapsync running
+query("UPDATE imapsync SET is_running = 0")
+
+with open('/etc/sogo/sieve.creds') as f:
+  master_user, master_pass = [s.strip() for s in f.readline().split(':', 1)]
+
+# Worker pool: keep up to max_parallel workers busy, claim the next due job whenever a slot is free
+# (after every worker exit and at least every 10 seconds while workers run)
+last_token_sync = time.monotonic()
+try:
+  while True:
+    for proc, job in list(running.items()):
+      if proc.poll() is not None:
+        del running[proc]
+        reap_job(job)
+
+    if time.monotonic() - last_token_sync >= token_sync_interval:
+      last_token_sync = time.monotonic()
+      for job in running.values():
+        if job['oauth']:
+          try:
+            sync_token_file(job['oauth'])
+          except MySQLdb.Error as e:
+            print('imapsync job %d: %s' % (job['id'], e), file=sys.stderr)
+
+    max_parallel, global_bw = read_settings()
+    # Deferred and finished jobs drop out of next_sql, only failed claims are capped
+    failed_claims = 0
+    while not stopping and len(running) < max_parallel and failed_claims < 100:
+      skip = ('AND i.id NOT IN (%s)' % ', '.join(['%s'] * len(deferred))) if deferred else ''
+      j = query(next_sql.format(skip=skip), tuple(deferred) or None).fetchone()
+      if not j:
+        break
+      if query(claim_sql, (j['id'],)).rowcount != 1:
+        failed_claims += 1
+        continue
+
+      files = []
+      try:
+        job = prepare_job(j, global_bw, files)
+      except Exception as e:
+        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
+        finish_job(j['id'], 'Could not start or finish imapsync', 0, None)
+        job = None
+      if not job:
+        unlink(files)
+        continue
+      try:
+        running[start_job(job)] = job
+      except OSError as e:
+        # Release the claim, the job is picked up again later
+        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
+        unlink(files)
+        query("UPDATE imapsync SET is_running = 0 WHERE id = %s", (j['id'],))
+        break
+
+    if not running:
+      break
+    # A worker exit ends the wait early
+    for _ in range(50):
+      if any(proc.poll() is not None for proc in running):
+        break
+      time.sleep(0.2)
+finally:
+  # Never leave password/token files behind
+  for job in running.values():
+    unlink(job['files'])

data/Dockerfiles/dovecot/quarantine_notify.py

@@ -47,7 +47,7 @@ try:
   if max_score == "":
     max_score = 9999.0
 
-  def query_mysql(query, headers = True, update = False):
+  def query_mysql(query, params = None, headers = True, update = False):
     while True:
       try:
         cnx = MySQLdb.connect(user=os.environ.get('DBUSER'), password=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8mb4", collation="utf8mb4_general_ci")
@@ -57,7 +57,10 @@ try:
       else:
         break
     cur = cnx.cursor()
-    cur.execute(query)
+    if params:
+      cur.execute(query, params)
+    else:
+      cur.execute(query)
     if not update:
       result = []
       columns = tuple( [d[0] for d in cur.description] )
@@ -76,7 +79,7 @@ try:
 
   def notify_rcpt(rcpt, msg_count, quarantine_acl, category):
     if category == "add_header": category = "add header"
-    meta_query = query_mysql('SELECT `qhash`, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
+    meta_query = query_mysql('SELECT `qhash`, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = %s AND score < %s AND (action = %s OR "all" = %s)', (rcpt, max_score, category, category))
     print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count))
     if len(meta_query) == 0:
       return
@@ -130,7 +133,7 @@ try:
             server.sendmail(msg['From'], [str(redirect)] + [str(bcc)], text)
         server.quit()
         for res in meta_query:
-         query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
+         query_mysql('UPDATE quarantine SET notified = 1 WHERE id = %s', (res['id'],), update = True)
         r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
         break
       except Exception as ex:
@@ -138,7 +141,7 @@ try:
         print('%s'  % (ex))
         time.sleep(3)
 
-  records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND score < %f AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt' % (max_score))
+  records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND score < %s AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt', (max_score,))
 
   for record in records:
     attrs = ''
@@ -156,7 +159,7 @@ try:
     except Exception as ex:
       print('Could not determine last notification for %s, assuming never' % (record['rcpt']))
       last_notification = 0
-    attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
+    attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = %s', (record['rcpt'],))
     attrs = attrs_json[0]['attributes']
     if isinstance(attrs, str):
       # if attr is str then just load it

data/Dockerfiles/netfilter/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/netfilter/docker-entrypoint.sh

@@ -17,8 +17,8 @@ if [ $nftables_found -gt $iptables_found ]; then
 fi
 
 if [ $nftables_found -eq 0 ] && [ $nftables_found -eq $iptables_found ]; then
-  nftables_lines=$(nft list ruleset | wc -l)
-  iptables_lines=$(iptables-save | wc -l)
+  nftables_lines=$(nft list ruleset 2>/dev/null | wc -l)
+  iptables_lines=$(iptables-save 2>/dev/null | wc -l)
   if [ $nftables_lines -gt $iptables_lines ]; then
     backend=nftables
   else

data/Dockerfiles/netfilter/main.py

@@ -1,548 +1,57 @@
-#!/usr/bin/env python3
-
-DEBUG = False
-
-import re
-import os
-import sys
-import time
-import atexit
-import signal
-import ipaddress
-from collections import Counter
-from random import randint
-from threading import Thread
-from threading import Lock
-import redis
-import json
-import dns.resolver
-import dns.exception
-import uuid
-from modules.Logger import Logger
-from modules.IPTables import IPTables
-from modules.NFTables import NFTables
-
-def logdebug(msg):
-  if DEBUG:
-    logger.logInfo("DEBUG: %s" % msg)
-
-# Globals
-WHITELIST = []
-BLACKLIST = []
-bans = {}
-quit_now = False
-exit_code = 0
-lock = Lock()
-chain_name = "MAILCOW"
-r = None
-pubsub = None
-clear_before_quit = False
-
-def refreshF2boptions():
-  global f2boptions
-  global quit_now
-  global exit_code
-  f2boptions = {}
-
-  if not r.get('F2B_OPTIONS'):
-    f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
-    f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
-    f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
-    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
-    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
-    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
-    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
-  else:
-    try:
-      f2boptions = json.loads(r.get('F2B_OPTIONS'))
-    except ValueError as e:
-      logger.logCrit(
-        'Error loading F2B options: F2B_OPTIONS is not json. Exception: %s' % e)
-      quit_now = True
-      exit_code = 2
-
-  verifyF2boptions(f2boptions)
-  r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
-
-def verifyF2boptions(f2boptions):
-  verifyF2boption(f2boptions, 'ban_time', 1800)
-  verifyF2boption(f2boptions, 'max_ban_time', 10000)
-  verifyF2boption(f2boptions, 'ban_time_increment', True)
-  verifyF2boption(f2boptions, 'max_attempts', 10)
-  verifyF2boption(f2boptions, 'retry_window', 600)
-  verifyF2boption(f2boptions, 'netban_ipv4', 32)
-  verifyF2boption(f2boptions, 'netban_ipv6', 128)
-  verifyF2boption(f2boptions, 'banlist_id', str(uuid.uuid4()))
-  verifyF2boption(f2boptions, 'manage_external', 0)
-
-def verifyF2boption(f2boptions, f2boption, f2bdefault):
-  f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
-
-def refreshF2bregex():
-  global f2bregex
-  global quit_now
-  global exit_code
-  if not r.get('F2B_REGEX'):
-    f2bregex = {}
-    f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-    f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
-    f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
-    f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
-    f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
-    f2bregex[6] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[7] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[8] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-    f2bregex[9] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
-    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
-  else:
-    try:
-      f2bregex = {}
-      f2bregex = json.loads(r.get('F2B_REGEX'))
-    except ValueError:
-      logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
-      quit_now = True
-      exit_code = 2
-
-def get_ip(address):
-  ip = ipaddress.ip_address(address)
-  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
-    ip = ip.ipv4_mapped
-  if ip.is_private or ip.is_loopback:
-    return False
-
-  return ip
-
-def ban(address):
-  global f2boptions
-  global lock
-  logdebug("ban() called with address=%s" % address)
-  refreshF2boptions()
-  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-  RETRY_WINDOW = int(f2boptions['retry_window'])
-  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
-  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
-
-  ip = get_ip(address)
-  if not ip:
-    logdebug("No valid IP -- skipping ban()")
-    return
-  address = str(ip)
-  self_network = ipaddress.ip_network(address)
-
-  with lock:
-    temp_whitelist = set(WHITELIST)
-    logdebug("Checking if %s overlaps with any WHITELIST entries" % self_network)
-    if temp_whitelist:
-      for wl_key in temp_whitelist:
-        wl_net = ipaddress.ip_network(wl_key, False)
-        logdebug("Checking overlap between %s and %s" % (self_network, wl_net))
-        if wl_net.overlaps(self_network):
-          logger.logInfo(
-            'Address %s is allowlisted by rule %s' % (self_network, wl_net))
-          return
-
-  net = ipaddress.ip_network(
-    (address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
-  net = str(net)
-  logdebug("Ban net: %s" % net)
-
-  if not net in bans:
-    bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
-    logdebug("Initing new ban counter for %s" % net)
-
-  current_attempt = time.time()
-  logdebug("Current attempt ts=%s, previous: %s, retry_window: %s" %
-           (current_attempt, bans[net]['last_attempt'], RETRY_WINDOW))
-  if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
-    bans[net]['attempts'] = 0
-    logdebug("Ban counter for %s reset as window expired" % net)
-
-  bans[net]['attempts'] += 1
-  bans[net]['last_attempt'] = current_attempt
-  logdebug("%s attempts now %d" % (net, bans[net]['attempts']))
-
-  if bans[net]['attempts'] >= MAX_ATTEMPTS:
-    cur_time = int(round(time.time()))
-    NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
-    logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
-    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
-      with lock:
-        logdebug("Calling tables.banIPv4(%s)" % net)
-        tables.banIPv4(net)
-    elif int(f2boptions['manage_external']) != 1:
-      with lock:
-        logdebug("Calling tables.banIPv6(%s)" % net)
-        tables.banIPv6(net)
-
-    logdebug("Updating F2B_ACTIVE_BANS[%s]=%d" %
-              (net, cur_time + NET_BAN_TIME))
-    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
-  else:
-    logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
-      MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
-
-def unban(net):
-  global lock
-  logdebug("Calling unban() with net=%s" % net)
-  if not net in bans:
-    logger.logInfo(
-      '%s is not banned, skipping unban and deleting from queue (if any)' % net)
-    r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-    return
-  logger.logInfo('Unbanning %s' % net)
-  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
-    with lock:
-      logdebug("Calling tables.unbanIPv4(%s)" % net)
-      tables.unbanIPv4(net)
-  else:
-    with lock:
-      logdebug("Calling tables.unbanIPv6(%s)" % net)
-      tables.unbanIPv6(net)
-  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
-  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-  if net in bans:
-    logdebug("Unban for %s, setting attempts=0, ban_counter+=1" % net)
-    bans[net]['attempts'] = 0
-    bans[net]['ban_counter'] += 1
-
-def permBan(net, unban=False):
-  global f2boptions
-  global lock
-
-  is_unbanned = False
-  is_banned = False
-  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv4(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv4(net)
-  else:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv6(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv6(net)
-
-
-  if is_unbanned:
-    r.hdel('F2B_PERM_BANS', '%s' % net)
-    logger.logCrit('Removed host/network %s from denylist' % net)
-  elif is_banned:
-    r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
-    logger.logCrit('Added host/network %s to denylist' % net)
-
-def clear():
-  global lock
-  logger.logInfo('Clearing all bans')
-  for net in bans.copy():
-    logdebug("Unbanning net: %s" % net)
-    unban(net)
-  with lock:
-    logdebug("Clearing IPv4/IPv6 table")
-    tables.clearIPv4Table()
-    tables.clearIPv6Table()
-    try:
-      if r is not None:
-        r.delete('F2B_ACTIVE_BANS')
-        r.delete('F2B_PERM_BANS')
-    except Exception as ex:
-      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
-
-def watch():
-  global pubsub
-  global quit_now
-  global exit_code
-
-  logger.logInfo('Watching Redis channel F2B_CHANNEL')
-  pubsub.subscribe('F2B_CHANNEL')
-
-  while not quit_now:
-    try:
-      for item in pubsub.listen():
-        refreshF2bregex()
-        for rule_id, rule_regex in f2bregex.items():
-          if item['data'] and item['type'] == 'message':
-            try:
-              result = re.search(rule_regex, item['data'])
-            except re.error:
-              result = False
-            if result:
-              addr = result.group(1)
-              ip = ipaddress.ip_address(addr)
-              if ip.is_private or ip.is_loopback:
-                continue
-              logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
-              ban(addr)
-    except Exception as ex:
-      logger.logWarn('Error reading log line from pubsub: %s' % ex)
-      pubsub = None
-      quit_now = True
-      exit_code = 2
-
-def snat4(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
-
-def snat6(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
-
-def autopurge():
-  global f2boptions
-  logdebug("autopurge thread started")
-  while not quit_now:
-    logdebug("autopurge tick")
-    time.sleep(10)
-    refreshF2boptions()
-    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
-    logdebug("QUEUE_UNBAN: %s" % QUEUE_UNBAN)
-    if QUEUE_UNBAN:
-      for net in QUEUE_UNBAN:
-        logdebug("Autopurge: unbanning queued net: %s" % net)
-        unban(str(net))
-    # Only check expiry for actively banned IPs:
-    active_bans = r.hgetall('F2B_ACTIVE_BANS')
-    now = time.time()
-    for net_str, expire_str in active_bans.items():
-      logdebug("Checking ban expiry for (actively banned): %s" % net_str)
-      # Defensive: always process if timer missing or expired
-      try:
-        expire = float(expire_str)
-      except Exception:
-        logdebug("Invalid expire time for %s; unbanning" % net_str)
-        unban(net_str)
-        continue
-      time_left = expire - now
-      logdebug("Time left for %s: %.1f seconds" % (net_str, time_left))
-      if time_left <= 0:
-        logdebug("Ban expired for %s" % net_str)
-        unban(net_str)
-
-def mailcowChainOrder():
-  global lock
-  global quit_now
-  global exit_code
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      quit_now, exit_code = tables.checkIPv4ChainOrder()
-      if quit_now: return
-      quit_now, exit_code = tables.checkIPv6ChainOrder()
-
-def calcNetBanTime(ban_counter):
-  global f2boptions
-
-  BAN_TIME = int(f2boptions['ban_time'])
-  MAX_BAN_TIME = int(f2boptions['max_ban_time'])
-  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
-  NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
-  NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
-  return NET_BAN_TIME
-
-def isIpNetwork(address):
-  try:
-    ipaddress.ip_network(address, False)
-  except ValueError:
-    return False
-  return True
-
-def genNetworkList(list):
-  resolver = dns.resolver.Resolver()
-  hostnames = []
-  networks = []
-  for key in list:
-    if isIpNetwork(key):
-      networks.append(key)
-    else:
-      hostnames.append(key)
-  for hostname in hostnames:
-    hostname_ips = []
-    for rdtype in ['A', 'AAAA']:
-      try:
-        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
-      except dns.exception.Timeout:
-        logger.logInfo('Hostname %s timedout on resolve' % hostname)
-        break
-      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-        continue
-      except dns.exception.DNSException as dnsexception:
-        logger.logInfo('%s' % dnsexception)
-        continue
-      for rdata in answer:
-        hostname_ips.append(rdata.to_text())
-    networks.extend(hostname_ips)
-  return set(networks)
-
-def whitelistUpdate():
-  global lock
-  global quit_now
-  global WHITELIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_WHITELIST')
-    new_whitelist = []
-    if list:
-      new_whitelist = genNetworkList(list)
-    with lock:
-      if Counter(new_whitelist) != Counter(WHITELIST):
-        WHITELIST = new_whitelist
-        logger.logInfo('Allowlist was changed, it has %s entries' % len(WHITELIST))
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def blacklistUpdate():
-  global quit_now
-  global BLACKLIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_BLACKLIST')
-    new_blacklist = []
-    if list:
-      new_blacklist = genNetworkList(list)
-    if Counter(new_blacklist) != Counter(BLACKLIST):
-      addban = set(new_blacklist).difference(BLACKLIST)
-      delban = set(BLACKLIST).difference(new_blacklist)
-      BLACKLIST = new_blacklist
-      logger.logInfo('Denylist was changed, it has %s entries' % len(BLACKLIST))
-      if addban:
-        for net in addban:
-          permBan(net=net)
-      if delban:
-        for net in delban:
-          permBan(net=net, unban=True)
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def sigterm_quit(signum, frame):
-  global clear_before_quit
-  logdebug("SIGTERM received, setting clear_before_quit to True and exiting")
-  clear_before_quit = True
-  sys.exit(exit_code)
-
-def before_quit():
-  logdebug("before_quit called, clear_before_quit=%s" % clear_before_quit)
-  if clear_before_quit:
-    clear()
-  if pubsub is not None:
-    pubsub.unsubscribe()
-
-if __name__ == '__main__':
-  logger = Logger()
-  logdebug("Sys.argv: %s" % sys.argv)
-  atexit.register(before_quit)
-  signal.signal(signal.SIGTERM, sigterm_quit)
-
-  backend = sys.argv[1]
-  logdebug("Backend: %s" % backend)
-  if backend == "nftables":
-    logger.logInfo('Using NFTables backend')
-    tables = NFTables(chain_name, logger)
-  else:
-    logger.logInfo('Using IPTables backend')
-    logger.logWarn(
-        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
-        "Please switch to nftables on your host to ensure complete compatibility."
-    )
-    time.sleep(5)
-    tables = IPTables(chain_name, logger)
-
-  clear()
-  logger.logInfo("Initializing mailcow netfilter chain")
-  tables.initChainIPv4()
-  tables.initChainIPv6()
-
-  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE", "").lower() in ("y", "yes"):
-    logger.logInfo(f"Skipping {chain_name} isolation")
-  else:
-    logger.logInfo(f"Setting {chain_name} isolation")
-    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
-
-  # connect to redis
-  while True:
-    try:
-      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
-      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
-      logdebug(
-        "Connecting redis (SLAVEOF_IP:%s, PORT:%s)" % (redis_slaveof_ip, redis_slaveof_port))
-      if "".__eq__(redis_slaveof_ip):
-        r = redis.StrictRedis(
-          host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
-      else:
-        r = redis.StrictRedis(
-          host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0, password=os.environ['REDISPASS'])
-      r.ping()
-      pubsub = r.pubsub()
-    except Exception as ex:
-      logdebug(
-        'Redis connection failed: %s - trying again in 3 seconds' % (ex))
-      time.sleep(3)
-    else:
-      break
-  logger.set_redis(r)
-  logdebug("Redis connection established, setting up F2B keys")
-
-  if r.exists('F2B_LOG'):
-    logdebug("Renaming F2B_LOG to NETFILTER_LOG")
-    r.rename('F2B_LOG', 'NETFILTER_LOG')
-  r.delete('F2B_ACTIVE_BANS')
-  r.delete('F2B_PERM_BANS')
-
-  refreshF2boptions()
-
-  watch_thread = Thread(target=watch)
-  watch_thread.daemon = True
-  watch_thread.start()
-
-  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv4Address:
-        snat4_thread = Thread(target=snat4, args=(snat_ip,))
-        snat4_thread.daemon = True
-        snat4_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
-
-  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT6_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv6Address:
-        snat6_thread = Thread(target=snat6,args=(snat_ip,))
-        snat6_thread.daemon = True
-        snat6_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
-
-  autopurge_thread = Thread(target=autopurge)
-  autopurge_thread.daemon = True
-  autopurge_thread.start()
-
-  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
-  mailcowchainwatch_thread.daemon = True
-  mailcowchainwatch_thread.start()
-
-  blacklistupdate_thread = Thread(target=blacklistUpdate)
-  blacklistupdate_thread.daemon = True
-  blacklistupdate_thread.start()
-
-  whitelistupdate_thread = Thread(target=whitelistUpdate)
-  whitelistupdate_thread.daemon = True
-  whitelistupdate_thread.start()
-
-  while not quit_now:
-    time.sleep(0.5)
-
-  logdebug("Exiting with code %s" % exit_code)
-  sys.exit(exit_code)
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import signal
+import redis
+from modules.Logger import Logger
+from modules.Netfilter import Netfilter
+
+chain_name = "MAILCOW"
+
+def connect_redis(logger):
+  if os.getenv('REDIS_SLAVEOF_IP', ''):
+    host, port = os.getenv('REDIS_SLAVEOF_IP'), os.getenv('REDIS_SLAVEOF_PORT', '')
+  else:
+    host, port = os.getenv('IPV4_NETWORK', '172.22.1') + '.249', 6379
+  while True:
+    try:
+      # log lines are not always valid UTF-8
+      r = redis.StrictRedis(host=host, port=port, db=0, password=os.environ['REDISPASS'], decode_responses=True,
+                            encoding_errors='replace', socket_connect_timeout=5, socket_timeout=30)
+      r.ping()
+      return r
+    except Exception as ex:
+      logger.logWarn('Redis connection failed: %s - trying again in 3 seconds' % ex)
+      time.sleep(3)
+
+if __name__ == '__main__':
+  logger = Logger()
+
+  backend = sys.argv[1]
+  if backend == "nftables":
+    from modules.NFTables import NFTables
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    from modules.IPTables import IPTables
+    logger.logInfo('Using IPTables backend')
+    logger.logWarn(
+        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
+        "Please switch to nftables on your host to ensure complete compatibility."
+    )
+    time.sleep(5)
+    tables = IPTables(chain_name, logger)
+
+  r = connect_redis(logger)
+  logger.set_redis(r)
+
+  netfilter = Netfilter(r, tables, logger, os.environ)
+  signal.signal(signal.SIGTERM, lambda signum, frame: netfilter.stop.set())
+  signal.signal(signal.SIGINT, lambda signum, frame: netfilter.stop.set())
+
+  netfilter.setup()
+  netfilter.run()
+  netfilter.shutdown()
+  sys.exit(0)

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -0,0 +1,236 @@
+import ipaddress
+import time
+
+
+class BanManager:
+  """Failed attempts and automatic bans (Redis F2B_ACTIVE_BANS: network => expiry), the denylist (F2B_PERM_BANS)
+  and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
+  (chains, SNAT), because the backends must not run in parallel."""
+
+  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
+  EXCUSE_TIME = 60
+
+  def __init__(self, r, tables, options, logger, lock, clock=time.time):
+    self.r = r
+    self.tables = tables
+    self.options = options
+    self.logger = logger
+    self.lock = lock
+    self.clock = clock
+    self.counters = {}
+    self.banned = set()
+    self.denied = {}
+    self.allowlist = []
+    self.excused = {}
+
+  def excuse(self, ip):
+    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
+    with self.lock:
+      now = self.clock()
+      if self.excused.get(ip, 0) <= now:
+        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
+          ip, self.EXCUSE_TIME))
+      self.excused[ip] = now + self.EXCUSE_TIME
+
+  def attempt(self, ip):
+    """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
+    options = self.options.values
+    prefix = options['netban_ipv4'] if ip.version == 4 else options['netban_ipv6']
+    net = str(ipaddress.ip_network('%s/%d' % (ip, prefix), strict=False))
+    with self.lock:
+      now = self.clock()
+      allowed = self.allowed_by(ip)
+      if allowed is not None:
+        self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
+        return
+      if self.excused.get(ip, 0) > now:
+        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
+        return
+      if net in self.banned:
+        # log lines of connections from before the ban
+        return
+      counter = self.counter(net)
+      if now - counter['last_attempt'] > options['retry_window']:
+        counter['attempts'] = 0
+      counter['attempts'] += 1
+      counter['last_attempt'] = now
+      if counter['attempts'] < options['max_attempts']:
+        self.logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
+          options['max_attempts'] - counter['attempts'], options['retry_window'], net))
+        return
+
+      ban_time = self.options.ban_time(counter['ban_counter'])
+      self.logger.logCrit('Banning %s for %d minutes' % (net, ban_time / 60))
+      # Redis first: a firewall rule without expiry would never be lifted
+      self.r.hset('F2B_ACTIVE_BANS', net, int(round(now + ban_time)))
+      self.banned.add(net)
+      if options['manage_external'] != 1:
+        self.apply([net])
+
+  def unban(self, net):
+    """Lifts the automatic ban of a network; the rule stays while the denylist bans the same network"""
+    with self.lock:
+      normalized = self.normalize(net)
+      if normalized is None:
+        self.logger.logWarn('Removing invalid network %s from the active bans' % net)
+      else:
+        self.logger.logInfo('Unbanning %s' % net)
+        if normalized not in self.denied:
+          self.remove([normalized])
+      self.r.hdel('F2B_ACTIVE_BANS', net)
+      self.r.hdel('F2B_QUEUE_UNBAN', net)
+      if net in self.banned:
+        self.banned.discard(net)
+        counter = self.counter(net)
+        counter['attempts'] = 0
+        counter['ban_counter'] += 1
+
+  def unban_queued(self, entry):
+    """Unban request of the UI: a banned network, or a network that contains banned ones (allowlisting)"""
+    network = self.network(entry)
+    if network is None:
+      self.logger.logWarn('Removing invalid network %s from the unban queue' % entry)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+      return
+    with self.lock:
+      banned = set(self.r.hkeys('F2B_ACTIVE_BANS')) | self.banned
+      targets = [net for net in banned if self.overlaps(net, network)]
+      for net in targets:
+        self.unban(net)
+      if not targets and str(network) not in self.denied:
+        # a rule without known ban, e.g. left by an older netfilter
+        if self.remove([str(network)]):
+          self.logger.logInfo('Unbanning %s' % network)
+        else:
+          self.logger.logInfo('%s is not banned, removing it from the unban queue' % network)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+
+  def purge(self):
+    """Processes the unban queue of the UI and lifts expired bans"""
+    for entry in self.r.hkeys('F2B_QUEUE_UNBAN'):
+      self.unban_queued(entry)
+    now = self.clock()
+    for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+      try:
+        expired = self.network(net) is None or float(expires) <= now
+      except (TypeError, ValueError):
+        expired = True
+      if expired:
+        self.unban(net)
+    with self.lock:
+      # bans whose Redis entry is gone, e.g. deleted by hand
+      for net in self.banned - set(self.r.hkeys('F2B_ACTIVE_BANS')):
+        self.unban(net)
+    self.forget_idle(now)
+
+  def restore(self):
+    """Applies the unexpired automatic bans of Redis again after a start of netfilter and drops the expired ones"""
+    now = self.clock()
+    restored = []
+    with self.lock:
+      for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+        try:
+          valid = self.network(net) is not None and float(expires) > now
+        except (TypeError, ValueError):
+          valid = False
+        if not valid:
+          self.r.hdel('F2B_ACTIVE_BANS', net)
+          continue
+        self.banned.add(net)
+        self.counter(net)
+        restored.append(net)
+      if restored and self.options.values['manage_external'] != 1:
+        self.apply(restored)
+    if restored:
+      self.logger.logInfo('Restored %d active bans' % len(restored))
+
+  def reapply(self):
+    """Applies all bans again, e.g. after the MAILCOW chain had to be recreated"""
+    with self.lock:
+      if self.options.values['manage_external'] != 1:
+        self.apply(list(self.banned) + list(self.denied))
+
+  def set_denylist(self, entries):
+    """Bans the networks of the denylist permanently and lifts the bans of removed entries"""
+    networks = {}
+    for entry in entries:
+      net = self.normalize(entry)
+      if net is None:
+        self.logger.logWarn('Skipping invalid denylist entry %s' % entry)
+        continue
+      networks[net] = entry
+    with self.lock:
+      if networks.keys() == self.denied.keys():
+        return
+      self.logger.logInfo('Denylist was changed, it has %d entries' % len(networks))
+      added = [net for net in networks if net not in self.denied]
+      removed = [net for net in self.denied if net not in networks]
+      if added and self.options.values['manage_external'] != 1:
+        self.apply(added)
+      for net in added:
+        self.r.hset('F2B_PERM_BANS', networks[net], int(round(self.clock())))
+        self.logger.logCrit('Added host/network %s to denylist' % networks[net])
+      # an automatic ban of the same network keeps its rule
+      auto_banned = {self.normalize(net) for net in self.banned}
+      self.remove([net for net in removed if net not in auto_banned])
+      for net in removed:
+        self.r.hdel('F2B_PERM_BANS', self.denied[net])
+        self.logger.logCrit('Removed host/network %s from denylist' % self.denied[net])
+      self.denied = networks
+
+  def set_allowlist(self, entries):
+    networks = []
+    for entry in entries:
+      network = self.network(entry)
+      if network is None:
+        self.logger.logWarn('Skipping invalid allowlist entry %s' % entry)
+        continue
+      networks.append(network)
+    with self.lock:
+      if set(networks) != set(self.allowlist):
+        self.allowlist = networks
+        self.logger.logInfo('Allowlist was changed, it has %d entries' % len(networks))
+
+  def allowed_by(self, ip):
+    address = ipaddress.ip_network(ip)
+    for network in self.allowlist:
+      if network.version == address.version and network.overlaps(address):
+        return network
+    return None
+
+  def counter(self, net):
+    return self.counters.setdefault(net, {'attempts': 0, 'last_attempt': 0.0, 'ban_counter': 0})
+
+  def forget_idle(self, now):
+    """Drops counters of networks that were never banned and had no attempt within the retry window"""
+    window = self.options.values['retry_window']
+    with self.lock:
+      for net in [net for net, counter in self.counters.items()
+                  if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
+        del self.counters[net]
+      for ip in [ip for ip, until in self.excused.items() if until <= now]:
+        del self.excused[ip]
+
+  def apply(self, nets):
+    for net in self.tables.ban(nets):
+      self.logger.logWarn('Could not add the firewall rule for %s' % net)
+
+  def remove(self, nets):
+    return self.tables.unban(nets) if nets else []
+
+  @staticmethod
+  def network(entry):
+    try:
+      return ipaddress.ip_network(entry, strict=False)
+    except (TypeError, ValueError):
+      return None
+
+  @classmethod
+  def normalize(cls, entry):
+    network = cls.network(entry)
+    return str(network) if network is not None else None
+
+  @classmethod
+  def overlaps(cls, net, network):
+    other = cls.network(net)
+    return other is not None and other.version == network.version and other.overlaps(network)

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,6 +1,7 @@
 import iptc
 import time
-import os
+import ipaddress
+import contextlib
 
 class IPTables:
   def __init__(self, chain_name, logger):
@@ -8,62 +9,10 @@ class IPTables:
     self.logger = logger
 
   def initChainIPv4(self):
-    if not iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name) in iptc.Table(iptc.Table.FILTER).chains:
-      iptc.Table(iptc.Table.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
-      rule = iptc.Rule()
-      rule.src = '0.0.0.0/0'
-      rule.dst = '0.0.0.0/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
+    self.init_chain(iptc.Table(iptc.Table.FILTER))
 
   def initChainIPv6(self):
-    if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name) in iptc.Table6(iptc.Table6.FILTER).chains:
-      iptc.Table6(iptc.Table6.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
-      rule = iptc.Rule6()
-      rule.src = '::/0'
-      rule.dst = '::/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
-
-  def checkIPv4ChainOrder(self):
-    filter_table = iptc.Table(iptc.Table.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkIPv6ChainOrder(self):
-    filter_table = iptc.Table6(iptc.Table6.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    forward_chain = iptc.Chain(filter_table, 'FORWARD')
-    input_chain = iptc.Chain(filter_table, 'INPUT')
-    for chain in [forward_chain, input_chain]:
-      target_found = False
-      for position, item in enumerate(chain.rules):
-        if item.target.name == self.chain_name:
-          target_found = True
-          if position > 2:
-            self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-            err = True
-            exit_code = 2
-      if not target_found:
-        self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain(iptc.Table6(iptc.Table6.FILTER))
 
   def clearIPv4Table(self):
     self.clearTable(iptc.Table(iptc.Table.FILTER))
@@ -71,68 +20,133 @@ class IPTables:
   def clearIPv6Table(self):
     self.clearTable(iptc.Table6(iptc.Table6.FILTER))
 
-  def clearTable(self, filter_table):
-    filter_table.autocommit = False
-    forward_chain = iptc.Chain(filter_table, "FORWARD")
-    input_chain = iptc.Chain(filter_table, "INPUT")
-    mailcow_chain = iptc.Chain(filter_table, self.chain_name)
-    if mailcow_chain in filter_table.chains:
-      for rule in mailcow_chain.rules:
-        mailcow_chain.delete_rule(rule)
-      for rule in forward_chain.rules:
-        if rule.target.name == self.chain_name:
-          forward_chain.delete_rule(rule)
-      for rule in input_chain.rules:
-        if rule.target.name == self.chain_name:
-          input_chain.delete_rule(rule)
-      filter_table.delete_chain(self.chain_name)
-    filter_table.commit()
+  @contextlib.contextmanager
+  def transaction(self, table):
+    # One commit for all changes, autocommit rewrites the whole table for every change
+    table.autocommit = False
+    table.refresh()
+    try:
+      yield
+      table.commit()
+    finally:
+      table.refresh()
+      table.autocommit = True
+
+  def new_rule(self, table):
+    return iptc.Rule6() if isinstance(table, iptc.Table6) else iptc.Rule()
+
+  def family(self, table):
+    return 'IPv6' if isinstance(table, iptc.Table6) else 'IPv4'
+
+  def init_chain(self, filter_table):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
     filter_table.refresh()
-    filter_table.autocommit = True
+    if filter_table.is_chain(self.chain_name):
+      filter_table.flush_entries(self.chain_name)
+    else:
+      filter_table.create_chain(self.chain_name)
+    self.ensure_jumps(filter_table)
 
-  def banIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    recreated = False
+    for filter_table in [iptc.Table(iptc.Table.FILTER), iptc.Table6(iptc.Table6.FILTER)]:
+      filter_table.refresh()
+      if not filter_table.is_chain(self.chain_name):
+        filter_table.create_chain(self.chain_name)
+        self.logger.logWarn(f'MAILCOW {self.family(filter_table)} chain was missing and has been recreated')
+        recreated = True
+      self.ensure_jumps(filter_table)
+    return recreated
 
-  def banIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_jumps(self, filter_table):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jump (DOCKER-USER) at the top
+    for name in ['INPUT', 'FORWARD']:
+      chain = iptc.Chain(filter_table, name)
+      positions = [position for position, rule in enumerate(chain.rules) if rule.target.name == self.chain_name]
+      if positions == [0]: continue
 
-  def unbanIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules: 
-      return False
-    chain.delete_rule(rule)
-    return True
+      with self.transaction(filter_table):
+        for rule in [rule for rule in chain.rules if rule.target.name == self.chain_name]:
+          chain.delete_rule(rule)
+        rule = self.new_rule(filter_table)
+        rule.target = iptc.Target(rule, self.chain_name)
+        chain.insert_rule(rule)
+      if positions:
+        self.logger.logInfo(f'Moved MAILCOW jump to the top of the {self.family(filter_table)} {name} chain, it was at position {positions[0] + 1}')
+      else:
+        self.logger.logInfo(f'Added MAILCOW jump to the {self.family(filter_table)} {name} chain')
 
-  def unbanIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules:
-      return False
-    chain.delete_rule(rule)
-    return True
+  def clearTable(self, filter_table):
+    try:
+      with self.transaction(filter_table):
+        if not filter_table.is_chain(self.chain_name):
+          return
+        for chain in filter_table.chains:
+          for rule in chain.rules:
+            if rule.target.name == self.chain_name:
+              chain.delete_rule(rule)
+        filter_table.flush_entries(self.chain_name)
+        filter_table.delete_chain(self.chain_name)
+      self.logger.logInfo(f'Clear completed: {self.family(filter_table)}')
+    except Exception as ex:
+      self.logger.logWarn(f'Error clearing the {self.family(filter_table)} MAILCOW chain: {ex}')
+
+  def ban_rule_network(self, rule):
+    # Source network of a ban rule, None for every other rule
+    if rule.target.name not in ('REJECT', 'DROP') or rule.matches or rule.in_interface or rule.out_interface:
+      return None
+    try:
+      return ipaddress.ip_network(rule.src, strict=False)
+    except ValueError:
+      return None
+
+  def split_tables(self, sources):
+    networks = {4: [], 6: []}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      networks[network.version].append(network)
+    tables = [(iptc.Table(iptc.Table.FILTER), networks[4]), (iptc.Table6(iptc.Table6.FILTER), networks[6])]
+    return [(table, table_networks) for table, table_networks in tables if table_networks]
+
+  def ban(self, sources):
+    # One REJECT rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for filter_table, networks in self.split_tables(sources):
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          existing = {self.ban_rule_network(rule) for rule in chain.rules}
+          for network in dict.fromkeys(networks):
+            if network in existing: continue
+            rule = self.new_rule(filter_table)
+            rule.src = str(network)
+            rule.target = iptc.Target(rule, "REJECT")
+            chain.insert_rule(rule)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        failed += networks
+    return [str(network) for network in failed]
+
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for filter_table, networks in self.split_tables(sources):
+      targets = set(networks)
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          found = set()
+          for rule in chain.rules:
+            network = self.ban_rule_network(rule)
+            if network in targets:
+              chain.delete_rule(rule)
+              found.add(network)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        continue
+      removed += found
+    return [str(network) for network in removed]
 
   def snat4(self, snat_target, source):
     try:
@@ -188,7 +202,7 @@ class IPTables:
           if item == new_rule:
             if position != 0:
               chain.delete_rule(new_rule)
-    
+
       table.commit()
       table.autocommit = True
     except:
@@ -215,7 +229,9 @@ class IPTables:
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
     try:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+      table = iptc.Table(iptc.Table.FILTER)
+      table.refresh()
+      chain = iptc.Chain(table, self.chain_name)
 
       # insert mailcow isolation rule
       rule = iptc.Rule()
@@ -231,7 +247,7 @@ class IPTables:
       chain.insert_rule(rule, position=0)
 
       # insert mailcow isolation exception rule
-      if _allow != "":
+      if _allow:
         rule = iptc.Rule()
         rule.src = _allow
         rule.in_interface = f'!{_interface}'
@@ -249,4 +265,4 @@ class IPTables:
       return True
     except Exception as e:
       self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
-      return False
+      return False

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -1,8 +1,9 @@
 import nftables
 import ipaddress
-import os
 
 class NFTables:
+  ISOLATION_COMMENTS = ('mailcow isolation', 'mailcow isolation allow')
+
   def __init__(self, chain_name, logger):
     self.chain_name = chain_name
     self.logger = logger
@@ -12,41 +13,15 @@ class NFTables:
     self.nft.set_handle_output(True)
     self.nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
                             'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
+    self.mailcow_chains = set()
 
     self.search_current_chains()
 
   def initChainIPv4(self):
-    self.insert_mailcow_chains("ip")
+    self.init_chain("ip")
 
   def initChainIPv6(self):
-    self.insert_mailcow_chains("ip6")
-
-  def checkIPv4ChainOrder(self):
-    return self.checkChainOrder("ip")
-
-  def checkIPv6ChainOrder(self):
-    return self.checkChainOrder("ip6")
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    for chain in ['input', 'forward']:
-      chain_position = self.check_mailcow_chains(filter_table, chain)
-      if chain_position is None: continue
-
-      if chain_position is False:
-        self.logger.logCrit(f'MAILCOW target not found in {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-      if chain_position > 0:
-        chain_position += 1
-        self.logger.logCrit(f'MAILCOW target is in position {chain_position} in the {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain("ip6")
 
   def clearIPv4Table(self):
     self.clearTable("ip")
@@ -54,84 +29,19 @@ class NFTables:
   def clearIPv6Table(self):
     self.clearTable("ip6")
 
-  def clearTable(self, _family):
-    is_empty_dict = True
-    json_command = self.get_base_dict()
-    chain_handle = self.get_chain_handle(_family, "filter", self.chain_name)
-    # if no handle, the chain doesn't exists
-    if chain_handle is not None:
-      is_empty_dict = False
-      # flush chain
-      mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-      flush_chain = {'flush': {'chain': mailcow_chain}}
-      json_command["nftables"].append(flush_chain)
-
-    # remove rule in forward chain
-    # remove rule in input chain
-    chains_family = [self.nft_chain_names[_family]['filter']['input'],
-                    self.nft_chain_names[_family]['filter']['forward'] ]
-
-    for chain_base in chains_family:
-      if not chain_base: continue
-
-      rules_handle = self.get_rules_handle(_family, "filter", chain_base)
-      if rules_handle is not None:
-        for r_handle in rules_handle:
-          is_empty_dict = False
-          mailcow_rule = {'family':_family,
-                          'table': 'filter',
-                          'chain': chain_base,
-                          'handle': r_handle }
-          delete_rules = {'delete': {'rule': mailcow_rule} }
-          json_command["nftables"].append(delete_rules)
-
-    # remove chain
-    # after delete all rules referencing this chain
-    if chain_handle is not None:
-      mc_chain_handle = {'family':_family,
-                        'table': 'filter',
-                        'name': self.chain_name,
-                        'handle': chain_handle }
-      delete_chain = {'delete': {'chain': mc_chain_handle} }
-      json_command["nftables"].append(delete_chain)
-
-    if is_empty_dict == False:
-      if self.nft_exec_dict(json_command):
-        self.logger.logInfo(f"Clear completed: {_family}")
-
-  def banIPv4(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip")
-    return self.nft_exec_dict(ban_dict)
-
-  def banIPv6(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip6")
-    return self.nft_exec_dict(ban_dict)
-
-  def unbanIPv4(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
-  def unbanIPv6(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip6")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
   def snat4(self, snat_target, source):
     self.snat_rule("ip", snat_target, source)
 
   def snat6(self, snat_target, source):
     self.snat_rule("ip6", snat_target, source)
 
-
-  def nft_exec_dict(self, query: dict):
+  def nft_exec_dict(self, query: dict, quiet: bool = False):
     if not query: return False
 
     rc, output, error = self.nft.json_cmd(query)
     if rc != 0:
-      #self.logger.logCrit(f"Nftables Error: {error}")
+      if not quiet:
+        self.logger.logWarn(f"nftables: {str(error).strip() or f'error {rc}'}")
       return False
 
     # Prevent returning False or empty string on commands that do not produce output
@@ -144,118 +54,165 @@ class NFTables:
     return {'nftables': [{ 'metainfo': { 'json_schema_version': 1} } ] }
 
   def search_current_chains(self):
+    nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
+                       'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
     nft_chain_priority = {'ip': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} },
-                      'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+                          'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+    mailcow_chains = set()
 
     # Command: 'nft list chains'
     _list = {'list' : {'chains': 'null'} }
     command = self.get_base_dict()
     command['nftables'].append(_list)
     kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset['nftables']:
-        chain = _object.get("chain")
-        if not chain: continue
+    if not kernel_ruleset:
+      return
 
-        _family = chain['family']
-        _table = chain['table']
-        _hook = chain.get("hook")
-        _priority = chain.get("prio")
-        _name = chain['name']
-
-        if _family not in self.nft_chain_names: continue
-        if _table not in self.nft_chain_names[_family]: continue
-        if _hook not in self.nft_chain_names[_family][_table]: continue
-        if _priority is None: continue
-
-        _saved_priority = nft_chain_priority[_family][_table][_hook]
-        if _saved_priority is None or _priority < _saved_priority:
-          # at this point, we know the chain has:
-          # hook and priority set
-          # and it has the lowest priority
-          nft_chain_priority[_family][_table][_hook] = _priority
-          self.nft_chain_names[_family][_table][_hook] = _name
-
-  def search_for_chain(self, kernel_ruleset: dict, chain_name: str):
-    found = False
-    for _object in kernel_ruleset["nftables"]:
+    for _object in kernel_ruleset['nftables']:
       chain = _object.get("chain")
-      if not chain:
-        continue
-      ch_name = chain.get("name")
-      if ch_name == chain_name:
-        found = True
-        break
-    return found
+      if not chain: continue
+
+      _family = chain['family']
+      _table = chain['table']
+      _hook = chain.get("hook")
+      _priority = chain.get("prio")
+      _name = chain['name']
+
+      if _family not in nft_chain_names: continue
+      if _table not in nft_chain_names[_family]: continue
+      if _table == 'filter' and _name == self.chain_name:
+        mailcow_chains.add(_family)
+      if _hook not in nft_chain_names[_family][_table]: continue
+      if _priority is None: continue
+
+      _saved_priority = nft_chain_priority[_family][_table][_hook]
+      if _saved_priority is None or _priority < _saved_priority:
+        # at this point, we know the chain has:
+        # hook and priority set
+        # and it has the lowest priority
+        nft_chain_priority[_family][_table][_hook] = _priority
+        nft_chain_names[_family][_table][_hook] = _name
+
+    self.nft_chain_names = nft_chain_names
+    self.mailcow_chains = mailcow_chains
+
+  def has_filter_chains(self, _family: str):
+    # no base chains: the filter table does not exist, e.g. ip6 if docker has ipv6 disabled
+    return any(self.nft_chain_names[_family]['filter'].values())
+
+  def list_rules(self, _family: str, _table: str, _chain: str):
+    # Command: 'nft list chain {family} {table} {chain}', None if the chain does not exist
+    _list = {'list': {'chain': {'family': _family, 'table': _table, 'name': _chain}}}
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return None
+    return [_object["rule"] for _object in kernel_ruleset["nftables"] if _object.get("rule")]
 
   def get_chain_dict(self, _family: str, _name: str):
-    # nft (add | create) chain [<family>] <table> <name> 
+    # nft (add | create) chain [<family>] <table> <name>
     _chain_opts = {'family': _family, 'table': 'filter', 'name': _name  }
     _add = {'add': {'chain': _chain_opts} }
     final_chain = self.get_base_dict()
     final_chain["nftables"].append(_add)
     return final_chain
 
-  def get_mailcow_jump_rule_dict(self, _family: str, _chain: str):
-    _jump_rule = self.get_base_dict()
-    _expr_opt=[]
-    _expr_counter = {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}
-    _counter_dict = {'counter': _expr_counter}
-    _expr_opt.append(_counter_dict)
-
-    _jump_opts = {'jump': {'target': self.chain_name} }
-
-    _expr_opt.append(_jump_opts)
-
+  def get_mailcow_jump_rule(self, _family: str, _chain: str):
+    _expr_opt = [{'counter': {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}},
+                 {'jump': {'target': self.chain_name}}]
     _rule_params = {'family': _family,
                     'table': 'filter',
                     'chain': _chain,
                     'expr': _expr_opt,
                     'comment': "mailcow" }
+    return {'insert': {'rule': _rule_params} }
 
-    _add_rule = {'insert': {'rule': _rule_params} }
+  def is_mailcow_jump(self, rule: dict):
+    # by target, jumps restored by iptables-restore carry no nft comment
+    return any(isinstance(expr, dict) and isinstance(expr.get('jump'), dict) and expr['jump'].get('target') == self.chain_name
+               for expr in rule.get('expr', []))
 
-    _jump_rule["nftables"].append(_add_rule)
+  def init_chain(self, _family: str):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
+    if not self.has_filter_chains(_family): return
 
-    return _jump_rule
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+        self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    else:
+      flush_chain = {'flush': {'chain': {'family': _family, 'table': 'filter', 'name': self.chain_name}}}
+      command = self.get_base_dict()
+      command['nftables'].append(flush_chain)
+      self.nft_exec_dict(command)
 
-  def insert_mailcow_chains(self, _family: str):
-    nft_input_chain = self.nft_chain_names[_family]['filter']['input']
-    nft_forward_chain = self.nft_chain_names[_family]['filter']['forward']
-    # Command: 'nft list table <family> filter'
-    _table_opts = {'family': _family, 'name': 'filter'}
-    _list = {'list': {'table': _table_opts} }
+    self.ensure_jumps(_family)
+
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    self.search_current_chains()
+    recreated = False
+    for _family in ['ip', 'ip6']:
+      if not self.has_filter_chains(_family): continue
+
+      if _family not in self.mailcow_chains:
+        if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+          self.logger.logWarn(f"MAILCOW {_family} chain was missing and has been recreated")
+          recreated = True
+      self.ensure_jumps(_family)
+    return recreated
+
+  def ensure_jumps(self, _family: str):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jumps (DOCKER-USER, DOCKER-FORWARD) at the top
+    for hook in ['input', 'forward']:
+      base_chain = self.nft_chain_names[_family]['filter'][hook]
+      if not base_chain: continue
+
+      rules = self.list_rules(_family, 'filter', base_chain)
+      if rules is None: continue
+
+      positions = [position for position, rule in enumerate(rules) if self.is_mailcow_jump(rule)]
+      if positions == [0]: continue
+
+      # delete and insert in one transaction, the chain is never without jump
+      command = self.get_base_dict()
+      for position in positions:
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': base_chain, 'handle': rules[position]['handle']}
+        command['nftables'].append({'delete': {'rule': _rule_opts}})
+      command['nftables'].append(self.get_mailcow_jump_rule(_family, base_chain))
+      if self.nft_exec_dict(command):
+        if positions:
+          self.logger.logInfo(f"Moved MAILCOW jump to the top of the {_family} {base_chain} chain, it was at position {positions[0] + 1}")
+        else:
+          self.logger.logInfo(f"Added MAILCOW jump to the {_family} {base_chain} chain")
+
+  def clearTable(self, _family: str):
+    # Jumps first, the chain can only be deleted without references. Separate steps, so one failing step
+    # does not roll back the others
+    _list = {'list': {'table': {'family': _family, 'name': 'filter'}}}
     command = self.get_base_dict()
     command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      # chain
-      if not self.search_for_chain(kernel_ruleset, self.chain_name):
-        cadena = self.get_chain_dict(_family, self.chain_name)
-        if self.nft_exec_dict(cadena):
-          self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return
 
-      input_jump_found, forward_jump_found = False, False
+    delete_jumps = self.get_base_dict()
+    for _object in kernel_ruleset["nftables"]:
+      rule = _object.get("rule")
+      if rule and self.is_mailcow_jump(rule):
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': rule['chain'], 'handle': rule['handle']}
+        delete_jumps['nftables'].append({'delete': {'rule': _rule_opts}})
+    if len(delete_jumps['nftables']) > 1:
+      self.nft_exec_dict(delete_jumps)
 
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if nft_input_chain and rule["chain"] == nft_input_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            input_jump_found = True
-        if nft_forward_chain and rule["chain"] == nft_forward_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            forward_jump_found = True
-
-      if not input_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_input_chain)
-        self.nft_exec_dict(command)
-
-      if not forward_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_forward_chain)
-        self.nft_exec_dict(command)
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      return
+    mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+    command = self.get_base_dict()
+    command['nftables'].append({'flush': {'chain': mailcow_chain}})
+    command['nftables'].append({'delete': {'chain': mailcow_chain}})
+    if self.nft_exec_dict(command):
+      self.logger.logInfo(f"Clear completed: {_family}")
 
   def delete_nat_rule(self, _family:str, _chain: str, _handle:str):
     delete_command = self.get_base_dict()
@@ -285,23 +242,14 @@ class NFTables:
     # no postrouting chain, may occur if docker has ipv6 disabled.
     if not chain_name: return
 
-    # Command: nft list chain <family> nat <chain_name>
-    _chain_opts = {'family': _family, 'table': 'nat', 'name': chain_name}
-    _list = {'list':{'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if not kernel_ruleset:
+    rules = self.list_rules(_family, 'nat', chain_name)
+    if rules is None:
       return
 
     rule_position = 0
     rule_handle = None
     rule_found = False
-    for _object in kernel_ruleset["nftables"]:
-      if not _object.get("rule"):
-        continue
-
-      rule = _object["rule"]
+    for rule in rules:
       if not rule.get("comment") or not rule["comment"] == "mailcow":
         rule_position +=1
         continue
@@ -314,23 +262,28 @@ class NFTables:
     target_net = ipaddress.ip_network(snat_target, strict=False)
 
     if rule_found:
-      saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
-      saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
+      try:
+        saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
+        saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
 
-      daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
-      daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
+        daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
+        daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
 
-      target_ip = rule["expr"][3]["snat"]["addr"]
+        target_ip = rule["expr"][3]["snat"]["addr"]
 
-      saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
-      daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
-      current_target_net = ipaddress.ip_network(target_ip, strict=False)
+        saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
+        daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
+        current_target_net = ipaddress.ip_network(target_ip, strict=False)
 
-      match = all((
-                dest_net == saddr_net,
-                dest_net == daddr_net,
-                target_net == current_target_net
-              ))
+        match = all((
+                  dest_net == saddr_net,
+                  dest_net == daddr_net,
+                  target_net == current_target_net
+                ))
+      except (KeyError, IndexError, TypeError, ValueError):
+        # rule of another format, replace it
+        saddr_net = dest_net
+        match = False
       try:
         if rule_position == 0:
           if not match:
@@ -377,49 +330,13 @@ class NFTables:
       except:
         self.logger.logCrit(f"Error running SNAT on {_family}, retrying...")
 
-  def get_chain_handle(self, _family: str, _table: str, chain_name: str):
-    chain_handle = None
-    # Command: 'nft list chains {family}'
-    _list = {'list': {'chains': {'family': _family} } }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("chain"):
-          continue
-        chain = _object["chain"]
-        if chain["family"] == _family and chain["table"] == _table and chain["name"] == chain_name:
-          chain_handle = chain["handle"]
-          break
-    return chain_handle
-
   def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
-    rule_handle = []
-    # Command: 'nft list chain {family} {table} {chain_name}'
-    _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
-          if rule.get("comment") and rule["comment"] == _comment_filter:
-            rule_handle.append(rule["handle"])
-    return rule_handle
-
-  def get_ban_ip_dict(self, ipaddr: str, _family: str):
-    json_command = self.get_base_dict()
+    rules = self.list_rules(_family, _table, chain_name) or []
+    return [rule["handle"] for rule in rules if rule.get("comment") == _comment_filter]
 
+  def get_ban_rule(self, network, _family: str):
     expr_opt = []
-    ipaddr_net = ipaddress.ip_network(ipaddr, strict=False)
-    right_dict = {'prefix': {'addr': str(ipaddr_net.network_address), 'len': int(ipaddr_net.prefixlen) } }
+    right_dict = {'prefix': {'addr': str(network.network_address), 'len': int(network.prefixlen) } }
 
     left_dict = {'payload': {'protocol': _family, 'field': 'saddr'} }
     match_dict = {'op': '==', 'left': left_dict, 'right': right_dict }
@@ -432,228 +349,115 @@ class NFTables:
 
     rule_dict = {'family': _family, 'table': "filter", 'chain': self.chain_name, 'expr': expr_opt}
 
-    base_dict = {'insert': {'rule': rule_dict} }
-    json_command["nftables"].append(base_dict)
+    return {'insert': {'rule': rule_dict} }
 
-    return json_command
+  def ban_rule_network(self, rule: dict, _family: str):
+    # Source network of a ban rule (saddr match and drop), None for every other rule
+    if rule.get("comment") in self.ISOLATION_COMMENTS:
+      return None
+    exprs = [expr for expr in rule.get("expr", []) if isinstance(expr, dict)]
+    if not any('drop' in expr or 'reject' in expr for expr in exprs):
+      return None
+    for expr in exprs:
+      match = expr.get("match")
+      if not isinstance(match, dict) or match.get("op") != "==":
+        continue
+      left = match.get("left")
+      payload = left.get("payload") if isinstance(left, dict) else None
+      if not isinstance(payload, dict) or payload.get("protocol") != _family or payload.get("field") != "saddr":
+        continue
+      right = match.get("right")
+      try:
+        if isinstance(right, dict) and isinstance(right.get("prefix"), dict):
+          return ipaddress.ip_network(f'{right["prefix"]["addr"]}/{right["prefix"]["len"]}', strict=False)
+        if isinstance(right, str):
+          return ipaddress.ip_network(right, strict=False)
+      except (KeyError, ValueError):
+        return None
+    return None
 
-  def get_unban_ip_dict(self, ipaddr:str, _family: str):
-    json_command = self.get_base_dict()
-    # Command: 'nft list chain {s_family} filter  MAILCOW'
-    _chain_opts = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    rule_handle = None
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
+  def split_families(self, sources):
+    families = {}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      families.setdefault('ip' if network.version == 4 else 'ip6', []).append(network)
+    return families
 
-        rule = _object["rule"]["expr"][0]["match"]
-        if not "payload" in rule["left"]:
-          continue
-        left_opt = rule["left"]["payload"]
-        if not left_opt["protocol"] == _family:
-          continue
-        if not left_opt["field"] =="saddr":
-          continue
+  def ban(self, sources):
+    # One drop rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if rules is None:
+        failed += networks
+        continue
+      existing = {self.ban_rule_network(rule, _family) for rule in rules}
+      missing = [network for network in dict.fromkeys(networks) if network not in existing]
+      if not missing: continue
 
-        # ip currently banned
-        rule_right = rule["right"]
-        if isinstance(rule_right, dict):
-          current_rule_ip = rule_right["prefix"]["addr"] + '/' + str(rule_right["prefix"]["len"])
-        else:
-          current_rule_ip = rule_right
-        current_rule_net = ipaddress.ip_network(current_rule_ip)
+      command = self.get_base_dict()
+      for network in missing:
+        command['nftables'].append(self.get_ban_rule(network, _family))
+      if not self.nft_exec_dict(command):
+        failed += missing
+    return [str(network) for network in failed]
 
-        # ip to ban
-        candidate_net = ipaddress.ip_network(ipaddr, strict=False)
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if not rules: continue
 
-        if current_rule_net == candidate_net:
-          rule_handle = _object["rule"]["handle"]
-          break
-
-      if rule_handle is not None:
-        mailcow_rule = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule_handle}
-        delete_rule = {'delete': {'rule': mailcow_rule} }
-        json_command["nftables"].append(delete_rule)
-      else:
-        return False
-
-    return json_command
-
-  def check_mailcow_chains(self, family: str, chain: str):
-    position = 0
-    rule_found = False
-    chain_name = self.nft_chain_names[family]['filter'][chain]
-
-    if not chain_name: return None
-
-    _chain_opts = {'family': family, 'table': 'filter', 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts}}
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-        rule = _object["rule"]
-        if rule.get("comment") and rule["comment"] == "mailcow":
-          rule_found = True
-          break
-
-        position+=1
-
-    return position if rule_found else False
+      targets = set(networks)
+      command = self.get_base_dict()
+      found = set()
+      for rule in rules:
+        network = self.ban_rule_network(rule, _family)
+        if network in targets:
+          _rule_opts = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule['handle']}
+          command['nftables'].append({'delete': {'rule': _rule_opts}})
+          found.add(network)
+      if found and self.nft_exec_dict(command):
+        removed += found
+    return [str(network) for network in removed]
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
-    family = "ip"
-    table = "filter"
-    comment_filter_drop = "mailcow isolation"
-    comment_filter_allow = "mailcow isolation allow"
-    json_command = self.get_base_dict()
+    # One rule per port: iptables-nft cannot show an anonymous set, a host that saves and restores the ruleset
+    # with iptables-save (e.g. netfilter-persistent) would get a drop for all ports
+    for comment in self.ISOLATION_COMMENTS:
+      for handle in self.get_rules_handle("ip", "filter", self.chain_name, comment):
+        self.delete_filter_rule("ip", self.chain_name, handle)
 
-    # Delete old mailcow isolation rules
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-
-    # insert mailcow isolation rule
-    _match_dict_drop = [
-      {
-        "match": {
-          "op": "!=",
-          "left": {
-            "meta": {
-              "key": "iifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "meta": {
-              "key": "oifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "payload": {
-              "protocol": "tcp",
-              "field": "dport"
-            }
-          },
-          "right": {
-            "set": _dports
-          }
-        }
-      },
-      {
-        "counter": {
-          "packets": 0,
-          "bytes": 0
-        }
-      },
-      {
-        "drop": None
-      }
-    ]
-    rule_drop = { "insert": { "rule": {
-      "family": family,
-      "table": table,
-      "chain": self.chain_name,
-      "comment": comment_filter_drop,
-      "expr": _match_dict_drop
-    }}}
-    json_command["nftables"].append(rule_drop)
-
-    # insert mailcow isolation allow rule
-    if _allow != "":
-      _match_dict_allow = [
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "ip",
-                "field": "saddr"
-              }
-            },
-            "right": _allow
-          }
-        },
-        {
-          "match": {
-            "op": "!=",
-            "left": {
-              "meta": {
-                "key": "iifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "meta": {
-                "key": "oifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "tcp",
-                "field": "dport"
-              }
-            },
-            "right": {
-              "set": _dports
-            }
-          }
-        },
-        {
-          "counter": {
-            "packets": 0,
-            "bytes": 0
-          }
-        },
-        {
-          "accept": None
-        }
-      ]
-      rule_allow = { "insert": { "rule": {
-        "family": family,
-        "table": table,
-        "chain": self.chain_name,
-        "comment": comment_filter_allow,
-        "expr": _match_dict_allow
-      }}}
-      json_command["nftables"].append(rule_allow)
-
-    success = self.nft_exec_dict(json_command)
-    if success == False:
+    if not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[0], {"drop": None})):
       self.logger.logCrit(f"Error adding {self.chain_name} isolation")
       return False
 
-    return True
+    # separate, an invalid address must not prevent the drop rules. Inserted last, so they are above them
+    if _allow and not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[1], {"accept": None}, _allow)):
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation exception for {_allow}")
+      return False
+
+    return True
+
+  def get_isolation_dict(self, _interface:str, _dports:list, _comment:str, _verdict:dict, _saddr:str = ""):
+    json_command = self.get_base_dict()
+    for port in _dports:
+      expr = []
+      if _saddr:
+        expr.append({"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": _saddr}})
+      expr += [
+        {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": int(port)}},
+        {"counter": {"packets": 0, "bytes": 0}},
+        _verdict
+      ]
+      json_command["nftables"].append({"insert": {"rule": {
+        "family": "ip",
+        "table": "filter",
+        "chain": self.chain_name,
+        "comment": _comment,
+        "expr": expr
+      }}})
+    return json_command

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -0,0 +1,200 @@
+import ipaddress
+import threading
+import dns.exception
+import dns.resolver
+from modules.BanManager import BanManager
+from modules.Options import Options
+from modules.Regexes import Regexes
+
+
+class Netfilter:
+  """Runs the workers of netfilter. Every worker is a loop that logs errors and goes on, so neither a broken log
+  line nor a Redis restart can stop the expiry of bans."""
+
+  ISOLATION_INTERFACE = 'br-mailcow'
+  ISOLATION_PORTS = [3306, 6379, 8983, 12345]
+
+  def __init__(self, r, tables, logger, env):
+    self.r = r
+    self.tables = tables
+    self.logger = logger
+    self.env = env
+    # reentrant, bans.reapply() runs within the chain check
+    self.lock = threading.RLock()
+    self.stop = threading.Event()
+    self.options = Options(logger)
+    self.regexes = Regexes(logger)
+    self.bans = BanManager(r, tables, self.options, logger, self.lock)
+    self.snat4_target = self.snat_target('SNAT_TO_SOURCE', 4)
+    self.snat6_target = self.snat_target('SNAT6_TO_SOURCE', 6)
+    self.resolved = {}
+    self.workers = {
+      'watch': (self.watch, 3),
+      'purge': (self.purge, 10),
+      'chains': (self.check_chains, 10),
+      'lists': (self.update_lists, 60),
+    }
+    if self.snat4_target or self.snat6_target:
+      self.workers['snat'] = (self.snat, 10)
+    self.threads = {}
+
+  def setup(self):
+    if self.r.exists('F2B_LOG'):
+      self.r.rename('F2B_LOG', 'NETFILTER_LOG')
+    # filled again by the lists worker
+    self.r.delete('F2B_PERM_BANS')
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    with self.lock:
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+      self.logger.logInfo('Initializing mailcow netfilter chain')
+      self.tables.initChainIPv4()
+      self.tables.initChainIPv6()
+      self.isolate()
+    self.bans.restore()
+
+  def run(self):
+    for name in self.workers:
+      self.start(name)
+    while not self.stop.wait(5):
+      for name, thread in list(self.threads.items()):
+        if not thread.is_alive():
+          self.logger.logCrit('Worker %s stopped unexpectedly, restarting it' % name)
+          self.start(name)
+
+  def shutdown(self):
+    """Removes the firewall rules, the active bans stay in Redis and are restored on the next start"""
+    self.stop.set()
+    for thread in self.threads.values():
+      thread.join(timeout=5)
+    with self.lock:
+      self.logger.logInfo('Clearing all bans')
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+    try:
+      self.r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      self.logger.logWarn('Error clearing redis key F2B_PERM_BANS: %s' % ex)
+
+  def start(self, name):
+    task, interval = self.workers[name]
+    thread = threading.Thread(target=self.loop, args=(name, task, interval), name=name, daemon=True)
+    self.threads[name] = thread
+    thread.start()
+
+  def loop(self, name, task, interval):
+    while not self.stop.is_set():
+      try:
+        task()
+      except Exception as ex:
+        self.logger.logWarn('Error in worker %s: %s' % (name, ex))
+      self.stop.wait(interval)
+
+  def watch(self):
+    pubsub = self.r.pubsub(ignore_subscribe_messages=True)
+    try:
+      pubsub.subscribe('F2B_CHANNEL')
+      self.logger.logInfo('Watching Redis channel F2B_CHANNEL')
+      while not self.stop.is_set():
+        message = pubsub.get_message(timeout=1.0)
+        if message and message['type'] == 'message' and message['data']:
+          self.handle_line(message['data'])
+    finally:
+      try:
+        pubsub.close()
+      except Exception:
+        pass
+
+  def handle_line(self, line):
+    # a broken line must not end the watcher
+    try:
+      excused = Regexes.temporary_failure(line)
+      if excused is not None:
+        ip = Regexes.parse_ip(excused)
+        if self.external(ip):
+          self.bans.excuse(ip)
+        return
+      result = self.regexes.match(line)
+      if result is None:
+        return
+      rule_id, address = result
+      ip = Regexes.parse_ip(address)
+      if ip is None:
+        self.logger.logWarn('Rule id %s matched an invalid address %s (%s)' % (rule_id, address, line))
+        return
+      if not self.external(ip):
+        return
+      self.logger.logWarn('%s matched rule id %s (%s)' % (ip, rule_id, line))
+      self.bans.attempt(ip)
+    except Exception as ex:
+      self.logger.logWarn('Error processing log line %r: %s' % (line, ex))
+
+  @staticmethod
+  def external(ip):
+    # private and loopback addresses are never banned
+    return ip is not None and not (ip.is_private or ip.is_loopback)
+
+  def purge(self):
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    self.bans.purge()
+
+  def check_chains(self):
+    with self.lock:
+      if self.tables.ensure_chains():
+        self.isolate()
+        self.bans.reapply()
+
+  def snat(self):
+    with self.lock:
+      if self.snat4_target:
+        self.tables.snat4(self.snat4_target, self.env.get('IPV4_NETWORK', '172.22.1') + '.0/24')
+      if self.snat6_target:
+        self.tables.snat6(self.snat6_target, self.env.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
+
+  def update_lists(self):
+    self.bans.set_allowlist(self.resolve(self.r.hkeys('F2B_WHITELIST')))
+    self.bans.set_denylist(self.resolve(self.r.hkeys('F2B_BLACKLIST')))
+
+  def isolate(self):
+    if self.env.get('DISABLE_NETFILTER_ISOLATION_RULE', '').lower() in ('y', 'yes'):
+      self.logger.logInfo('Skipping MAILCOW isolation')
+      return
+    self.logger.logInfo('Setting MAILCOW isolation')
+    self.tables.create_mailcow_isolation_rule(self.ISOLATION_INTERFACE, self.ISOLATION_PORTS, self.env.get('MAILCOW_REPLICA_IP', ''))
+
+  def resolve(self, entries):
+    """Networks of the entries; hostnames are resolved, on DNS errors to their last known addresses"""
+    resolver = dns.resolver.Resolver()
+    networks = set()
+    for entry in entries:
+      if BanManager.network(entry) is not None:
+        networks.add(entry)
+        continue
+      addresses = set()
+      for rdtype in ['A', 'AAAA']:
+        try:
+          answer = resolver.resolve(qname=entry, rdtype=rdtype, lifetime=3)
+        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+          continue
+        except dns.exception.DNSException as ex:
+          self.logger.logInfo('Could not resolve %s: %s' % (entry, ex))
+          addresses = self.resolved.get(entry, set())
+          break
+        addresses.update(rdata.to_text() for rdata in answer)
+      self.resolved[entry] = addresses
+      networks |= addresses
+    return networks
+
+  def snat_target(self, name, version):
+    value = self.env.get(name, '')
+    if not value or value == 'n':
+      return None
+    try:
+      if ipaddress.ip_address(value).version == version:
+        return value
+    except ValueError:
+      pass
+    self.logger.logWarn('%s is not a valid IPv%d address' % (value, version))
+    return None

data/Dockerfiles/netfilter/modules/Options.py

@@ -0,0 +1,120 @@
+import json
+import uuid
+
+
+class Options:
+  """Fail2ban options of the mailcow UI (Redis F2B_OPTIONS). The values are replaced as a whole and never changed
+  in place, so a thread that took them (options.values) keeps a consistent set."""
+
+  DEFAULTS = {
+    'ban_time': 1800,
+    'max_ban_time': 10000,
+    'ban_time_increment': True,
+    'max_attempts': 10,
+    'retry_window': 600,
+    'netban_ipv4': 32,
+    'netban_ipv6': 128,
+    'manage_external': 0,
+  }
+  # Single keys of old installations, read while F2B_OPTIONS does not exist
+  LEGACY_KEYS = {
+    'ban_time': 'F2B_BAN_TIME',
+    'max_ban_time': 'F2B_MAX_BAN_TIME',
+    'ban_time_increment': 'F2B_BAN_TIME_INCREMENT',
+    'max_attempts': 'F2B_MAX_ATTEMPTS',
+    'retry_window': 'F2B_RETRY_WINDOW',
+    'netban_ipv4': 'F2B_NETBAN_IPV4',
+    'netban_ipv6': 'F2B_NETBAN_IPV6',
+  }
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.values = self.normalize({})
+
+  def refresh(self, r):
+    """Reads the options of the UI; options it lacks are added to Redis once, existing ones are never overwritten"""
+    raw = r.get('F2B_OPTIONS')
+    if raw:
+      try:
+        stored = json.loads(raw)
+      except ValueError as ex:
+        self.logger.logWarn('F2B_OPTIONS is no valid JSON, keeping the current options: %s' % ex)
+        return
+      if not isinstance(stored, dict):
+        self.logger.logWarn('F2B_OPTIONS is no JSON object, keeping the current options')
+        return
+    else:
+      stored = {key: r.get(name) for key, name in self.LEGACY_KEYS.items()}
+      stored = {key: value for key, value in stored.items() if value is not None}
+
+    values = self.normalize(stored)
+    self.values = values
+    # First start or old installation: store the complete set, else only what the UI has not saved yet
+    missing = {key: value for key, value in values.items() if not raw or stored.get(key) is None}
+    if missing:
+      self.add_missing(r, missing)
+
+  def ban_time(self, ban_counter):
+    """Seconds of the next ban of a network that was banned ban_counter times before"""
+    values = self.values
+    ban_time = values['ban_time']
+    if values['ban_time_increment']:
+      # the exponent is capped, max_ban_time limits the result anyway
+      ban_time = ban_time * 2 ** min(ban_counter, 32)
+    return max(values['ban_time'], min(ban_time, values['max_ban_time']))
+
+  @classmethod
+  def normalize(cls, stored):
+    values = dict(stored)
+    for key, default in cls.DEFAULTS.items():
+      if isinstance(default, bool):
+        values[key] = cls.to_bool(stored.get(key), default)
+      else:
+        values[key] = cls.to_int(stored.get(key), default)
+    values['ban_time'] = max(1, values['ban_time'])
+    values['max_ban_time'] = max(1, values['max_ban_time'])
+    values['max_attempts'] = max(1, values['max_attempts'])
+    values['retry_window'] = max(1, values['retry_window'])
+    # A short prefix would ban whole regions or everything (/0)
+    values['netban_ipv4'] = min(32, max(8, values['netban_ipv4']))
+    values['netban_ipv6'] = min(128, max(8, values['netban_ipv6']))
+    values['manage_external'] = 1 if values['manage_external'] == 1 else 0
+    if not values.get('banlist_id'):
+      values['banlist_id'] = str(uuid.uuid4())
+    return values
+
+  @staticmethod
+  def to_int(value, default):
+    try:
+      return int(value)
+    except (TypeError, ValueError):
+      return default
+
+  @staticmethod
+  def to_bool(value, default):
+    if isinstance(value, bool):
+      return value
+    if isinstance(value, (int, float)):
+      return value != 0
+    if isinstance(value, str):
+      return value.strip().lower() in ('1', 'true', 'yes', 'on')
+    return default
+
+  def add_missing(self, r, missing):
+    # WATCH keeps a save of the UI between reading and writing
+    def fill(pipe):
+      raw = pipe.get('F2B_OPTIONS')
+      current = json.loads(raw) if raw else {}
+      if not isinstance(current, dict):
+        return
+      added = {key: value for key, value in missing.items() if current.get(key) is None}
+      if not added:
+        return
+      current.update(added)
+      pipe.multi()
+      pipe.set('F2B_OPTIONS', json.dumps(current, ensure_ascii=False))
+
+    try:
+      r.transaction(fill, 'F2B_OPTIONS')
+    except Exception as ex:
+      self.logger.logWarn('Could not add default options to F2B_OPTIONS: %s' % ex)

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -0,0 +1,88 @@
+import ipaddress
+import json
+import re
+
+
+class Regexes:
+  """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
+
+  DEFAULTS = {
+    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
+    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
+    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
+    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
+    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
+    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
+    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
+    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+  }
+  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
+  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
+  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.rules = []
+    self.source = None
+
+  def refresh(self, r):
+    """Compiles the rules of the UI again when they changed; invalid rules are skipped"""
+    raw = r.get('F2B_REGEX')
+    if not raw:
+      raw = json.dumps(self.DEFAULTS, ensure_ascii=False)
+      r.set('F2B_REGEX', raw, nx=True)
+    if raw == self.source:
+      return
+    try:
+      stored = json.loads(raw)
+    except ValueError as ex:
+      self.logger.logWarn('F2B_REGEX is no valid JSON, keeping the current rules: %s' % ex)
+      return
+    if not isinstance(stored, dict):
+      self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
+      return
+
+    rules = []
+    for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
+      try:
+        rules.append((str(rule_id), re.compile(regex)))
+      except (re.error, TypeError) as ex:
+        self.logger.logWarn('Skipping regex of rule id %s: %s' % (rule_id, ex))
+    self.rules = rules
+    self.source = raw
+
+  def match(self, line):
+    """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    for rule_id, regex in self.rules:
+      result = regex.search(line)
+      if result and result.lastindex and result.group(1):
+        return rule_id, result.group(1)
+    return None
+
+  @classmethod
+  def temporary_failure(cls, line):
+    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
+    result = cls.TEMPORARY_FAILURE.search(line)
+    return result.group(1) if result else None
+
+  @staticmethod
+  def parse_ip(text):
+    """IP address of a captured address, IPv4-mapped IPv6 as IPv4; a trailing colon (e.g. "1.2.3.4:") is
+    tolerated. None if the text is no address."""
+    for candidate in (text, text.rstrip(':')):
+      try:
+        ip = ipaddress.ip_address(candidate)
+      except ValueError:
+        continue
+      if ip.version == 6 and ip.ipv4_mapped:
+        return ip.ipv4_mapped
+      return ip
+    return None
+
+  @staticmethod
+  def sort_key(rule_id):
+    try:
+      return (0, int(rule_id))
+    except (TypeError, ValueError):
+      return (1, str(rule_id))

data/Dockerfiles/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:alpine
+FROM nginx:1.30.6-alpine
 LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 ENV PIP_BREAK_SYSTEM_PACKAGES=1

data/Dockerfiles/olefy/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/olefy/olefy.py

@@ -67,7 +67,7 @@ logger.debug('olefy listen address string: {} (type {})'.format(olefy_listen_add
 if not olefy_listen_addr_string:
     olefy_listen_addr = ""
 else:
-    addr_re = re.compile('[\[" \]]')
+    addr_re = re.compile(r'[\[" \]]')
     olefy_listen_addr = addr_re.sub('', olefy_listen_addr_string.replace("'", "")).split(',')
 
 # log runtime variables
@@ -206,8 +206,9 @@ class AIO(asyncio.Protocol):
         self.transport.close()
 
 
-# start the listeners
-loop = asyncio.get_event_loop()
+# start the listeners, Python >= 3.14 no longer creates a loop in get_event_loop()
+loop = asyncio.new_event_loop()
+asyncio.set_event_loop(loop)
 # each client connection will create a new protocol instance
 coro = loop.create_server(AIO, olefy_listen_addr, olefy_listen_port)
 server = loop.run_until_complete(coro)

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,4 +1,4 @@
-FROM php:8.2-fpm-alpine3.21
+FROM php:8.2-fpm-alpine3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
@@ -7,13 +7,13 @@ ARG APCU_PECL_VERSION=5.1.28
 # renovate: datasource=github-tags depName=Imagick/imagick versioning=semver-coerced extractVersion=(?<version>.*)$
 ARG IMAGICK_PECL_VERSION=3.8.1
 # renovate: datasource=github-tags depName=php/pecl-mail-mailparse versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG MAILPARSE_PECL_VERSION=3.1.9
+ARG MAILPARSE_PECL_VERSION=3.2.0
 # renovate: datasource=github-tags depName=php-memcached-dev/php-memcached versioning=semver-coerced extractVersion=^v(?<version>.*)$
 ARG MEMCACHED_PECL_VERSION=3.4.0
 # renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?<version>.*)$
 ARG REDIS_PECL_VERSION=6.3.0
 # renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?<version>.*)$
-ARG COMPOSER_VERSION=2.9.5
+ARG COMPOSER_VERSION=2.10.2
 
 RUN apk add -U --no-cache autoconf \
   aspell-dev \
@@ -47,6 +47,7 @@ RUN apk add -U --no-cache autoconf \
   libressl-dev \
   librsvg \
   libtool \
+  libwebp \
   libwebp-dev \
   libxml2-dev \
   libxpm \
@@ -63,13 +64,16 @@ RUN apk add -U --no-cache autoconf \
   samba-client \
   zlib-dev \
   tzdata \
-  && pecl install APCu-${APCU_PECL_VERSION} \
-  && pecl install imagick-${IMAGICK_PECL_VERSION} \
-  && pecl install mailparse-${MAILPARSE_PECL_VERSION} \
-  && pecl install memcached-${MEMCACHED_PECL_VERSION} \
-  && pecl install redis-${REDIS_PECL_VERSION} \
-  && docker-php-ext-enable apcu imagick memcached mailparse redis \
-  && pecl clear-cache \
+  && docker-php-source extract \
+  # PECL extensions from their GitHub tags (pecl.php.net is not reliable enough for builds)
+  && mkdir -p /usr/src/php/ext/apcu /usr/src/php/ext/imagick /usr/src/php/ext/mailparse /usr/src/php/ext/memcached /usr/src/php/ext/redis \
+  && curl -fsSL https://github.com/krakjoe/apcu/archive/refs/tags/v${APCU_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/apcu \
+  && curl -fsSL https://github.com/Imagick/imagick/archive/refs/tags/${IMAGICK_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/imagick \
+  && sed -i "s/@PACKAGE_VERSION@/${IMAGICK_PECL_VERSION}/" /usr/src/php/ext/imagick/php_imagick.h \
+  && curl -fsSL https://github.com/php/pecl-mail-mailparse/archive/refs/tags/v${MAILPARSE_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/mailparse \
+  && curl -fsSL https://github.com/php-memcached-dev/php-memcached/archive/refs/tags/v${MEMCACHED_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/memcached \
+  && curl -fsSL https://github.com/phpredis/phpredis/archive/refs/tags/${REDIS_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/redis \
+  && docker-php-ext-install -j 4 apcu imagick mailparse memcached redis \
   && docker-php-ext-configure intl \
   && docker-php-ext-configure exif \
   && docker-php-ext-configure gd --with-freetype=/usr/include/ \  
@@ -83,6 +87,7 @@ RUN apk add -U --no-cache autoconf \
   && curl --silent --show-error https://getcomposer.org/installer | php -- --version=${COMPOSER_VERSION} \
   && mv composer.phar /usr/local/bin/composer \
   && chmod +x /usr/local/bin/composer \
+  && docker-php-source delete \
   && apk del --purge autoconf \
     aspell-dev \
     cyrus-sasl-dev \

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-function array_by_comma { local IFS=","; echo "$*"; }
-
 # Wait for containers
 while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for SQL..."
@@ -132,33 +130,6 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   done
   fi
 
-  # Set API options if env vars are not empty
-  if [[ ${API_ALLOW_FROM} != "invalid" ]] && [[ ! -z ${API_ALLOW_FROM} ]]; then
-    IFS=',' read -r -a API_ALLOW_FROM_ARR <<< "${API_ALLOW_FROM}"
-    declare -a VALIDATED_API_ALLOW_FROM_ARR
-    REGEX_IP6='^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$'
-    REGEX_IP4='^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/([0-9]|[1-2][0-9]|3[0-2]))?$'
-    for IP in "${API_ALLOW_FROM_ARR[@]}"; do
-      if [[ ${IP} =~ ${REGEX_IP6} ]] || [[ ${IP} =~ ${REGEX_IP4} ]]; then
-        VALIDATED_API_ALLOW_FROM_ARR+=("${IP}")
-      fi
-    done
-    VALIDATED_IPS=$(array_by_comma ${VALIDATED_API_ALLOW_FROM_ARR[*]})
-    if [[ ! -z ${VALIDATED_IPS} ]]; then
-      if [[ ${API_KEY} != "invalid" ]] && [[ ! -z ${API_KEY} ]]; then
-        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-DELETE FROM api WHERE access = 'rw';
-INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY}", "1", "${VALIDATED_IPS}", "rw");
-EOF
-      fi
-      if [[ ${API_KEY_READ_ONLY} != "invalid" ]] && [[ ! -z ${API_KEY_READ_ONLY} ]]; then
-        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-DELETE FROM api WHERE access = 'ro';
-INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY_READ_ONLY}", "1", "${VALIDATED_IPS}", "ro");
-EOF
-      fi
-    fi
-  fi
 
   # Create events (master only, STATUS for event on slave will be SLAVESIDE_DISABLED)
   mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF

data/Dockerfiles/postfix-tlspol/Dockerfile

@@ -1,17 +1,17 @@
-FROM golang:1.25-bookworm AS builder
+FROM golang:1.26-trixie AS builder
 WORKDIR /src
 
 ENV CGO_ENABLED=0 \
     GO111MODULE=on \
 		NOOPT=1 \
-    VERSION=1.8.22
+    VERSION=1.14.0
 
 RUN git clone --branch v${VERSION} https://github.com/Zuplu/postfix-tlspol && \
     cd /src/postfix-tlspol && \
     scripts/build.sh build-only
 
 
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive

data/Dockerfiles/postfix-tlspol/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/postfix-tlspol/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/postfix/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/postfix/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/postfix/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/rspamd/Dockerfile

@@ -2,7 +2,7 @@ FROM debian:trixie-slim
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG RSPAMD_VER=rspamd_3.14.2-82~90302bc
+ARG RSPAMD_VER=rspamd_4.2.3-1~50bde75
 ARG CODENAME=trixie
 ENV LC_ALL=C
 

data/Dockerfiles/sogo/Dockerfile

@@ -1,9 +1,9 @@
 # SOGo built from source to enable security patch application
 # Repository: https://github.com/Alinto/sogo
-# Version: SOGo-5.12.4
+# Version: SOGo-5.12.12
 #
 # Applied security patches:
-# - 16ab99e7cf8db2c30b211f0d5e338d7f9e3a9efb: XSS vulnerability in theme parameter
+# -
 #
 # To add new patches, modify SOGO_SECURITY_PATCHES ARG below with space-separated commit hashes
 
@@ -12,12 +12,12 @@ FROM debian:bookworm
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SOGO_VERSION=SOGo-5.12.4
-ARG SOPE_VERSION=SOPE-5.12.4
+ARG SOGO_VERSION=SOGo-5.12.12
+ARG SOPE_VERSION=SOPE-5.12.12
 # Security patches to apply (space-separated commit hashes)
-ARG SOGO_SECURITY_PATCHES="16ab99e7cf8db2c30b211f0d5e338d7f9e3a9efb"
+ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.19
+ARG GOSU_VERSION=1.20
 ENV LC_ALL=C
 
 # Install dependencies, build SOPE and SOGo, then clean up (all in one layer to minimize image size)
@@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git \
     build-essential \
     gobjc \
+    pkg-config \
     gnustep-make \
     gnustep-base-runtime \
     libgnustep-base-dev \
@@ -40,6 +41,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libcurl4-openssl-dev \
     libzip-dev \
     libytnef0-dev \
+    libwbxml2-dev \
     curl \
     ca-certificates \
     # Runtime dependencies
@@ -68,6 +70,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libcurl4 \
     libzip4 \
     libytnef0 \
+    libwbxml2-1 \
   # Download gosu
   && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
@@ -97,6 +100,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   && ./configure --disable-debug --disable-strip \
   && make -j$(nproc) \
   && make install \
+  && cd /tmp/sogo/ActiveSync \
+  && . /usr/share/GNUstep/Makefiles/GNUstep.sh \
+  && make -j$(nproc) install \
   && cd / \
   && rm -rf /tmp/sogo \
   # Strip binaries
@@ -146,8 +152,8 @@ RUN echo "/usr/lib64" > /etc/ld.so.conf.d/sogo.conf \
 # Create sogo user and group
 RUN groupadd -r -g 999 sogo \
   && useradd -r -u 999 -g sogo -d /var/lib/sogo -s /bin/bash -c "SOGo Daemon" sogo \
-  && mkdir -p /var/lib/sogo /var/run/sogo /var/log/sogo \
-  && chown -R sogo:sogo /var/lib/sogo /var/run/sogo /var/log/sogo
+  && mkdir -p /var/lib/sogo /var/run/sogo /var/log/sogo /var/spool/sogo \
+  && chown -R sogo:sogo /var/lib/sogo /var/run/sogo /var/log/sogo /var/spool/sogo
 
 # Create symlinks for SOGo binaries
 RUN ln -s /usr/local/sbin/sogod /usr/sbin/sogod \

data/Dockerfiles/sogo/acl.diff

@@ -1,5 +1,5 @@
---- /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:57.987504204 +0200
-+++ /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:35.918291298 +0200
+--- /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:57.987504204 +0200
++++ /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:35.918291298 +0200
 @@ -46,7 +46,7 @@
            </md-item-template>
          </md-autocomplete>

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -13,13 +13,11 @@ do
   sleep 3
 done
 
-# Wait for updated schema
+# Wait for updated schema (DB_VERSION from docker-compose.yml)
 DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
-while [[ "${DBV_NOW}" != "${DBV_NEW}" ]]; do
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
   echo "Waiting for schema update..."
   DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-  DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
   sleep 5
 done
 echo "DB schema is ${DBV_NOW}"
@@ -130,18 +128,22 @@ chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist
 # Patch ACLs
 #if [[ ${ACL_ANYONE} == 'allow' ]]; then
 #  #enable any or authenticated targets for ACL
-#  if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-#    patch -R /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+#  if patch -R -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+#    patch -R /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
 #  fi
 #else
 #  #disable any or authenticated targets for ACL
-#  if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-#    patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+#  if patch -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+#    patch /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
 #  fi
 #fi
 
-if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff > /dev/null; then
-  patch -R /usr/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff;
+# Apply custom UI patch (reverse patch to ADD buttons)
+if patch -R -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff > /dev/null; then
+  echo "Applying navMailcowBtns patch (reverse to add buttons)..."
+  patch -R /usr/local/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff;
+else
+  echo "navMailcowBtns patch already applied or cannot be applied"
 fi
 
 # Rename custom logo, if any
@@ -149,7 +151,7 @@ fi
 
 # Rsync web content
 echo "Syncing web content with named volume"
-rsync -a /usr/lib/GNUstep/SOGo/. /sogo_web/
+rsync -a /usr/local/lib/GNUstep/SOGo/. /sogo_web/
 
 # Chown backup path
 chown -R sogo:sogo /sogo_backup

data/Dockerfiles/unbound/Dockerfile

@@ -1,12 +1,16 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
+# install unbound from alpine:edge to get security patches
+# minimum version is pinned
+RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main "unbound>=1.26.2-r0"
+
+# install other packages from regular alpine stable repo
 RUN apk add --update --no-cache \
 	curl \
 	bind-tools \
 	coreutils \
-	unbound \
 	bash \
 	openssl \
 	drill \

data/Dockerfiles/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.23
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
@@ -37,5 +37,6 @@ RUN apk add --update \
 COPY watchdog.sh /watchdog.sh
 COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
 COPY check_dns.sh /usr/lib/mailcow/check_dns.sh
+COPY client.cnf /etc/my.cnf.d/client.cnf
 
 CMD ["/watchdog.sh"]

data/Dockerfiles/watchdog/client.cnf

@@ -0,0 +1,3 @@
+[client]
+ssl = false
+ssl-verify-server-cert = false

data/Dockerfiles/watchdog/watchdog.sh

@@ -38,7 +38,7 @@ if [[ ! -p /tmp/com_pipe ]]; then
 fi
 
 # Wait for containers
-while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
+while ! mariadb-admin status --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for SQL..."
   sleep 2
 done
@@ -359,8 +359,8 @@ mysql_checks() {
   while [ ${err_count} -lt ${THRESHOLD} ]; do
     touch /tmp/mysql-mailcow; echo "$(tail -50 /tmp/mysql-mailcow)" > /tmp/mysql-mailcow
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql -f /etc/my.cnf.d/client.cnf -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql_query -f /etc/my.cnf.d/client.cnf -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -384,7 +384,7 @@ mysql_repl_checks() {
   while [ ${err_count} -lt ${THRESHOLD} ]; do
     touch /tmp/mysql_repl_checks; echo "$(tail -50 /tmp/mysql_repl_checks)" > /tmp/mysql_repl_checks
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_mysql_slavestatus.sh -S /var/run/mysqld/mysqld.sock -u root -p ${DBROOT} 2>> /tmp/mysql_repl_checks 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql_slavestatus.sh -o /etc/my.cnf.d/client.cnf -S /var/run/mysqld/mysqld.sock -u root -p ${DBROOT} 2>> /tmp/mysql_repl_checks 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "MySQL/MariaDB replication" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}

data/conf/dovecot/auth/mailcowauth.php

@@ -101,7 +101,16 @@ if ($result === false){
   // Init Identity Provider
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
-  $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+  try {
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+  } catch (\Mailcow\Auth\IdpUnavailable $e) {
+    // temporary failure for Dovecot, no failed login
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
+    http_response_code(503); // Service Unavailable
+    echo json_encode($return);
+    session_destroy();
+    exit;
+  }
   if ($result) {
     error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
     set_sasl_log($post['username'], $post['real_rip'], $post['service']);

data/conf/dovecot/auth/passwd-verify.lua

@@ -1,5 +1,4 @@
 function auth_password_verify(request, password)
-  request.domain = request.auth_user:match("@(.+)") or nil
   if request.domain == nil then
     return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
   end
@@ -10,10 +9,10 @@ function auth_password_verify(request, password)
   https.TIMEOUT = 30
 
   local req = {
-    username = request.auth_user,
+    username = request.user,
     password = password,
-    real_rip = request.remote_ip,
-    service = request.protocol
+    real_rip = request.real_rip,
+    service = request.service
   }
   local req_json = json.encode(req)
   local res = {}
@@ -33,7 +32,23 @@ function auth_password_verify(request, password)
   -- Returning PASSDB_RESULT_PASSWORD_MISMATCH will reset the user's auth cache entry.
   -- Returning PASSDB_RESULT_INTERNAL_FAILURE keeps the existing cache entry,
   -- even if the TTL has expired. Useful to avoid cache eviction during backend issues.
+
+  -- On a network-level failure (nginx unreachable, DNS failure, timeout) https.request
+  -- returns nil plus an error string, so c is not a numeric HTTP status code. Treat this
+  -- as a backend outage and keep the cache entry, rather than wiping it as a mismatch.
+  if type(c) ~= "number" then
+    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unreachable"
+  end
+
+  -- 5xx: the backend or the identity provider cannot answer, a temporary failure and no failed login
+  if c >= 500 then
+    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unavailable"
+  end
+
   if c ~= 200 and c ~= 401 then
+    dovecot.i_info("HTTP request failed with " .. tostring(c) .. " for user " .. request.user)
     return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Upstream error"
   end
 
@@ -46,7 +61,7 @@ function auth_password_verify(request, password)
   end
 
   if response_json.success == true then
-    return dovecot.auth.PASSDB_RESULT_OK, { msg = "" }
+    return dovecot.auth.PASSDB_RESULT_OK, ""
   end
 
   return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Failed to authenticate"
@@ -55,7 +70,3 @@ end
 function auth_passdb_lookup(req)
    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, ""
 end
-
-function auth_passdb_get_cache_key()
-  return "%{protocol}:%{user | username}\t:%{password}"
-end

data/conf/dovecot/conf.d/05-core.conf

@@ -1,3 +0,0 @@
-# /etc/dovecot/conf.d/05-core.conf
-# Core, single-line settings that don't fit elsewhere.
-recipient_delimiter = +

data/conf/dovecot/conf.d/10-logging.conf

@@ -1,13 +0,0 @@
-# /etc/dovecot/conf.d/10-logging.conf
-# Logging and debug.
-#mail_debug = yes
-#auth_debug = yes
-#log_debug = category=fts-flatcurve
-log_path = syslog
-log_timestamp = "%Y-%m-%d %H:%M:%S "
-login_log_format_elements = "user=<%{user}> method=%{mechanism} rip=%{remote_ip} lip=%{local_ip} mpid=%{mail_pid} %{secured} session=<%{session}>"
-
-# Mail event logging.
-mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
-mail_log_fields = uid box msgid size
-mail_log_cached_only = yes

data/conf/dovecot/conf.d/10-mail.conf

@@ -1,10 +0,0 @@
-# /etc/dovecot/conf.d/10-mail.conf
-# Mail storage paths and core mail settings.
-mail_home = /var/vmail/%{user | domain }/%{user | username }
-mail_driver = maildir
-mail_path = ~/Maildir
-mail_index_path = /var/vmail_index/%{user}
-mail_plugins = </etc/dovecot/mail_plugins
-mail_shared_explicit_inbox = yes
-mailbox_list_storage_escape_char = "\\"
-mail_prefetch_count = 30

data/conf/dovecot/conf.d/10-ssl.conf

@@ -1,13 +0,0 @@
-# /etc/dovecot/conf.d/10-ssl.conf
-# TLS/SSL settings.
-ssl_min_protocol = TLSv1.2
-ssl_cipher_list = ALL:!ADH:!LOW:!SSLv2:!SSLv3:!EXP:!aNULL:!eNULL:!3DES:!MD5:!PSK:!DSS:!RC4:!SEED:!IDEA:+HIGH:+MEDIUM
-ssl_options = no_ticket
-#ssl_dh_parameters_length = 2048
-
-ssl_server {
-  prefer_ciphers = server
-  dh_file = /etc/ssl/mail/dhparams.pem
-  cert_file = /etc/ssl/mail/cert.pem
-  key_file = /etc/ssl/mail/key.pem
-}

data/conf/dovecot/conf.d/11-sql.conf

@@ -1,3 +0,0 @@
-# /etc/dovecot/conf.d/11-sql.conf
-# Default SQL driver used by SQL-based dicts/userdb.
-sql_driver = mysql

data/conf/dovecot/conf.d/12-mysql.conf

@@ -1,8 +0,0 @@
-# Autogenerated by mailcow - DO NOT TOUCH!
-mysql /var/run/mysqld/mysqld.sock {
-  dbname=mailcow
-  user=mailcow
-  password=D8O9BIivJc7Pb2VCfpAeLbAzUOZ0
-
-  ssl = no
-}

data/conf/dovecot/conf.d/12-storage-attachments.conf

@@ -1,7 +0,0 @@
-# /etc/dovecot/conf.d/12-storage-attachments.conf
-# External attachment storage.
-fs mail_ext_attachment {
-  fs_driver = posix
-  mail_ext_attachment_path = /var/attachments
-  mail_ext_attachment_min_size = 128k
-}

data/conf/dovecot/conf.d/15-performance.conf

@@ -1,10 +0,0 @@
-# /etc/dovecot/conf.d/15-performance.conf
-# Performance and mailbox tuning.
-# Enable only when you do not manually touch cur/.
-maildir_very_dirty_syncs = yes
-
-# NFS examples | Only modify if using NFS!:
-#mm ap_disable = yes
-#mail_fsync = always
-#mail_nfs_index = yes
-#mail_nfs_storage = yes

data/conf/dovecot/conf.d/20-auth.conf

@@ -1,40 +0,0 @@
-# /etc/dovecot/conf.d/20-auth.conf
-# Authentication mechanisms, master/user separation, passdb chain, auth cache.
-auth_mechanisms = plain login
-auth_allow_cleartext = yes
-auth_master_user_separator = *
-
-auth_cache_verify_password_with_worker = yes
-auth_cache_negative_ttl = 60s
-auth_cache_ttl = 300s
-auth_cache_size = 10M
-auth_verbose_passwords = sha1:6
-
-# 1) Lua password verification (blocking, return mapping).
-passdb lua {
-  driver = lua
-  lua_file = /etc/dovecot/auth/passwd-verify.lua
-  lua_settings {
-    blocking=yes
-    result_success = return-ok
-    result_failure = continue
-    result_internalfail = continue
-  }
-}
-
-# 2) Master password for master user logins.
-passdb master {
-  driver = passwd-file
-  passwd_file_path = /etc/dovecot/dovecot-master.passwd 
-  master = yes
-  skip = authenticated
-}
-
-# 3) Mandatory return layer: empty Lua (e.g. for forced reset).
-passdb empty-lua {
-  driver = lua
-  lua_file = /etc/dovecot/auth/passwd-verify.lua
-  lua_settings {
-    blocking = yes
-  }
-}

data/conf/dovecot/conf.d/20-userdb.conf

@@ -1,11 +0,0 @@
-# /etc/dovecot/conf.d/20-userdb.conf
-# User database chain.
-userdb passwd {
-  driver = passwd-file
-  passwd_file_path = /etc/dovecot/dovecot-master.userdb
-}
-
-userdb sql {
-  !include /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
-  skip = found
-}

data/conf/dovecot/conf.d/25-services.conf

@@ -1,144 +0,0 @@
-# /etc/dovecot/conf.d/25-services.conf
-# All service listeners and workers.
-
-# doveadm remote admin
-# Set doveadm_password in extra.conf.
-service doveadm {
-  inet_listener doveadm {
-    port = 12345
-  }
-  vsz_limit = 2048 MB
-}
-
-# dict
-service dict {
-  unix_listener dict {
-    mode = 0660
-    user = vmail
-    group = vmail
-  }
-}
-
-# log
-service log {
-  user = dovenull
-}
-
-# config socket
-service config {
-  unix_listener config {
-    user = root
-    group = vmail
-    mode = 0660
-  }
-}
-
-# anvil socket
-service anvil {
-  unix_listener anvil {
-    user = vmail
-    group = vmail
-    mode = 0660
-  }
-}
-
-# auth sockets and inet
-service auth {
-  inet_listener auth-inet {
-    port = 10001
-  }
-  unix_listener auth-master {
-    mode = 0600
-    user = vmail
-  }
-  unix_listener auth-userdb {
-    mode = 0600
-    user = vmail
-  }
-  vsz_limit = 2G
-}
-
-# managesieve login
-service managesieve-login {
-  inet_listener sieve {
-    port = 4190
-  }
-  inet_listener sieve_haproxy {
-    port = 14190
-    haproxy = yes
-  }
-  service_restart_request_count = 1
-  process_min_avail = 2
-  vsz_limit = 1G
-}
-
-# imap login
-service imap-login {
-  service_restart_request_count = 1
-  process_min_avail = 2
-  process_limit = 10000
-  vsz_limit = 1G
-  user = dovenull
-  inet_listener imap_haproxy {
-    port = 10143
-    haproxy = yes
-  }
-  inet_listener imaps_haproxy {
-    port = 10993
-    ssl = yes
-    haproxy = yes
-  }
-}
-
-# pop3 login
-service pop3-login {
-  service_restart_request_count = 1
-  process_min_avail = 1
-  vsz_limit = 1G
-  inet_listener pop3_haproxy {
-    port = 10110
-    haproxy = yes
-  }
-  inet_listener pop3s_haproxy {
-    port = 10995
-    ssl = yes
-    haproxy = yes
-  }
-}
-
-# imap worker
-service imap {
-  executable = imap
-  user = vmail
-  vsz_limit = 1G
-}
-
-# managesieve worker
-service managesieve {
-  process_limit = 256
-}
-
-# lmtp
-service lmtp {
-  inet_listener lmtp-inet {
-    port = 24
-  }
-  user = vmail
-}
-
-# quota warning hook
-service quota-warning {
-  executable = script /usr/local/bin/quota_notify.py
-  user = vmail
-  unix_listener quota-warning {
-    user = vmail
-  }
-}
-
-# stats
-service stats {
-  unix_listener stats-writer {
-    mode = 0660
-    user = vmail
-  }
-}

data/conf/dovecot/conf.d/30-protocols.conf

@@ -1,17 +0,0 @@
-# /etc/dovecot/conf.d/30-protocols.conf
-# IMAP protocol specifics.
-protocol imap {
-  mail_plugins = </etc/dovecot/mail_plugins_imap
-  imap_metadata = yes
-}
-
-# LMTP protocol specifics.
-protocol lmtp {
-  mail_plugins = </etc/dovecot/mail_plugins_lmtp
-  auth_socket_path = /var/run/dovecot/auth-master
-}
-
-# ManageSieve protocol specifics.
-protocol sieve {
-  managesieve_logout_format = bytes=%i/%o
-}

data/conf/dovecot/conf.d/35-fts.conf

@@ -1,45 +0,0 @@
-# mailcow FTS Flatcurve Settings, change them as you like.
-
-# Maximum term length can be set via the 'maxlen' argument (maxlen is
-# specified in bytes, not number of UTF-8 characters)
-language_tokenizer_address_token_maxlen = 100
-language_tokenizer_generic_algorithm = simple
-language_tokenizer_generic_token_maxlen = 30
-
-# These are not flatcurve settings, but required for Dovecot FTS. See
-# Dovecot FTS Configuration link above for further information.
-language en {
-  default = yes
-  language_filters = lowercase snowball english-possessive stopwords
-}
-
-language de {
-  language_filters = lowercase snowball stopwords
-}
-
-language es {
-  language_filters = lowercase snowball stopwords
-}
-
-language_tokenizers = generic email-address
-
-fts_search_timeout = 300s
-
-fts_autoindex = yes
-# Tweak this setting if you only want to ensure big and frequent folders are indexed, not all.
-fts_autoindex_max_recent_msgs = 20
-fts flatcurve {
-  substring_search = no
-}
-
-### THIS PART WILL BE CHANGED BY MODIFYING mailcow.conf AUTOMATICALLY DURING RUNTIME! ###
-
-service indexer-worker {
-  # Max amount of simultaniously running indexer jobs.
-  process_limit=1
-
-  # Max amount of RAM used by EACH indexer process.
-  vsz_limit=128 MB
-}
-
-### THIS PART WILL BE CHANGED BY MODIFYING mailcow.conf AUTOMATICALLY DURING RUNTIME! ###

data/conf/dovecot/conf.d/40-acl.conf

@@ -1,12 +0,0 @@
-# /etc/dovecot/conf.d/40-acl.conf
-# ACL and shared mailboxes.
-imap_acl_allow_anyone = </etc/dovecot/acl_anyone
-
-acl_sharing_map {
-  dict file {
-    path = /var/vmail/shared-mailboxes.db
-  }
-}
-
-acl_driver = vfile
-acl_user = %{user}

data/conf/dovecot/conf.d/40-attributes.conf

@@ -1,7 +0,0 @@
-# /etc/dovecot/conf.d/40-attributes.conf
-# User/mail attributes.
-mail_attribute {
-  dict file {
-    path = /etc/dovecot/dovecot-attributes
-  }
-}

data/conf/dovecot/conf.d/50-quota.conf

@@ -1,25 +0,0 @@
-# /etc/dovecot/conf.d/50-quota.conf
-# Quota configuration and notifications.
-quota "User quota" {
-  driver = count
-
-  warning warn-95 {
-    quota_storage_percentage = 95
-    execute quota-warning {
-      args = 95 %{user}
-    }
-  }
-
-  warning warn-80 {
-    quota_storage_percentage = 80
-    execute quota-warning {
-      args = 80 %{user}
-    }
-  }
-}
-
-quota_clone {
-  dict proxy {
-    name = mysql_quota
-  }
-}

data/conf/dovecot/conf.d/60-sieve-pipeline.conf

@@ -1,97 +0,0 @@
-# /etc/dovecot/conf.d/60-sieve-pipeline.conf
-# Complete Sieve pipeline: personal/global scripts, plugins, limits, training.
-
-# Global before/after (file and dict)
-sieve_script before {
-  type = before
-  driver = file
-  path = /var/vmail/sieve/global_sieve_before.sieve
-}
-
-sieve_script before2 {
-  type = before
-  driver = dict
-  name = active
-  dict proxy {
-    name = sieve_before
-  }
-  bin_path = /var/vmail/sieve_before_bindir/%{user}
-}
-
-sieve_script after {
-  type = after
-  driver = file
-  path = /var/vmail/sieve/global_sieve_after.sieve
-}
-
-sieve_script after2 {
-  type = after
-  driver = dict
-  name = active
-  dict proxy {
-    name = sieve_after
-  }
-  bin_path = /var/vmail/sieve_after_bindir/%{user}
-}
-
-# Personal scripts
-sieve_script personal {
-  type = personal
-  driver = file
-  path = ~/sieve
-  active_path = ~/.dovecot.sieve
-}
-
-# Plugins and behavior
-sieve_plugins = sieve_imapsieve sieve_extprograms
-sieve_vacation_send_from_recipient = yes
-sieve_redirect_envelope_from = recipient
-
-# IMAPSieve training
-imapsieve_from Junk {
-  sieve_script ham {
-    type = before
-    cause = copy
-    path = /usr/lib/dovecot/sieve/report-ham.sieve
-  }
-}
-mailbox Junk {
-  sieve_script spam {
-    type = before
-    cause = copy
-    path = /usr/lib/dovecot/sieve/report-spam.sieve
-  }
-}
-
-# Extprograms and extensions
-sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
-sieve_plugins {
-  sieve_extprograms = yes
-}
-sieve_global_extensions {
-  vnd.dovecot.pipe = yes
-  vnd.dovecot.execute = yes
-}
-
-# Limits and duplicate handling
-sieve_max_script_size = 1M
-sieve_max_redirects = 100
-sieve_max_actions = 101
-sieve_quota_script_count = 0
-sieve_quota_storage_size = 0
-sieve_vacation_min_period = 5s
-sieve_vacation_max_period = 365d
-sieve_vacation_default_period = 60s
-sieve_duplicate_default_period = 1m
-sieve_duplicate_max_period = 7d
-
-sieve_extensions {
-    vacation-seconds = yes
-    editheader = yes
-}
-
-# pipe sockets in /var/run/dovecot/sieve-pipe
-sieve_pipe_socket_dir = sieve-pipe
-
-# execute sockets in /var/run/dovecot/sieve-execute
-sieve_execute_socket_dir = sieve-execute

data/conf/dovecot/conf.d/70-crypto.conf

@@ -1,6 +0,0 @@
-# /etc/dovecot/conf.d/70-crypto.conf
-# Global mail-crypt keys.
-crypt_global_private_key global {
-  crypt_private_key_file = /mail_crypt/ecprivkey.pem
-}
-crypt_global_public_key_file = /mail_crypt/ecpubkey.pem

data/conf/dovecot/conf.d/80-compress.conf

@@ -1,3 +0,0 @@
-# /etc/dovecot/conf.d/80-compress.conf
-# Compression settings.
-mail_compress_write_method = lz4

data/conf/dovecot/conf.d/90-dict.conf

@@ -1,18 +0,0 @@
-# /etc/dovecot/conf.d/90-dict.conf
-# Dict declarations and SQL bindings.
-dict_server {
-  dict sieve_after {
-    driver = sql
-    !include /etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf
-  }
-
-  dict sieve_before {
-    driver = sql
-    !include /etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
-  }
-
-  dict mysql_quota {
-    driver = sql
-    !include /etc/dovecot/sql/dovecot-dict-sql-quota.conf
-  }
-}

data/conf/dovecot/conf.d/90-limits.conf

@@ -1,7 +0,0 @@
-# /etc/dovecot/conf.d/90-limits.conf
-# Connection and memory limits; doveadm port.
-mail_max_userip_connections = 500
-imap_max_line_length = 2 M
-default_client_limit = 10400
-default_vsz_limit = 1024 M
-doveadm_port = 12345

data/conf/dovecot/conf.d/99-includes.conf

@@ -1,22 +0,0 @@
-# /etc/dovecot/conf.d/99-includes.conf
-# Late includes and site-specific bits.
-
-# Mailbox layout includes (if used)
-!include /etc/dovecot/dovecot.folders.conf
-
-# Optional replication
-!include_try /etc/dovecot/mail_replica.conf
-
-# Existing includes you already had
-!include_try /etc/dovecot/sni.conf
-!include_try /etc/dovecot/sogo_trusted_ip.conf
-!include_try /etc/dovecot/shared_namespace.conf
-!include_try /etc/dovecot/conf.d/fts.conf
-
-# Remote auth override
-remote 127.0.0.1 {
-  auth_allow_cleartext = yes
-}
-
-# Outbound submission target
-submission_host = postfix:588

data/conf/dovecot/conf.d/fts.conf

@@ -0,0 +1,37 @@
+# mailcow FTS Flatcurve Settings, change them as you like.
+plugin {
+    fts_autoindex = yes
+    fts_autoindex_exclude = \Junk
+    fts_autoindex_exclude2 = \Trash
+    # Tweak this setting if you only want to ensure big and frequent folders are indexed, not all.
+    fts_autoindex_max_recent_msgs = 20
+    fts = flatcurve
+
+    # Maximum term length can be set via the 'maxlen' argument (maxlen is
+    # specified in bytes, not number of UTF-8 characters)
+    fts_tokenizer_email_address = maxlen=100
+    fts_tokenizer_generic = algorithm=simple maxlen=30
+
+    # These are not flatcurve settings, but required for Dovecot FTS. See
+    # Dovecot FTS Configuration link above for further information.
+    fts_languages = en es de
+    fts_tokenizers = generic email-address
+
+    # OPTIONAL: Recommended default FTS core configuration
+    fts_filters = normalizer-icu snowball stopwords
+    fts_filters_en = lowercase snowball english-possessive stopwords
+
+    fts_index_timeout = 300s
+}
+
+### THIS PART WILL BE CHANGED BY MODIFYING mailcow.conf AUTOMATICALLY DURING RUNTIME! ###
+
+service indexer-worker {
+  # Max amount of simultaniously running indexer jobs.
+  process_limit=1
+
+  # Max amount of RAM used by EACH indexer process.
+  vsz_limit=128 MB
+}
+
+### THIS PART WILL BE CHANGED BY MODIFYING mailcow.conf AUTOMATICALLY DURING RUNTIME! ###

data/conf/dovecot/dovecot.conf

@@ -1,34 +1,313 @@
-# /etc/dovecot/dovecot.conf
-# Base file kept minimal. All real config lives under conf.d/.
-dovecot_config_version = 2.4.0
-dovecot_storage_version = 2.4.0
+# --------------------------------------------------------------------------
+# Please create a file "extra.conf" for persistent overrides to dovecot.conf
+# --------------------------------------------------------------------------
+# LDAP example:
+#passdb {
+#  args = /etc/dovecot/ldap/passdb.conf
+#  driver = ldap
+#}
 
-listen = *,[::]
+auth_mechanisms = plain login
+#mail_debug = yes
+#auth_debug = yes
+#log_debug = category=fts-flatcurve # Activate Logging for Flatcurve FTS Searchings
+log_path = syslog
+disable_plaintext_auth = yes
+# Uncomment on NFS share
+#mmap_disable = yes
+#mail_fsync = always
+#mail_nfs_index = yes
+#mail_nfs_storage = yes
+login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c %k"
+mail_home = /var/vmail/%d/%n
+mail_location = maildir:~/
+mail_plugins = </etc/dovecot/mail_plugins
+mail_attachment_fs = crypt:set_prefix=mail_crypt_global:posix:
+mail_attachment_dir = /var/attachments
+mail_attachment_min_size = 128k
+# Significantly speeds up very large mailboxes, but is only safe to enable if
+# you do not manually modify the files in the `cur` directories in
+# mailcowdockerized_vmail-vol-1.
+# https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-performance/
+maildir_very_dirty_syncs = yes
+
+# Dovecot 2.2
+#ssl_protocols = !SSLv3
+# Dovecot 2.3
+ssl_min_protocol = TLSv1.2
+
+ssl_prefer_server_ciphers = yes
+ssl_cipher_list = ALL:!ADH:!LOW:!SSLv2:!SSLv3:!EXP:!aNULL:!eNULL:!3DES:!MD5:!PSK:!DSS:!RC4:!SEED:!IDEA:+HIGH:+MEDIUM
+
+# Default in Dovecot 2.3
+ssl_options = no_compression no_ticket
+
+# New in Dovecot 2.3
+ssl_dh = </etc/ssl/mail/dhparams.pem
+# Dovecot 2.2
+#ssl_dh_parameters_length = 2048
+log_timestamp = "%Y-%m-%d %H:%M:%S "
+recipient_delimiter = +
+auth_master_user_separator = *
+mail_shared_explicit_inbox = yes
+mail_prefetch_count = 30
+passdb {
+  driver = lua
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
+  result_success = return-ok
+  result_failure = continue
+  result_internalfail = continue
+}
+# try a master passwd
+passdb {
+  driver = passwd-file
+  args = /etc/dovecot/dovecot-master.passwd
+  master = yes
+  skip = authenticated
+}
+# check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
+# a return of the following passdb is mandatory
+# cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
+# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
+passdb {
+  driver = lua
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
+}
+# Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
+service doveadm {
+  inet_listener {
+    port = 12345
+  }
+  vsz_limit=2048 MB
+}
+!include /etc/dovecot/dovecot.folders.conf
 protocols = imap sieve lmtp pop3
+service dict {
+  unix_listener dict {
+    mode = 0660
+    user = vmail
+    group = vmail
+  }
+}
+service log {
+  user = dovenull
+}
+service config {
+  unix_listener config {
+    user = root
+    group = vmail
+    mode = 0660
+  }
+}
+service auth {
+  inet_listener auth-inet {
+    port = 10001
+  }
+  unix_listener auth-master {
+    mode = 0600
+    user = vmail
+  }
+  unix_listener auth-userdb {
+    mode = 0600
+    user = vmail
+  }
+  vsz_limit = 2G
+}
+service managesieve-login {
+  inet_listener sieve {
+    port = 4190
+  }
+  inet_listener sieve_haproxy {
+    port = 14190
+    haproxy = yes
+  }
+  service_count = 1
+  process_min_avail = 2
+  vsz_limit = 1G
+}
+service imap-login {
+  service_count = 1
+  process_min_avail = 2
+  process_limit = 10000
+  vsz_limit = 1G
+  user = dovenull
+  inet_listener imap_haproxy {
+    port = 10143
+    haproxy = yes
+  }
+  inet_listener imaps_haproxy {
+    port = 10993
+    ssl = yes
+    haproxy = yes
+  }
+}
+service pop3-login {
+  service_count = 1
+  process_min_avail = 1
+  vsz_limit = 1G
+  inet_listener pop3_haproxy {
+    port = 10110
+    haproxy = yes
+  }
+  inet_listener pop3s_haproxy {
+    port = 10995
+    ssl = yes
+    haproxy = yes
+  }
+}
+service imap {
+  executable = imap
+  user = vmail
+  vsz_limit = 1G
+}
+service managesieve {
+  process_limit = 256
+}
+service lmtp {
+  inet_listener lmtp-inet {
+    port = 24
+  }
+  user = vmail
+}
+listen = *,[::]
+ssl_cert = </etc/ssl/mail/cert.pem
+ssl_key = </etc/ssl/mail/key.pem
+userdb {
+  driver = passwd-file
+  args = /etc/dovecot/dovecot-master.userdb
+}
+userdb {
+  args = /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
+  driver = sql
+  skip = found
+}
+protocol imap {
+  mail_plugins = </etc/dovecot/mail_plugins_imap
+  imap_metadata = yes
+}
+mail_attribute_dict = file:%h/dovecot-attributes
+protocol lmtp {
+  mail_plugins = </etc/dovecot/mail_plugins_lmtp
+  auth_socket_path = /var/run/dovecot/auth-master
+}
+protocol sieve {
+  managesieve_logout_format = bytes=%i/%o
+}
+plugin {
+  # Allow "any" or "authenticated" to be used in ACLs
+  acl_anyone = </etc/dovecot/acl_anyone
+  acl_shared_dict = file:/var/vmail/shared-mailboxes.db
+  acl = vfile
+  acl_user = %u
+  quota = dict:Userquota::proxy::sqlquota
+  quota_rule2 = Trash:storage=+100%%
+  sieve = /var/vmail/sieve/%u.sieve
+  sieve_plugins = sieve_imapsieve sieve_extprograms
+  sieve_vacation_send_from_recipient = yes
+  sieve_redirect_envelope_from = recipient
+  # From elsewhere to Spam folder
+  imapsieve_mailbox1_name = Junk
+  imapsieve_mailbox1_causes = COPY
+  imapsieve_mailbox1_before = file:/usr/lib/dovecot/sieve/report-spam.sieve
+  # END
+  # From Spam folder to elsewhere
+  imapsieve_mailbox2_name = *
+  imapsieve_mailbox2_from = Junk
+  imapsieve_mailbox2_causes = COPY
+  imapsieve_mailbox2_before = file:/usr/lib/dovecot/sieve/report-ham.sieve
+  # END
+  master_user = %u
+  quota_warning = storage=95%% quota-warning 95 %u
+  quota_warning2 = storage=80%% quota-warning 80 %u
+  sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
+  sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
+  sieve_extensions = +notify +imapflags +vacation-seconds +editheader
+  sieve_max_script_size = 1M
+  sieve_max_redirects = 100
+  sieve_max_actions = 101
+  sieve_quota_max_scripts = 0
+  sieve_quota_max_storage = 0
+  listescape_char = "\\"
+  sieve_vacation_min_period = 5s
+  sieve_vacation_max_period = 0
+  sieve_vacation_default_period = 60s
+  sieve_before = /var/vmail/sieve/global_sieve_before.sieve
+  sieve_before2 = dict:proxy::sieve_before;name=active;bindir=/var/vmail/sieve_before_bindir
+  sieve_after = dict:proxy::sieve_after;name=active;bindir=/var/vmail/sieve_after_bindir
+  sieve_after2 = /var/vmail/sieve/global_sieve_after.sieve
+  sieve_duplicate_default_period = 1m
+  sieve_duplicate_max_period = 7d
 
-!include_try /etc/dovecot/conf.d/05-core.conf
-!include_try /etc/dovecot/conf.d/10-logging.conf
-!include_try /etc/dovecot/conf.d/10-mail.conf
-!include_try /etc/dovecot/conf.d/10-ssl.conf
-!include_try /etc/dovecot/conf.d/11-sql.conf
-!include_try /etc/dovecot/conf.d/12-mysql.conf
-!include_try /etc/dovecot/conf.d/12-storage-attachments.conf
-!include_try /etc/dovecot/conf.d/15-performance.conf
-!include_try /etc/dovecot/conf.d/20-auth.conf
-!include_try /etc/dovecot/conf.d/20-userdb.conf
-!include_try /etc/dovecot/conf.d/25-services.conf
-!include_try /etc/dovecot/conf.d/30-protocols.conf
-!include_try /etc/dovecot/conf.d/35-fts.conf
-!include_try /etc/dovecot/conf.d/40-acl.conf
-!include_try /etc/dovecot/conf.d/40-attributes.conf
-!include_try /etc/dovecot/conf.d/50-quota.conf
-!include_try /etc/dovecot/conf.d/60-sieve-pipeline.conf
-!include_try /etc/dovecot/conf.d/70-crypto.conf
-!include_try /etc/dovecot/conf.d/80-compress.conf
-!include_try /etc/dovecot/conf.d/80-mail-logging.conf
-!include_try /etc/dovecot/conf.d/90-limits.conf
-!include_try /etc/dovecot/conf.d/90-dict.conf
-!include_try /etc/dovecot/conf.d/99-includes.conf
+  # -- Global keys
+  mail_crypt_global_private_key = </mail_crypt/ecprivkey.pem
+  mail_crypt_global_public_key = </mail_crypt/ecpubkey.pem
+  mail_crypt_save_version = 2
 
-# Last: local overrides
-!include_try /etc/dovecot/extra.conf
+  # Enable compression while saving, lz4 Dovecot v2.3.17+
+  zlib_save = lz4
+
+  mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
+  mail_log_fields = uid box msgid size
+  mail_log_cached_only = yes
+
+  # Try set mail_replica
+  !include_try /etc/dovecot/mail_replica.conf
+}
+service quota-warning {
+  executable = script /usr/local/bin/quota_notify.py
+  # use some unprivileged user for executing the quota warnings
+  user = vmail
+  unix_listener quota-warning {
+    user = vmail
+  }
+}
+dict {
+  sqlquota = mysql:/etc/dovecot/sql/dovecot-dict-sql-quota.conf
+  sieve_after = mysql:/etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf
+  sieve_before = mysql:/etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf
+}
+remote 127.0.0.1 {
+  disable_plaintext_auth = no
+}
+submission_host = postfix:588
+mail_max_userip_connections = 500
+service stats {
+  unix_listener stats-writer {
+    mode = 0660
+    user = vmail
+  }
+}
+imap_max_line_length = 2 M
+auth_cache_verify_password_with_worker = yes
+auth_cache_negative_ttl = 60s
+auth_cache_ttl = 300s
+auth_cache_size = 10M
+auth_verbose_passwords = sha1:6
+service replicator {
+  process_min_avail = 1
+}
+service aggregator {
+  fifo_listener replication-notify-fifo {
+    user = vmail
+  }
+  unix_listener replication-notify {
+    user = vmail
+  }
+}
+service replicator {
+  unix_listener replicator-doveadm {
+    mode = 0666
+  }
+}
+replication_max_conns = 10
+doveadm_port = 12345
+replication_dsync_parameters = -d -l 30 -U -n INBOX
+# <Includes>
+!include_try /etc/dovecot/sni.conf
+!include_try /etc/dovecot/sogo_trusted_ip.conf
+!include_try /etc/dovecot/extra.conf
+!include_try /etc/dovecot/shared_namespace.conf
+!include_try /etc/dovecot/conf.d/fts.conf
+# </Includes>
+default_client_limit = 10400
+default_vsz_limit = 1024 M

data/conf/dovecot/dovecot.folders.conf

@@ -1,14 +1,10 @@
 namespace inbox {
   inbox = yes
+  location =
   separator = /
-  mailbox storage/* {
-    quota_storage_extra = 100M
-  }
   mailbox "Trash" {
     auto = subscribe
     special_use = \Trash
-    quota_storage_percentage = 100
-    fts_autoindex = no
   }
   mailbox "Deleted Messages" {
     special_use = \Trash
@@ -199,7 +195,6 @@ namespace inbox {
   mailbox "Junk" {
     auto = subscribe
     special_use = \Junk
-    fts_autoindex = no
   }
   mailbox "Junk-E-Mail" {
     special_use = \Junk

data/conf/dovecot/global_sieve_before

@@ -1,13 +1,3 @@
 # global_sieve_before script
 # global_sieve_before -> user sieve_before (mailcow UI) -> user sieve_after (mailcow UI) -> global_sieve_after
 
-require ["mailbox", "fileinto"];
-
-if header :contains ["Chat-Version"] [""] {
-  if mailboxexists "DeltaChat" {
-    fileinto "DeltaChat";
-  } else {
-    fileinto :create "DeltaChat";
-  }
-  stop;
-}

data/conf/nginx/templates/nginx.conf.j2

@@ -42,12 +42,19 @@ http {
         https https;
     }
 
+    map $server_port $sogo_auth_internal {
+        65510   "1";
+        default "";
+    }
+
     {% if HTTP_REDIRECT %}
     # HTTP to HTTPS redirect
     server {
         root /web;
         listen {{ HTTP_PORT }} default_server;
+        {% if ENABLE_IPV6 %}
         listen [::]:{{ HTTP_PORT }} default_server;
+        {%endif%}
 
         server_name {{ MAILCOW_HOSTNAME }} autodiscover.* autoconfig.* mta-sts.* {{ ADDITIONAL_SERVER_NAMES | join(' ') }};
 

data/conf/nginx/templates/sites-default.conf.j2

@@ -116,6 +116,9 @@ location ~ \.php$ {
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
+    fastcgi_param HTTP_X_REAL_IP $remote_addr;
+    # trusted internal-auth marker; empty for external clients (see nginx.conf map)
+    fastcgi_param SOGO_AUTH_INTERNAL $sogo_auth_internal;
     fastcgi_read_timeout 3600;
     fastcgi_send_timeout 3600;
 }
@@ -261,19 +264,19 @@ location ~* /sogo$ {
 }
 
 location /SOGo.woa/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location /.woa/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location /SOGo/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location (^/SOGo/so/ControlPanel/Products/[^/]*UI/Resources/.*\.(jpg|png|gif|css|js)$) {
-    alias /usr/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
+    alias /usr/local/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
 }
 {% endif %}
 

data/conf/phpfpm/crons/group-grants.php

@@ -0,0 +1,65 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Group Shares") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/group-grants.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+$sharing = \Mailcow\MailboxGroup\MailboxGroupModule::create($pdo, \Mailcow\Core\Context::system())->sharing;
+
+// Every minute: subscriptions of changed groups; failed ones are retried and logged by the full run
+if ((int)date('i') % 15 !== 0) {
+  $sharing->syncSubscriptions();
+  exit(0);
+}
+
+// Every 15 minutes: IMAP ACLs on new top-level folders and failed grants, SOGo grants of owners that opened SOGo meanwhile
+foreach ($sharing->reconcile() as $warning) {
+  logMsg("warning", $warning[0] . ": " . html_entity_decode(implode(", ", array_slice($warning, 1))));
+}

data/conf/phpfpm/crons/identity-sync.php

@@ -0,0 +1,95 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Identity Sync") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+
+// The legacy mailbox functions run as admin
+$_SESSION['mailcow_cc_username'] = "admin";
+$_SESSION['mailcow_cc_role'] = "admin";
+$_SESSION['acl']['tls_policy'] = "1";
+$_SESSION['acl']['quarantine_notification'] = "1";
+$_SESSION['acl']['quarantine_category'] = "1";
+$_SESSION['acl']['ratelimit'] = "1";
+$_SESSION['acl']['sogo_access'] = "1";
+$_SESSION['acl']['protocol_access'] = "1";
+$_SESSION['acl']['mailbox_relayhost'] = "1";
+$_SESSION['acl']['unlimited_quota'] = "1";
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/identity-sync.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+// Users, offboarding and groups of Keycloak or LDAP; requested runs first, else every sync_interval minutes.
+// The IdP is only contacted when a run is due.
+$iam_settings = identity_provider('get');
+$identity = \Mailcow\Identity\IdentityModule::create($pdo, \Mailcow\Core\Context::system(), $iam_settings, function () {
+  return identity_provider('init');
+});
+$run = $identity->sync->runDue();
+if ($run !== null) {
+  $summary = $run['summary'];
+  if ($run['status'] == 'error') {
+    logMsg("err", ucfirst($run['mode']) . " failed: " . $summary['error']);
+  } else {
+    $counts = array();
+    foreach ($summary['counts'] as $key => $count) {
+      if ($count > 0) $counts[] = $key . "=" . $count;
+    }
+    logMsg(($run['status'] == 'blocked') ? "warning" : "info", ucfirst($run['mode']) . " " . $run['status'] . ": " . (empty($counts) ? "no changes" : implode(", ", $counts)));
+    foreach ($summary['details']['conflicts'] as $conflict) {
+      logMsg("warning", $conflict);
+    }
+    foreach ($summary['errors'] ?? array() as $error) {
+      logMsg("err", $error);
+    }
+  }
+}
+session_destroy();

data/conf/phpfpm/crons/imapsync-oauth-refresh.php

@@ -0,0 +1,73 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "imapsync OAuth refresh") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+
+// File-lock to prevent concurrent runs
+$lock_file = '/tmp/imapsync-oauth-refresh.lock';
+if (file_exists($lock_file)) {
+  $pid = (int)trim(@file_get_contents($lock_file));
+  if ($pid > 0 && posix_kill($pid, 0)) {
+    logMsg("info", "Previous refresh still running (pid=$pid), exiting");
+    exit(0);
+  }
+  @unlink($lock_file);
+}
+file_put_contents($lock_file, getmypid());
+
+try {
+  // client_credentials source tokens, unused per-user tokens (1 day grace), authorization_code per-user tokens;
+  // failing tokens are retried 15 minutes after their last attempt (C2), errors are sanitized
+  $imapsync = \Mailcow\Imapsync\ImapsyncModule::create($pdo, \Mailcow\Core\Context::system(), [
+    'hostname' => (string)getenv('MAILCOW_HOSTNAME'),
+  ]);
+  foreach ($imapsync->oauth->refreshDue() as $e) {
+    logMsg($e['priority'], $e['message']);
+  }
+} finally {
+  @unlink($lock_file);
+}

data/conf/phpfpm/crons/keycloak-sync.php

@@ -1,231 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "Keycloak Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "keycloak" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Keycloak Provider
-$iam_provider = identity_provider('init');
-
-// Loop until all users have been retrieved
-while (true) {
-  // Get admin access token
-  $admin_token = identity_provider("get-keycloak-admin-token");
-
-  // Make the API request to retrieve the users
-  $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users?first=$start&max=$max";
-  $ch = curl_init();
-  curl_setopt($ch, CURLOPT_URL, $url);
-  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-  curl_setopt($ch, CURLOPT_HTTPHEADER, [
-    "Content-Type: application/json",
-    "Authorization: Bearer " . $admin_token
-  ]);
-  $response = curl_exec($ch);
-  $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-  curl_close($ch);
-
-  if ($code != 200){
-    logMsg("err", "Received HTTP {$code}");
-    session_destroy();
-    exit;
-  }
-  try {
-    $response = json_decode($response, true);
-  } catch (Exception $e) {
-    logMsg("err", $e->getMessage());
-    break;
-  }
-  if (!is_array($response)){
-    logMsg("err", "Received malformed response from keycloak api");
-    break;
-  }
-  if (count($response) == 0) {
-    break;
-  }
-
-  // Process the batch of users
-  foreach ($response as $user) {
-    if (empty($user['email'])){
-      logMsg("warning", "No email address in keycloak found for user " . $user['name']);
-      continue;
-    }
-
-    // try get mailbox user
-    $stmt = $pdo->prepare("SELECT
-      mailbox.*,
-      domain.active AS d_active
-      FROM `mailbox`
-      INNER JOIN domain on mailbox.domain = domain.domain
-      WHERE `kind` NOT REGEXP 'location|thing|group'
-        AND `username` = :user");
-    $stmt->execute(array(':user' => $user['email']));
-    $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-    // check if matching attribute mapping exists
-    $user_template = $user['attributes']['mailcow_template'][0];
-    $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-    $_SESSION['access_all_exception'] = '1';
-    if (!$row && intval($iam_settings['import_users']) == 1){
-      if ($mapper_key === false){
-        if (!empty($iam_settings['default_template'])) {
-          $mbox_template = $iam_settings['default_template'];
-          logMsg("warning", "Using default template for user " . $user['email']);
-        } else {
-          logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-          continue;
-        }
-      } else {
-        $mbox_template = $iam_settings['templates'][$mapper_key];
-      }
-      // mailbox user does not exist, create...
-      logMsg("info", "Creating user " . $user['email']);
-      $create_res = mailbox('add', 'mailbox_from_template', array(
-        'domain' => explode('@', $user['email'])[1],
-        'local_part' => explode('@', $user['email'])[0],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'authsource' => 'keycloak',
-        'template' => $mbox_template
-      ));
-      if (!$create_res){
-        logMsg("err", "Could not create user " . $user['email']);
-        continue;
-      }
-    } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "keycloak") {
-      if ($mapper_key === false){
-        logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-        continue;
-      }
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-      // mailbox user does exist, sync attribtues...
-      logMsg("info", "Syncing attributes for user " . $user['email']);
-      mailbox('edit', 'mailbox_from_template', array(
-        'username' => $user['email'],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'template' => $mbox_template
-      ));
-    } else {
-      // skip mailbox user
-      logMsg("info", "Skipping user " . $user['email']);
-    }
-    $_SESSION['access_all_exception'] = '0';
-
-    sleep(0.025);
-  }
-
-  // Update the pagination variables for the next batch
-  $start += $max;
-  sleep(1);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/conf/phpfpm/crons/ldap-sync.php

@@ -1,198 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "LDAP Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "ldap" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Provider
-$iam_provider = identity_provider('init');
-
-// Get ldap users
-$ldap_query = $iam_provider->query();
-if (!empty($iam_settings['filter'])) {
-  $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
-}
-$response = $ldap_query->where($iam_settings['username_field'], "*")
-  ->where($iam_settings['attribute_field'], "*")
-  ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname'])
-  ->paginate($max);
-
-// Process the users
-foreach ($response as $user) {
-  // try get mailbox user
-  $stmt = $pdo->prepare("SELECT
-    mailbox.*,
-    domain.active AS d_active
-    FROM `mailbox`
-    INNER JOIN domain on mailbox.domain = domain.domain
-    WHERE `kind` NOT REGEXP 'location|thing|group'
-      AND `username` = :user");
-  $stmt->execute(array(':user' => $user[$iam_settings['username_field']][0]));
-  $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-  // check if matching attribute mapping exists
-  $user_template = $user[$iam_settings['attribute_field']][0];
-  $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-  if (empty($user[$iam_settings['username_field']][0])){
-    logMsg("warning", "Skipping user " . $user['displayname'][0] . " due to empty LDAP ". $iam_settings['username_field'] . " property.");
-    continue;
-  }
-
-  $_SESSION['access_all_exception'] = '1';
-  if (!$row && intval($iam_settings['import_users']) == 1){
-    if ($mapper_key === false){
-      if (!empty($iam_settings['default_template'])) {
-        $mbox_template = $iam_settings['default_template'];
-      } else {
-        logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-        continue;
-      }
-    } else {
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-    }
-    // mailbox user does not exist, create...
-    logMsg("info", "Creating user " .  $user[$iam_settings['username_field']][0]);
-    $create_res = mailbox('add', 'mailbox_from_template', array(
-      'domain' => explode('@',  $user[$iam_settings['username_field']][0])[1],
-      'local_part' => explode('@',  $user[$iam_settings['username_field']][0])[0],
-      'name' => $user['displayname'][0],
-      'authsource' => 'ldap',
-      'template' => $mbox_template
-    ));
-    if (!$create_res){
-      logMsg("err", "Could not create user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-  } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "ldap") {
-    if ($mapper_key === false){
-      logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-    $mbox_template = $iam_settings['templates'][$mapper_key];
-    // mailbox user does exist, sync attribtues...
-    logMsg("info", "Syncing attributes for user " . $user[$iam_settings['username_field']][0]);
-    mailbox('edit', 'mailbox_from_template', array(
-      'username' =>  $user[$iam_settings['username_field']][0],
-      'name' => $user['displayname'][0],
-      'template' => $mbox_template
-    ));
-  } else {
-    // skip mailbox user
-    logMsg("info", "Skipping user " .  $user[$iam_settings['username_field']][0]);
-  }
-  $_SESSION['access_all_exception'] = '0';
-
-  sleep(0.025);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/conf/phpfpm/php-conf.d/other.ini

@@ -1,3 +1,5 @@
 max_execution_time = 3600
 max_input_time = 3600
 memory_limit = 512M
+; not used by mailcow, disabled as hardening (PHP security release 2026-10-21)
+session.upload_progress.enabled = Off

data/conf/postfix/main.cf

@@ -37,7 +37,7 @@ postscreen_access_list = permit_mynetworks,
   cidr:/opt/postfix/conf/postscreen_access.cidr,
   tcp:127.0.0.1:10027
 postscreen_bare_newline_enable = no
-postscreen_blacklist_action = drop
+postscreen_denylist_action = drop
 postscreen_cache_cleanup_interval = 24h
 postscreen_cache_map = proxy:btree:$data_directory/postscreen_cache
 postscreen_dnsbl_action = enforce
@@ -107,8 +107,6 @@ smtpd_sender_restrictions = reject_authenticated_sender_login_mismatch,
   reject_unknown_sender_domain
 smtpd_soft_error_limit = 3
 smtpd_tls_auth_only = yes
-smtpd_tls_dh1024_param_file = /etc/ssl/mail/dhparams.pem
-smtpd_tls_eecdh_grade = auto
 smtpd_tls_exclude_ciphers = ECDHE-RSA-RC4-SHA, RC4, aNULL, DES-CBC3-SHA, ECDHE-RSA-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA
 smtpd_tls_loglevel = 1
 
@@ -152,7 +150,7 @@ smtp_sasl_auth_enable = yes
 smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf
 smtp_sasl_security_options =
 smtp_sasl_mechanism_filter = plain, login
-smtp_tls_policy_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf socketmap:inet:postfix-tlspol:8642:QUERY
+smtp_tls_policy_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf socketmap:inet:postfix-tlspol:8642:QUERYwithTLSRPT
 smtp_header_checks = pcre:/opt/postfix/conf/anonymize_headers.pcre
 mail_name = Postcow
 # local_transport map catches local destinations and prevents routing local dests when the next map would route "*"

data/conf/postfix/master.cf

@@ -29,7 +29,6 @@ smtps    inet  n       -       n       -       -       smtpd
 # TLS protocol can be modified by setting submission_smtpd_tls_mandatory_protocols in extra.cf
 submission inet n       -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
-  -o smtpd_enforce_tls=yes
   -o smtpd_tls_security_level=encrypt
   -o smtpd_tls_mandatory_protocols=$submission_smtpd_tls_mandatory_protocols
   -o tls_preempt_cipherlist=yes
@@ -38,7 +37,6 @@ submission inet n       -       n       -       -       smtpd
 10587      inet n       -       n       -       -       smtpd
   -o smtpd_upstream_proxy_protocol=haproxy
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
-  -o smtpd_enforce_tls=yes
   -o smtpd_tls_security_level=encrypt
   -o smtpd_tls_mandatory_protocols=$submission_smtpd_tls_mandatory_protocols
   -o tls_preempt_cipherlist=yes

data/conf/postfix/postscreen_access.cidr

@@ -1,7 +1,8 @@
-# Whitelist generated by Postwhite v3.4 on Sun Mar  1 00:29:01 UTC 2026
+# Whitelist generated by Postwhite v3.4 on Thu Oct  1 00:35:20 UTC 2026
 # https://github.com/stevejenkins/postwhite/
-# 2174 total rules
+# 2246 total rules
 2a00:1450:4000::/36	permit
+2a00:1450:4864::/56	permit
 2a01:111:f400::/48	permit
 2a01:111:f403:2800::/53	permit
 2a01:111:f403:8000::/51	permit
@@ -28,11 +29,15 @@
 2a01:b747:3004:200::/56	permit
 2a01:b747:3005:200::/56	permit
 2a01:b747:3006:200::/56	permit
+2a01:b747:3007::/56	permit
 2a02:a60:0:5::/64	permit
 2a0f:f640::/56	permit
 2c0f:fb50:4000::/36	permit
+2c0f:fb50:4864::/56	permit
+2.207.151.32/27	permit
 2.207.151.53	permit
 2.207.217.30	permit
+2.207.223.160/27	permit
 3.64.237.68	permit
 3.65.3.180	permit
 3.70.123.177	permit
@@ -53,13 +58,14 @@
 8.25.196.0/23	permit
 8.36.116.0/24	permit
 8.39.54.0/23	permit
+8.39.54.240/29	permit
 8.39.54.250/31	permit
 8.39.144.0/24	permit
 8.40.222.0/23	permit
+8.40.222.240/29	permit
 8.40.222.250/31	permit
+8.47.10.9	permit
 12.130.86.238	permit
-13.107.213.51	permit
-13.107.246.51	permit
 13.108.16.0/20	permit
 13.110.208.0/21	permit
 13.110.209.0/24	permit
@@ -119,7 +125,6 @@
 23.249.208.0/20	permit
 23.251.224.0/19	permit
 23.253.141.0/24	permit
-23.253.182.0/23	permit
 23.253.182.103	permit
 23.253.183.145	permit
 23.253.183.146	permit
@@ -141,24 +146,7 @@
 27.123.206.80/28	permit
 31.47.251.17	permit
 31.186.239.0/24	permit
-34.2.64.0/22	permit
-34.2.68.0/23	permit
-34.2.70.0/23	permit
-34.2.71.64/26	permit
-34.2.72.0/22	permit
-34.2.75.0/26	permit
-34.2.78.0/23	permit
-34.2.80.0/23	permit
-34.2.82.0/23	permit
-34.2.84.0/24	permit
-34.2.84.64/26	permit
-34.2.85.0/24	permit
-34.2.85.64/26	permit
-34.2.86.0/23	permit
-34.2.88.0/23	permit
-34.2.90.0/23	permit
-34.2.92.0/23	permit
-34.2.94.0/23	permit
+34.2.64.0/19	permit
 34.70.158.162	permit
 34.74.74.140	permit
 34.83.159.189	permit
@@ -185,6 +173,7 @@
 35.205.92.9	permit
 35.228.216.85	permit
 35.242.169.159	permit
+37.187.220.204	permit
 37.188.97.188	permit
 37.218.248.47	permit
 37.218.249.47	permit
@@ -258,8 +247,8 @@
 46.228.39.112/31	permit
 46.243.88.174	permit
 46.243.88.175	permit
-46.243.88.176	permit
-46.243.88.177	permit
+46.243.88.176/30	permit
+46.243.95.96/27	permit
 46.243.95.179	permit
 46.243.95.180	permit
 50.16.246.183	permit
@@ -277,6 +266,8 @@
 50.56.130.221	permit
 50.56.130.222	permit
 50.112.246.219	permit
+50.152.146.25	permit
+51.83.17.38	permit
 52.1.14.157	permit
 52.5.230.59	permit
 52.6.74.205	permit
@@ -367,7 +358,9 @@
 64.127.115.252	permit
 64.132.88.0/23	permit
 64.132.92.0/24	permit
+64.181.136.241	permit
 64.181.194.190	permit
+64.181.213.254	permit
 64.207.219.7	permit
 64.207.219.8	permit
 64.207.219.9	permit
@@ -453,7 +446,11 @@
 66.218.75.252/31	permit
 66.218.75.254	permit
 66.220.144.128/25	permit
+66.220.144.178	permit
+66.220.144.179	permit
 66.220.155.0/24	permit
+66.220.155.178	permit
+66.220.155.179	permit
 66.220.157.0/25	permit
 66.231.80.0/20	permit
 66.240.227.0/24	permit
@@ -518,6 +515,8 @@
 69.162.98.0/24	permit
 69.169.224.0/20	permit
 69.171.232.0/24	permit
+69.171.232.180	permit
+69.171.232.181	permit
 69.171.244.0/23	permit
 70.42.149.35	permit
 72.3.185.0/24	permit
@@ -639,6 +638,8 @@
 77.238.189.148/30	permit
 79.135.106.0/24	permit
 79.135.107.0/24	permit
+80.225.160.128/25	permit
+81.88.38.0/23	permit
 81.169.146.243	permit
 81.169.146.245	permit
 81.169.146.246	permit
@@ -657,6 +658,9 @@
 82.165.159.45	permit
 82.165.159.130	permit
 82.165.159.131	permit
+84.8.68.0/25	permit
+84.8.192.128/25	permit
+84.8.224.128/25	permit
 85.9.206.169	permit
 85.9.210.45	permit
 85.158.136.0/21	permit
@@ -720,6 +724,7 @@
 96.43.148.64/28	permit
 96.43.148.64/31	permit
 96.43.151.64/28	permit
+98.77.0.0/16	permit
 98.97.248.0/21	permit
 98.136.44.181	permit
 98.136.44.182/31	permit
@@ -1215,8 +1220,6 @@
 104.43.243.237	permit
 104.44.112.128/25	permit
 104.47.0.0/17	permit
-104.130.96.0/28	permit
-104.130.122.0/23	permit
 106.10.144.64/27	permit
 106.10.144.100/31	permit
 106.10.144.103	permit
@@ -1369,6 +1372,7 @@
 117.120.16.0/21	permit
 119.42.242.52/31	permit
 119.42.242.156	permit
+121.244.91.31	permit
 121.244.91.48	permit
 121.244.91.52	permit
 122.15.156.182	permit
@@ -1408,6 +1412,20 @@
 129.146.88.28	permit
 129.146.147.105	permit
 129.146.236.58	permit
+129.148.135.0/25	permit
+129.148.148.0/25	permit
+129.148.164.0/25	permit
+129.148.180.0/25	permit
+129.148.215.0/25	permit
+129.149.6.0/25	permit
+129.149.22.0/25	permit
+129.149.38.0/25	permit
+129.149.52.0/25	permit
+129.149.68.0/25	permit
+129.149.84.0/25	permit
+129.149.100.0/25	permit
+129.149.118.0/25	permit
+129.149.126.0/25	permit
 129.151.67.221	permit
 129.153.62.216	permit
 129.153.104.71	permit
@@ -1420,18 +1438,21 @@
 129.159.22.159	permit
 129.159.87.137	permit
 129.213.195.191	permit
+130.35.116.0/25	permit
 130.61.9.72	permit
 130.162.39.83	permit
+130.162.194.23	permit
 130.248.172.0/24	permit
 130.248.173.0/24	permit
+131.186.12.0/25	permit
 131.253.30.0/24	permit
 131.253.121.0/26	permit
 132.145.13.209	permit
 132.226.26.225	permit
 132.226.49.32	permit
 132.226.56.24	permit
-134.128.64.0/19	permit
-134.128.96.0/19	permit
+134.98.248.128/25	permit
+134.128.64.0/18	permit
 134.170.27.8	permit
 134.170.113.0/26	permit
 134.170.141.64/26	permit
@@ -1460,6 +1481,10 @@
 136.147.182.0/24	permit
 136.147.224.0/20	permit
 136.179.50.206	permit
+136.248.224.128/25	permit
+136.248.232.128/25	permit
+138.1.108.0/25	permit
+138.1.170.0/24	permit
 139.60.152.0/22	permit
 139.138.35.44	permit
 139.138.46.121	permit
@@ -1468,6 +1493,7 @@
 139.138.57.55	permit
 139.138.58.119	permit
 139.167.79.86	permit
+139.177.108.0/25	permit
 139.180.17.0/24	permit
 140.238.148.191	permit
 141.148.55.217	permit
@@ -1498,14 +1524,14 @@
 146.20.14.105	permit
 146.20.14.106	permit
 146.20.14.107	permit
-146.20.112.0/26	permit
-146.20.113.0/24	permit
-146.20.191.0/24	permit
 146.20.215.0/24	permit
 146.20.215.182	permit
 146.88.28.0/24	permit
 146.148.116.76	permit
 147.154.32.0/25	permit
+147.154.63.0/24	permit
+147.154.126.0/24	permit
+147.154.191.0/24	permit
 147.243.1.47	permit
 147.243.1.48	permit
 147.243.1.153	permit
@@ -1513,16 +1539,23 @@
 147.243.128.26	permit
 148.105.0.0/16	permit
 148.105.8.0/21	permit
+148.116.32.128/25	permit
 149.72.0.0/16	permit
 149.72.234.184	permit
 149.72.248.236	permit
 149.97.173.180	permit
+149.118.160.128/25	permit
 150.136.21.199	permit
+150.171.109.178	permit
 150.230.98.160	permit
 151.145.38.14	permit
 152.67.105.195	permit
+152.69.165.251	permit
 152.69.200.236	permit
 152.70.155.126	permit
+155.248.135.128/25	permit
+155.248.140.0/25	permit
+155.248.148.0/25	permit
 155.248.208.51	permit
 155.248.220.138	permit
 155.248.234.149	permit
@@ -1531,10 +1564,16 @@
 157.58.30.128/25	permit
 157.58.196.96/29	permit
 157.58.249.3	permit
+157.137.32.128/25	permit
+157.137.96.128/25	permit
 157.151.208.65	permit
 157.255.1.64/29	permit
 158.101.211.207	permit
 158.247.16.0/20	permit
+158.247.100.0/25	permit
+159.13.4.0/25	permit
+159.13.33.181	permit
+159.26.176.0/20	permit
 159.92.154.0/24	permit
 159.92.155.0/24	permit
 159.92.157.0/24	permit
@@ -1562,12 +1601,15 @@
 159.183.121.182	permit
 159.183.129.172	permit
 160.1.62.192	permit
+161.33.94.110	permit
+161.33.228.244	permit
 161.38.192.0/20	permit
 161.38.204.0/22	permit
 161.71.32.0/19	permit
 161.71.64.0/20	permit
 162.88.4.0/23	permit
 162.88.8.0/24	permit
+162.88.24.0/23	permit
 162.88.24.0/24	permit
 162.88.25.0/24	permit
 162.88.36.0/24	permit
@@ -1585,11 +1627,13 @@
 164.152.23.32	permit
 164.152.25.241	permit
 164.177.132.168/30	permit
+165.1.100.0/25	permit
 165.173.128.0/24	permit
-165.173.180.1	permit
+165.173.180.0/24	permit
 165.173.180.250/31	permit
+165.173.182.0/24	permit
 165.173.182.250/31	permit
-166.78.68.0/22	permit
+165.173.189.205	permit
 166.78.68.221	permit
 166.78.69.169	permit
 166.78.69.170	permit
@@ -1607,9 +1651,12 @@
 167.89.75.126	permit
 167.89.75.136	permit
 167.89.75.164	permit
-167.89.101.2	permit
-167.89.101.192/28	permit
 167.220.67.232/29	permit
+168.107.248.128/25	permit
+168.110.160.128/25	permit
+168.110.248.128/25	permit
+168.129.184.128/25	permit
+168.129.248.128/25	permit
 168.138.5.36	permit
 168.138.73.51	permit
 168.138.77.31	permit
@@ -1626,6 +1673,7 @@
 169.148.144.0/25	permit
 169.148.144.10	permit
 169.148.146.0/23	permit
+169.148.174.10	permit
 169.148.175.3	permit
 169.148.179.3	permit
 169.148.188.0/24	permit
@@ -1644,6 +1692,7 @@
 173.203.81.39	permit
 173.224.161.128/25	permit
 173.224.165.0/26	permit
+173.224.166.48/28	permit
 174.36.84.8/29	permit
 174.36.84.16/29	permit
 174.36.84.32/29	permit
@@ -1658,6 +1707,8 @@
 175.41.215.51	permit
 176.32.105.0/24	permit
 176.32.127.0/24	permit
+176.94.34.32/27	permit
+176.94.34.64/27	permit
 178.236.10.128/26	permit
 182.50.76.0/22	permit
 182.50.78.64/28	permit
@@ -1739,26 +1790,36 @@
 188.125.85.234/31	permit
 188.125.85.236/31	permit
 188.125.85.238	permit
-188.165.51.139	permit
 188.172.128.0/20	permit
 192.0.64.0/18	permit
+192.9.166.184	permit
 192.18.139.154	permit
 192.18.145.36	permit
 192.18.152.58	permit
+192.22.32.128/25	permit
+192.22.96.128/25	permit
+192.22.160.128/25	permit
+192.22.224.128/25	permit
 192.28.128.0/18	permit
+192.29.24.0/25	permit
+192.29.44.0/25	permit
+192.29.72.0/25	permit
+192.29.88.0/25	permit
 192.29.103.128/25	permit
+192.29.134.0/25	permit
+192.29.151.128/25	permit
+192.29.172.0/25	permit
+192.29.178.0/25	permit
+192.29.200.0/25	permit
+192.29.216.0/25	permit
+192.29.232.0/25	permit
+192.29.248.0/25	permit
 192.30.252.0/22	permit
 192.161.144.0/20	permit
 192.162.87.0/24	permit
-192.237.158.0/23	permit
 192.237.159.42	permit
 192.237.159.43	permit
 192.254.112.0/20	permit
-192.254.112.60	permit
-192.254.112.98/31	permit
-192.254.113.10	permit
-192.254.113.101	permit
-192.254.114.176	permit
 193.109.254.0/23	permit
 193.122.128.100	permit
 193.123.56.63	permit
@@ -1798,7 +1859,6 @@
 198.21.0.0/21	permit
 198.37.144.0/20	permit
 198.37.152.186	permit
-198.61.254.0/23	permit
 198.61.254.21	permit
 198.61.254.231	permit
 198.178.234.57	permit
@@ -1885,6 +1945,7 @@
 204.14.232.0/21	permit
 204.14.232.64/28	permit
 204.14.234.64/28	permit
+204.75.18.128/27	permit
 204.75.142.0/24	permit
 204.92.114.187	permit
 204.92.114.203	permit
@@ -1892,7 +1953,9 @@
 204.141.32.0/23	permit
 204.141.42.0/23	permit
 204.216.164.202	permit
-204.220.160.0/21	permit
+204.220.90.0/23	permit
+204.220.92.0/22	permit
+204.220.160.0/20	permit
 204.220.168.0/21	permit
 204.220.176.0/20	permit
 204.220.181.105	permit
@@ -1927,6 +1990,9 @@
 207.211.30.128/25	permit
 207.211.31.0/25	permit
 207.211.41.113	permit
+207.211.132.0/25	permit
+207.211.147.87	permit
+207.211.159.110	permit
 207.218.90.0/24	permit
 207.218.90.122	permit
 207.250.68.0/24	permit
@@ -1934,6 +2000,8 @@
 208.43.21.28/30	permit
 208.43.21.64/29	permit
 208.43.21.72/30	permit
+208.56.9.224	permit
+208.56.13.196	permit
 208.64.132.0/22	permit
 208.71.40.63	permit
 208.71.40.64/31	permit
@@ -1960,6 +2028,7 @@
 208.71.42.214	permit
 208.72.249.240/29	permit
 208.75.120.0/22	permit
+208.76.62.0/23	permit
 208.76.62.0/24	permit
 208.76.63.0/24	permit
 208.82.237.96/29	permit
@@ -1973,7 +2042,6 @@
 209.43.22.0/28	permit
 209.46.117.168	permit
 209.46.117.179	permit
-209.61.151.0/24	permit
 209.61.151.236	permit
 209.61.151.249	permit
 209.61.151.251	permit
@@ -2120,6 +2188,7 @@
 216.136.168.80/28	permit
 216.139.64.0/19	permit
 216.145.221.0/24	permit
+216.146.32.0/23	permit
 216.146.32.0/24	permit
 216.146.33.0/24	permit
 216.198.0.0/18	permit
@@ -2140,6 +2209,7 @@
 223.165.120.0/23	permit
 2001:0868:0100:0600::/64	permit
 2001:4860:4000::/36	permit
+2001:4860:4864::/56	permit
 2001:748:100:40::2:0/112	permit
 2001:748:400:1300::3	permit
 2001:748:400:1300::4	permit
@@ -2157,10 +2227,13 @@
 2001:748:400:3301::3	permit
 2001:748:400:3301::4	permit
 2404:6800:4000::/36	permit
+2404:6800:4864::/56	permit
+2603:1061:14:1d0::1	permit
 2607:13c0:0001:0000:0000:0000:0000:7000/116	permit
 2607:13c0:0002:0000:0000:0000:0000:1000/116	permit
 2607:13c0:0004:0000:0000:0000:0000:0000/116	permit
 2607:f8b0:4000::/36	permit
+2607:f8b0:4864::/56	permit
 2620:109:c003:104::/64	permit
 2620:109:c003:104::215	permit
 2620:109:c006:104::/64	permit
@@ -2172,8 +2245,7 @@
 2620:10d:c09c:400::8:1	permit
 2620:119:50c0:207::/64	permit
 2620:119:50c0:207::215	permit
-2620:1ec:46::51	permit
-2620:1ec:bdf::51	permit
 2800:3f0:4000::/36	permit
+2800:3f0:4864::/56	permit
 49.12.4.251 permit # checks.mailcow.email
 2a01:4f8:c17:7906::10 permit # checks.mailcow.email

data/conf/rspamd/local.d/metadata_exporter.conf

@@ -3,8 +3,7 @@ rules {
     backend = "http";
     url = "http://nginx:9081/pipe.php";
     selector = "reject_no_global_bl";
-    formatter = "default";
-    meta_headers = true;
+    formatter = "multipart";
   }
 	RLINFO {
 		backend = "http";
@@ -16,8 +15,7 @@ rules {
     backend = "http";
     url = "http://nginx:9081/pushover.php";
     selector = "mailcow_rcpt";
-    formatter = "json";
-    meta_headers = true;
+    formatter = "multipart";
   }
 }
 

data/conf/rspamd/lua/rspamd.local.lua

@@ -856,6 +856,8 @@ rspamd_config:register_symbol({
 
             local seen_cte
             local newline_s = newline(task)
+            -- use the CTE add_text_footer actually applied
+            local new_cte = rewrite.new_cte or 'quoted-printable'
 
             local function rewrite_ct_cb(name, hdr)
               if rewrite.need_rewrite_ct then
@@ -877,13 +879,13 @@ rspamd_config:register_symbol({
                   return
                 elseif name:lower() == 'content-transfer-encoding' then
                   out[#out + 1] = string.format('%s: %s',
-                      'Content-Transfer-Encoding', 'quoted-printable')
+                      'Content-Transfer-Encoding', new_cte)
                   -- update Content-Transfer-Encoding header
                   task:set_milter_reply({
                     remove_headers = {['Content-Transfer-Encoding'] = 0},
                   })
                   task:set_milter_reply({
-                    add_headers = {['Content-Transfer-Encoding'] = 'quoted-printable'}
+                    add_headers = {['Content-Transfer-Encoding'] = new_cte}
                   })
                   seen_cte = true
                   return
@@ -895,7 +897,7 @@ rspamd_config:register_symbol({
             task:headers_foreach(rewrite_ct_cb, {full = true})
 
             if not seen_cte and rewrite.need_rewrite_ct then
-              out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', 'quoted-printable')
+              out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', new_cte)
             end
 
             -- End of headers
@@ -914,7 +916,7 @@ rspamd_config:register_symbol({
                 out_parts[#out_parts + 1] = o
                 out_parts[#out_parts + 1] = newline_s
               else
-                local removePrefix = "--\x0D\x0AContent-Type"
+                local removePrefix = "--\x0D\x0AContent-"
                 if string.lower(string.sub(tostring(o[1]), 1, string.len(removePrefix))) == string.lower(removePrefix) then
                   o[1] = string.sub(tostring(o[1]), string.len("--\x0D\x0A") + 1)
                 end

data/conf/rspamd/meta_exporter/pipe.php

@@ -32,47 +32,42 @@ function parse_email($email) {
   $a = strrpos($email, '@');
   return array('local' => substr($email, 0, $a), 'domain' => substr(substr($email, $a), 1));
 }
-if (!function_exists('getallheaders'))  {
-  function getallheaders() {
-    if (!is_array($_SERVER)) {
-      return array();
-    }
-    $headers = array();
-    foreach ($_SERVER as $name => $value) {
-      if (substr($name, 0, 5) == 'HTTP_') {
-        $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value;
-      }
-    }
-    return $headers;
-  }
+// rspamd metadata_exporter (multipart formatter):
+//   - $_POST['metadata']        JSON with the rspamd metadata
+//   - $_FILES['message']        raw RFC822 message
+if (empty($_POST['metadata']) || !isset($_FILES['message']) || $_FILES['message']['error'] !== UPLOAD_ERR_OK) {
+  error_log("QUARANTINE: missing multipart parts from rspamd" . PHP_EOL);
+  http_response_code(400);
+  exit;
 }
 
-$raw_data_content = file_get_contents('php://input');
+$meta = json_decode($_POST['metadata'], true);
+if (!is_array($meta)) {
+  error_log("QUARANTINE: cannot decode metadata JSON" . PHP_EOL);
+  http_response_code(400);
+  exit;
+}
+
+$raw_data_content = file_get_contents($_FILES['message']['tmp_name']);
 $raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
-$headers = getallheaders();
+$raw_size = (int)$_FILES['message']['size'];
 
-$qid      = $headers['X-Rspamd-Qid'];
-$fuzzy    = $headers['X-Rspamd-Fuzzy'];
-$subject  = iconv_mime_decode($headers['X-Rspamd-Subject']);
-$score    = $headers['X-Rspamd-Score'];
-$rcpts    = $headers['X-Rspamd-Rcpt'];
-$user     = $headers['X-Rspamd-User'];
-$ip       = $headers['X-Rspamd-Ip'];
-$action   = $headers['X-Rspamd-Action'];
-$sender   = $headers['X-Rspamd-From'];
-$symbols  = $headers['X-Rspamd-Symbols'];
-
-$raw_size = (int)$_SERVER['CONTENT_LENGTH'];
+$qid      = $meta['qid']     ?? 'unknown';
+$subject  = $meta['subject'] ?? '';
+$score    = $meta['score']   ?? 0;
+$rcpts    = $meta['rcpt']    ?? array();
+$user     = $meta['user']    ?? 'unknown';
+$ip       = $meta['ip']      ?? 'unknown';
+$action   = $meta['action']  ?? 'no action';
+$sender   = $meta['from']    ?? '';
+$symbols  = json_encode($meta['symbols'] ?? array());
+$fuzzy    = json_encode(is_array($meta['fuzzy'] ?? null) ? $meta['fuzzy'] : array());
 
 if (empty($sender)) {
   error_log("QUARANTINE: Unknown sender, assuming empty-env-from@localhost" . PHP_EOL);
   $sender = 'empty-env-from@localhost';
 }
 
-if ($fuzzy == 'unknown') {
-  $fuzzy = '[]';
-}
-
 try {
   $max_size = (int)$redis->Get('Q_MAX_SIZE');
   if (($max_size * 1048576) < $raw_size) {
@@ -94,7 +89,7 @@ catch (RedisException $e) {
 $rcpt_final_mailboxes = array();
 
 // Loop through all rcpts
-foreach (json_decode($rcpts, true) as $rcpt) {
+foreach ($rcpts as $rcpt) {
   // Remove tag
   $rcpt = preg_replace('/^(.*?)\+.*(@.*)$/', '$1$2', $rcpt);
 
@@ -158,12 +153,12 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 
       // Loop through all found gotos
       foreach ($gotos_array as $index => &$goto) {
-        error_log("RCPT RESOVLER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
+        error_log("RCPT RESOLVER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
         $stmt = $pdo->prepare("SELECT `username` FROM `mailbox` WHERE `username` = :goto AND (`active`= '1' OR `active`= '2');");
         $stmt->execute(array(':goto' => $goto));
         $username = $stmt->fetch(PDO::FETCH_ASSOC)['username'];
         if (!empty($username)) {
-          error_log("RCPT RESOVLER: http pipe: mailbox found: " . $username . PHP_EOL);
+          error_log("RCPT RESOLVER: http pipe: mailbox found: " . $username . PHP_EOL);
           // Current goto is a mailbox, save to rcpt_final_mailboxes if not a duplicate
           if (!in_array($username, $rcpt_final_mailboxes)) {
             $rcpt_final_mailboxes[] = $username;
@@ -172,21 +167,21 @@ foreach (json_decode($rcpts, true) as $rcpt) {
         else {
           $parsed_goto = parse_email($goto);
           if (!$redis->hGet('DOMAIN_MAP', $parsed_goto['domain'])) {
-            error_log("RCPT RESOVLER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
+            error_log("RCPT RESOLVER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
           }
           else {
             $stmt = $pdo->prepare("SELECT `goto` FROM `alias` WHERE `address` = :goto AND `active` = '1'");
             $stmt->execute(array(':goto' => $goto));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
             if ($goto_branch) {
-              error_log("RCPT RESOVLER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
+              error_log("RCPT RESOLVER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
               $goto_branch_array = explode(',', $goto_branch);
             } else {
               $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'");
               $stmt->execute(array(':domain' => $parsed_goto['domain']));
               $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['target_domain'];
               if ($goto_branch) {
-                error_log("RCPT RESOVLER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
+                error_log("RCPT RESOLVER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
                 $goto_branch_array = array($parsed_goto['local'] . '@' . $goto_branch);
               }
             }
@@ -208,11 +203,11 @@ foreach (json_decode($rcpts, true) as $rcpt) {
       // Force exit if loop cannot be solved
       // Postfix does not allow for alias loops, so this should never happen.
       $loop_c++;
-      error_log("RCPT RESOVLER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
+      error_log("RCPT RESOLVER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
     }
   }
   catch (PDOException $e) {
-    error_log("RCPT RESOVLER: " . $e->getMessage() . PHP_EOL);
+    error_log("RCPT RESOLVER: " . $e->getMessage() . PHP_EOL);
     http_response_code(502);
     exit;
   }

data/conf/rspamd/meta_exporter/pushover.php

@@ -32,50 +32,46 @@ function parse_email($email) {
   $a = strrpos($email, '@');
   return array('local' => substr($email, 0, $a), 'domain' => substr(substr($email, $a), 1));
 }
-if (!function_exists('getallheaders'))  {
-  function getallheaders() {
-    if (!is_array($_SERVER)) {
-      return array();
-    }
-    $headers = array();
-    foreach ($_SERVER as $name => $value) {
-      if (substr($name, 0, 5) == 'HTTP_') {
-        $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value;
-      }
-    }
-    return $headers;
-  }
+// rspamd metadata_exporter (multipart formatter): metadata JSON arrives as $_POST['metadata'].
+if (empty($_POST['metadata'])) {
+  error_log("NOTIFY: missing metadata part from rspamd" . PHP_EOL);
+  http_response_code(400);
+  exit;
 }
 
-$headers = getallheaders();
-$json_body = json_decode(file_get_contents('php://input'));
+$meta = json_decode($_POST['metadata'], true);
+if (!is_array($meta)) {
+  error_log("NOTIFY: cannot decode metadata JSON" . PHP_EOL);
+  http_response_code(400);
+  exit;
+}
 
-$qid      = $headers['X-Rspamd-Qid'];
-$rcpts    = $headers['X-Rspamd-Rcpt'];
-$sender   = $headers['X-Rspamd-From'];
-$ip       = $headers['X-Rspamd-Ip'];
-$subject  = iconv_mime_decode($headers['X-Rspamd-Subject']);
-$messageid= $json_body->message_id;
+$qid      = $meta['qid']    ?? 'unknown';
+$rcpts    = $meta['rcpt']   ?? array();
+$sender   = $meta['from']   ?? '';
+$ip       = $meta['ip']     ?? 'unknown';
+$subject  = $meta['subject'] ?? '';
+$messageid= $meta['message_id'] ?? '';
 $priority = 0;
 
-$symbols_array = json_decode($headers['X-Rspamd-Symbols'], true);
+$symbols_array = $meta['symbols'] ?? array();
 if (is_array($symbols_array)) {
   foreach ($symbols_array as $symbol) {
-    if ($symbol['name'] == 'HAS_X_PRIO_ONE') {
+    if (($symbol['name'] ?? null) == 'HAS_X_PRIO_ONE') {
       $priority = 1;
       break;
     }
   }
 }
 
-$sender_address = $json_body->header_from[0];
+$sender_address = $meta['header_from'][0] ?? '';
 $sender_name = '-';
 if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $sender_address, $matches)) {
 	$sender_address = $matches['address'];
   $sender_name =  trim($matches['name'], '"\' ');
 }
 
-$to_address = $json_body->header_to[0];
+$to_address = $meta['header_to'][0] ?? '';
 $to_name = '-';
 if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $to_address, $matches)) {
 	$to_address = $matches['address'];
@@ -85,7 +81,7 @@ if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $to_address, $matches)) {
 $rcpt_final_mailboxes = array();
 
 // Loop through all rcpts
-foreach (json_decode($rcpts, true) as $rcpt) {
+foreach ($rcpts as $rcpt) {
   // Remove tag
   $rcpt = preg_replace('/^(.*?)\+.*(@.*)$/', '$1$2', $rcpt);
 
@@ -143,12 +139,12 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 
       // Loop through all found gotos
       foreach ($gotos_array as $index => &$goto) {
-        error_log("RCPT RESOVLER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
+        error_log("RCPT RESOLVER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
         $stmt = $pdo->prepare("SELECT `username` FROM `mailbox` WHERE `username` = :goto AND (`active`= '1' OR `active`= '2');");
         $stmt->execute(array(':goto' => $goto));
         $username = $stmt->fetch(PDO::FETCH_ASSOC)['username'];
         if (!empty($username)) {
-          error_log("RCPT RESOVLER: http pipe: mailbox found: " . $username . PHP_EOL);
+          error_log("RCPT RESOLVER: http pipe: mailbox found: " . $username . PHP_EOL);
           // Current goto is a mailbox, save to rcpt_final_mailboxes if not a duplicate
           if (!in_array($username, $rcpt_final_mailboxes)) {
             $rcpt_final_mailboxes[] = $username;
@@ -157,21 +153,21 @@ foreach (json_decode($rcpts, true) as $rcpt) {
         else {
           $parsed_goto = parse_email($goto);
           if (!$redis->hGet('DOMAIN_MAP', $parsed_goto['domain'])) {
-            error_log("RCPT RESOVLER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
+            error_log("RCPT RESOLVER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
           }
           else {
             $stmt = $pdo->prepare("SELECT `goto` FROM `alias` WHERE `address` = :goto AND `active` = '1'");
             $stmt->execute(array(':goto' => $goto));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
             if ($goto_branch) {
-              error_log("RCPT RESOVLER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
+              error_log("RCPT RESOLVER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
               $goto_branch_array = explode(',', $goto_branch);
             } else {
               $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'");
               $stmt->execute(array(':domain' => $parsed_goto['domain']));
               $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['target_domain'];
               if ($goto_branch) {
-                error_log("RCPT RESOVLER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
+                error_log("RCPT RESOLVER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
                 $goto_branch_array = array($parsed_goto['local'] . '@' . $goto_branch);
               }
             }
@@ -193,11 +189,11 @@ foreach (json_decode($rcpts, true) as $rcpt) {
       // Force exit if loop cannot be solved
       // Postfix does not allow for alias loops, so this should never happen.
       $loop_c++;
-      error_log("RCPT RESOVLER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
+      error_log("RCPT RESOLVER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
     }
   }
   catch (PDOException $e) {
-    error_log("RCPT RESOVLER: " . $e->getMessage() . PHP_EOL);
+    error_log("RCPT RESOLVER: " . $e->getMessage() . PHP_EOL);
     http_response_code(502);
     exit;
   }

data/web/admin/mailbox.php

@@ -9,6 +9,8 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 
+$js_minifier->add('/web/js/site/imapsync.js');
+$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -38,10 +40,12 @@ $template_data = [
   'global_filters' => mailbox('get', 'global_filter_details'),
   'domains' => $domains,
   'mailboxes' => $mailboxes,
+  'mbox_templates' => mailbox('get', 'mailbox_templates'),
   'lang_mailbox' => json_encode($lang['mailbox']),
   'lang_rl' => json_encode($lang['ratelimit']),
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
+  'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/system.php

@@ -12,6 +12,7 @@ $fido2_data = fido2(array("action" => "get_friendly_names"));
 $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
+$js_minifier->add('/web/js/site/identity.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -93,10 +94,6 @@ $template_data = [
   'tfa_id' => @$_SESSION['tfa_id'],
   'fido2_cid' => @$_SESSION['fido2_cid'],
   'fido2_data' => $fido2_data,
-  'api' => [
-    'ro' => admin_api('ro', 'get'),
-    'rw' => admin_api('rw', 'get'),
-  ],
   'dkim_domains' => $dkim_domains,
   'dkim_domains_with_keys' => $dkim_domains_with_keys,
   'dkim_blind_domains' => $dkim_blind_domains,
@@ -115,7 +112,10 @@ $template_data = [
   'logo_dark_specs' => customize('get', 'main_logo_dark_specs'),
   'ip_check' => customize('get', 'ip_check'),
   'custom_login' => customize('get', 'custom_login'),
+  'ui_themes' => customize('get', 'ui_themes'),
+  'ui_theme' => customize('get', 'ui_theme'),
   'password_complexity' => password_complexity('get'),
+  'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',

data/web/css/build/003-bootstrap-select.css

@@ -1,10 +1,10 @@
-/*!
- * Bootstrap-select v1.14.0-beta2 (https://developer.snapappointments.com/bootstrap-select)
- *
- * Copyright 2012-2021 SnapAppointments, LLC
- * Licensed under MIT (https://github.com/snapappointments/bootstrap-select/blob/master/LICENSE)
- */
-
+/*!
+ * Bootstrap-select v1.14.0-beta3 (https://developer.snapappointments.com/bootstrap-select)
+ *
+ * Copyright 2012-2022 SnapAppointments, LLC
+ * Licensed under MIT (https://github.com/snapappointments/bootstrap-select/blob/master/LICENSE)
+ */
+
 @-webkit-keyframes bs-notify-fadeOut {
   0% {
     opacity: 0.9;
@@ -35,8 +35,7 @@ select.selectpicker {
   display: none !important;
 }
 .bootstrap-select {
-  width: 220px \0;
-  /*IE9 and below*/
+  width: 220px;
   vertical-align: middle;
 }
 .bootstrap-select > .dropdown-toggle {

data/web/css/build/013-datatables.css

@@ -17,15 +17,15 @@ table.dataTable>tbody>tr.child ul.dtr-details>li {
 }
 table.dataTable.dtr-inline.collapsed>tbody>tr>td.dtr-control:before:hover, 
 table.dataTable.dtr-inline.collapsed>tbody>tr>th.dtr-control:before:hover {
-    background-color: #5e5e5e;
+    background-color: var(--mc-dtr-hover-bg);
 }
 table.dataTable.dtr-inline.collapsed>tbody>tr>td.dtr-control:before, 
 table.dataTable.dtr-inline.collapsed>tbody>tr>th.dtr-control:before,
 table.dataTable td.dt-control:before {
-    background-color: #979797 !important;
-    border: 1.5px solid #616161 !important;
+    background-color: var(--mc-dtr-bg) !important;
+    border: 1.5px solid var(--mc-dtr-border) !important;
     border-radius: 2px !important;
-    color: #fff;
+    color: var(--mc-dtr-color);
     height: 1em;
     width: 1em;
     line-height: 1.25em;
@@ -37,7 +37,7 @@ table.dataTable td.dt-control:before {
 table.dataTable.dtr-inline.collapsed>tbody>tr.parent>td.dtr-control:before, 
 table.dataTable.dtr-inline.collapsed>tbody>tr.parent>th.dtr-control:before,
 table.dataTable td.dt-control:before {
-    background-color: #979797 !important;
+    background-color: var(--mc-dtr-active-bg) !important;
 }
 table.dataTable.table-striped>tbody>tr>td {
     vertical-align: middle;

data/web/css/build/014-mailcow.css

@@ -34,11 +34,44 @@
        url('/fonts/noto-sans-v12-latin_greek_cyrillic-700italic.woff') format('woff');
 }
 
+/* theme tokens, override in css/themes/<name>/theme.css */
+:root {
+  --mc-font-family: "Noto Sans","Helvetica Neue",Helvetica,Arial,sans-serif;
+  --mc-font-size: 10.5pt;
+  --mc-body-bg: #fbfbfb;
+  --mc-primary: #0F688D;
+  --mc-primary-border: #0d526d;
+  --mc-info: #148DBC;
+  --mc-info-border: #127ea8;
+  --mc-border: #dfdfdf;
+  --mc-surface: #fff;
+  --mc-surface-alt: #f7f7f7;
+  --mc-muted: #959595;
+  --mc-hr: #ccc;
+  --mc-progress-bg: #d5d5d5;
+  --mc-placeholder: rgb(137 137 137);
+  --mc-btn-outline-color: #000000;
+  --mc-btn-outline-border: #cfcfcf;
+  --mc-btn-outline-hover-bg: #f0f0f0;
+  --mc-btn-outline-active-bg: #f0f0f0;
+  --mc-btn-light-active-color: #fff;
+  --mc-btn-light-active-bg: #555;
+  --mc-btn-light-active-border: #4d4d4d;
+  --mc-dtr-bg: #979797;
+  --mc-dtr-border: #616161;
+  --mc-dtr-color: #fff;
+  --mc-dtr-hover-bg: #5e5e5e;
+  --mc-dtr-active-bg: #979797;
+  --mc-login-user: #28b62c;
+  --mc-login-domainadmin: #ffc107;
+  --mc-login-admin: #ff4136;
+}
+
 body {
   min-height: 100vh;
   display: flex;
   flex-direction: column;
-  background-color: #fbfbfb;
+  background-color: var(--mc-body-bg);
 }
 #maxmsgsize { min-width: 80px; }
 #slider1 .slider-selection {
@@ -86,17 +119,17 @@ body {
   max-height: 1000px
 }
 .nav-tabs .nav-link, .nav-tabs .nav-link.disabled, .nav-tabs .nav-link.disabled:hover, .nav-tabs .nav-link.disabled:focus {
-  border-color: #dfdfdf;
+  border-color: var(--mc-border);
 }
 .nav-tabs .nav-link.active, .nav-tabs .nav-item.show .nav-link {
-  border-color: #dfdfdf;
-  border-bottom: 1px solid #ffffff;
+  border-color: var(--mc-border);
+  border-bottom: 1px solid var(--mc-surface);
 }
 .nav-tabs .nav-link:hover, .nav-tabs .nav-link:focus {
-  border-color: #dfdfdf;
+  border-color: var(--mc-border);
 }
 .nav-tabs {
-  border-bottom: 1px solid #dfdfdf;
+  border-bottom: 1px solid var(--mc-border);
 }
 .bi {
   display: inline-block;
@@ -164,13 +197,13 @@ body.modal-open {
   padding-right: inherit !important;
 }
 body {
-  font-family: "Noto Sans","Helvetica Neue",Helvetica,Arial,sans-serif;
-  font-size: 10.5pt;
+  font-family: var(--mc-font-family);
+  font-size: var(--mc-font-size);
   line-height: 1.5;
 }
 html {
-  font-family: "Noto Sans","Helvetica Neue",Helvetica,Arial,sans-serif;
-  font-size: 10.5pt;
+  font-family: var(--mc-font-family);
+  font-size: var(--mc-font-size);
   line-height: 1.5;
 }
 #mailcow-alert {
@@ -206,7 +239,7 @@ legend {
   cursor: not-allowed;
 }
 .overlay {
-  background: #fff;
+  background: var(--mc-surface);
   position: absolute;
   z-index: 10000;
   top: 0; right: 0; bottom: 0; left: 0;
@@ -215,9 +248,6 @@ legend {
 .bootstrap-select.btn-group .no-results {
   display: none;
 }
-.bootstrap-select>.dropdown-toggle.bs-placeholder.btn-secondary {
-  color: rgb(197, 197, 197) !important;
-}
 .haveibeenpwned {
   cursor: pointer;
   -webkit-user-select: none;
@@ -239,7 +269,7 @@ legend {
 .footer {
   margin-top: 27px;
   margin-bottom: 20px;
-  color: #959595;
+  color: var(--mc-muted);
   display: flex;
   flex-direction: column;
 }
@@ -337,7 +367,7 @@ code {
   overflow-x: auto!important;
 }
 .well {
-  border: 1px solid #dfdfdf;
+  border: 1px solid var(--mc-border);
   background-color: #f9f9f9;
   padding: 10px;
 }
@@ -356,47 +386,47 @@ button[aria-expanded='true'] > .caret {
 }
 
 .list-group-details {
-  background: #fff;
+  background: var(--mc-surface);
 }
 .list-group-header {
-  background: #f7f7f7;
+  background: var(--mc-surface-alt);
 }
 
 
 .bg-primary, .alert-primary, .btn-primary {
-  background-color: #0F688D !important;
-  border-color: #0d526d !important;
+  background-color: var(--mc-primary) !important;
+  border-color: var(--mc-primary-border) !important;
 }
 .bg-info, .alert-info, .btn-info {
-  background-color: #148DBC !important;
-  border-color: #127ea8 !important;
+  background-color: var(--mc-info) !important;
+  border-color: var(--mc-info-border) !important;
 }
 
 .bootstrap-select>.dropdown-toggle.bs-placeholder.btn-secondary {
-  color: rgb(137 137 137)!important;
+  color: var(--mc-placeholder) !important;
 }
 
 .progress {
   height: 16px;
-  background-color: #d5d5d5;
+  background-color: var(--mc-progress-bg);
 }
 
 
 .btn-outline-secondary:hover {
-    background-color: #f0f0f0;
+    background-color: var(--mc-btn-outline-hover-bg);
 }
 .btn.btn-outline-secondary {
-  color: #000000 !important;
-  border-color: #cfcfcf !important;
+  color: var(--mc-btn-outline-color) !important;
+  border-color: var(--mc-btn-outline-border) !important;
 }
 .btn-check:checked+.btn-outline-secondary, .btn-check:active+.btn-outline-secondary, .btn-outline-secondary:active, .btn-outline-secondary.active, .btn-outline-secondary.dropdown-toggle.show {
-    background-color: #f0f0f0 !important;
+    background-color: var(--mc-btn-outline-active-bg) !important;
 }
 .btn-check:checked+.btn-light, .btn-check:active+.btn-light, .btn-light:active, .btn-light.active, .show>.btn-light.dropdown-toggle {
-    color: #fff;
-    background-color: #555;
+    color: var(--mc-btn-light-active-color);
+    background-color: var(--mc-btn-light-active-bg);
     background-image: none;
-    border-color: #4d4d4d;
+    border-color: var(--mc-btn-light-active-border);
 }
 .btn-check:checked+.btn-light:focus, .btn-check:active+.btn-light:focus, .btn-light:active:focus, .btn-light.active:focus, .show>.btn-light.dropdown-toggle:focus,
 .btn-check:focus+.btn-light, .btn-light:focus {
@@ -412,6 +442,26 @@ button[aria-expanded='true'] > .caret {
     text-decoration: none;
 }
 
+/* role marker on login pages, header icons differ too (color blindness) */
+.login-card::before {
+  content: "";
+  display: block;
+  height: 5px;
+  background: var(--mc-login-color);
+  border-radius: inherit;
+  border-bottom-left-radius: 0;
+  border-bottom-right-radius: 0;
+}
+.login-user {
+  --mc-login-color: var(--mc-login-user);
+}
+.login-domainadmin {
+  --mc-login-color: var(--mc-login-domainadmin);
+}
+.login-admin {
+  --mc-login-color: var(--mc-login-admin);
+}
+
 .hr-title {
   display: flex;
   align-items: center;
@@ -423,7 +473,7 @@ button[aria-expanded='true'] > .caret {
 .hr-title::after {
   content: "";
   flex: 1;
-  border-bottom: 1px solid #ccc;
+  border-bottom: 1px solid var(--mc-hr);
 }
 
 .hr-title:not(:empty)::before {