[Web] Add identity provider outage translation, remove unused needs_restart

[PHP] Update base to Alpine 3.24, build PECL extensions from GitHub

.github/ISSUE_TEMPLATE/Bug_report.yml

@@ -11,22 +11,35 @@ body:
         required: true
   - type: checkboxes
     attributes:
-      label: I've found a bug and checked that ...
-      description: Prior to placing the issue, please check following:** *(fill out each checkbox with an `X` once done)*
+      label: Checklist prior issue creation
+      description: Prior to creating the issue...
       options:
-      - label: ... I understand that not following the below instructions will result in immediate closure and/or deletion of my issue.
+      - label: I understand that failure to follow below instructions may cause this issue to be closed.
         required: true
-      - label: ... I have understood that this bug report is dedicated for bugs, and not for support-related inquiries.
+      - label: I understand that vague, incomplete or inaccurate information may cause this issue to be closed.
         required: true
-      - label: ... I have understood that answers are voluntary and community-driven, and not commercial support.
+      - label: I understand that this form is intended solely for reporting software bugs and not for support-related inquiries.
         required: true
-      - label: ... I have verified that my issue has not been already answered in the past. I also checked previous [issues](https://github.com/mailcow/mailcow-dockerized/issues).
+      - label: I understand that all responses are voluntary and community-driven, and do not constitute commercial support.
+        required: true
+      - label: I confirm that I have reviewed previous [issues](https://github.com/mailcow/mailcow-dockerized/issues) to ensure this matter has not already been addressed.
+        required: true
+      - label: I confirm that my environment meets all [prerequisite requirements](https://docs.mailcow.email/getstarted/prerequisite-system/) as specified in the official documentation.
         required: true
   - type: textarea
     attributes:
       label: Description
-      description: Please provide a brief description of the bug in 1-2 sentences. If applicable, add screenshots to help explain your problem. Very useful for bugs in mailcow UI.
-      render: plain text
+      description: Please provide a brief description of the bug. If applicable, add screenshots to help explain your problem. (Very useful for bugs in mailcow UI.)
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: "Steps to reproduce:"
+      description: "Please describe the steps to reproduce the bug. Screenshots can be added, if helpful."
+      placeholder: |-
+        1. ...
+        2. ...
+        3. ...
     validations:
       required: true
   - type: textarea
@@ -36,45 +49,36 @@ body:
       render: plain text
     validations:
       required: true
-  - type: textarea
-    attributes:
-      label: "Steps to reproduce:"
-      description: "Please describe the steps to reproduce the bug. Screenshots can be added, if helpful."
-      render: plain text
-      placeholder: |-
-        1. ...
-        2. ...
-        3. ...
-    validations:
-      required: true
   - type: markdown
     attributes:
       value: |
         ## System information
-        ### In this stage we would kindly ask you to attach general system information about your setup.
+        In this stage we would kindly ask you to attach general system information about your setup.
   - type: dropdown
     attributes:
       label: "Which branch are you using?"
-      description: "#### `git rev-parse --abbrev-ref HEAD`"
+      description: "#### Run: `git rev-parse --abbrev-ref HEAD`"
       multiple: false
       options:
-        - master
+        - master (stable)
+        - staging
         - nightly
     validations:
       required: true
   - type: dropdown
     attributes:
       label: "Which architecture are you using?"
-      description: "#### `uname -m`"
+      description: "#### Run: `uname -m`"
       multiple: false
       options:
-        - x86
+        - x86_64
         - ARM64 (aarch64)
     validations:
       required: true
   - type: input
     attributes:
       label: "Operating System:"
+      description: "#### Run: `lsb_release -ds`"
       placeholder: "e.g. Ubuntu 22.04 LTS"
     validations:
       required: true
@@ -93,43 +97,44 @@ body:
   - type: input
     attributes:
       label: "Virtualization technology:"
-      placeholder: "KVM, VMware, Xen, etc - **LXC and OpenVZ are not supported**"
+      description: "LXC and OpenVZ are not supported!"
+      placeholder: "KVM, VMware ESXi, Xen, etc"
     validations:
       required: true
   - type: input
     attributes:
       label: "Docker version:"
-      description: "#### `docker version`"
+      description: "#### Run: `docker version`"
       placeholder: "20.10.21"
     validations:
       required: true
   - type: input
     attributes:
       label: "docker-compose version or docker compose version:"
-      description: "#### `docker-compose version` or `docker compose version`"
+      description: "#### Run: `docker-compose version` or `docker compose version`"
       placeholder: "v2.12.2"
     validations:
       required: true
   - type: input
     attributes:
       label: "mailcow version:"
-      description: "#### ```git describe --tags `git rev-list --tags --max-count=1` ```"
-      placeholder: "2022-08"
+      description: "#### Run: ```git describe --tags `git rev-list --tags --max-count=1` ```"
+      placeholder: "2022-08x"
     validations:
       required: true
   - type: input
     attributes:
       label: "Reverse proxy:"
-      placeholder: "e.g. Nginx/Traefik"
+      placeholder: "e.g. nginx/Traefik, or none"
     validations:
       required: true
   - type: textarea
     attributes:
       label: "Logs of git diff:"
-      description: "#### Output of `git diff origin/master`, any other changes to the code? If so, **please post them**:"
+      description: "#### Output of `git diff origin/master`, any other changes to the code? Sanitize if needed. If so, **please post them**:"
       render: plain text
     validations:
-      required: true
+      required: false
   - type: textarea
     attributes:
       label: "Logs of iptables -L -vn:"

.github/workflows/close_old_issues_and_prs.yml

@@ -14,7 +14,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Mark/Close Stale Issues and Pull Requests 🗑️
-        uses: actions/stale@v9.1.0
+        uses: actions/stale@v11.0.0
         with:
           repo-token: ${{ secrets.STALE_ACTION_PAT }}
           days-before-stale: 60

.github/workflows/image_builds.yml

@@ -27,7 +27,7 @@ jobs:
           - "watchdog-mailcow"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Setup Docker
         run: |
           curl -sSL https://get.docker.com/ | CHANNEL=stable sudo sh

.github/workflows/pr_to_nightly.yml

@@ -8,11 +8,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
       - name: Run the Action
-        uses: devops-infra/action-pull-request@v0.6.0
+        uses: devops-infra/action-pull-request@v1.4.0
         with:
           github_token: ${{ secrets.PRTONIGHTLY_ACTION_PAT }}
           title: Automatic PR to nightly from ${{ github.event.repository.updated_at}}

.github/workflows/rebuild_backup_image.yml

@@ -13,24 +13,24 @@ jobs:
       packages: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v7
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.github/workflows/update_postscreen_access_list.yml

@@ -15,14 +15,14 @@ jobs:
    runs-on: ubuntu-latest
    steps:
     - name: Checkout
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
 
     - name: Generate postscreen_access.cidr
       run: |
           bash helper-scripts/update_postscreen_whitelist.sh
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v7
+      uses: peter-evans/create-pull-request@v8
       with:
         token: ${{ secrets.mailcow_action_Update_postscreen_access_cidr_pat }}
         commit-message: update postscreen_access.cidr

.gitignore

@@ -51,6 +51,7 @@ data/conf/sogo/cron.creds
 data/conf/sogo/custom-fulllogo.svg
 data/conf/sogo/custom-shortlogo.svg
 data/conf/sogo/custom-fulllogo.png
+data/conf/acme/dns-01.conf
 data/gitea/
 data/gogs/
 data/hooks/dovecot/*
@@ -63,6 +64,8 @@ data/web/templates/cache/*
 !data/web/templates/cache/.gitkeep
 data/web/.well-known/acme-challenge
 data/web/css/build/0081-custom-mailcow.css
+data/web/css/themes/*/
+!data/web/css/themes/servercow/
 data/web/inc/vars.local.inc.php
 data/web/inc/app_info.inc.php
 data/web/nextcloud*/
@@ -75,3 +78,6 @@ refresh_images.sh
 update_diffs/
 create_cold_standby.sh
 !data/conf/nginx/mailcow_auth.conf
+data/conf/postfix/postfix-tlspol
+tests/php/vendor/
+tests/php/.phpunit.cache/

CONTRIBUTING.md

@@ -1,11 +1,11 @@
 # Contribution Guidelines
-**_Last modified on 15th August 2024_**
+**_Last modified on 12th November 2025_**
 
 First of all, thank you for wanting to provide a bugfix or a new feature for the mailcow community, it's because of your help that the project can continue to grow!
 
 As we want to keep mailcow's development structured we setup these Guidelines which helps you to create your issue/pull request accordingly.
 
-**PLEASE NOTE, THAT WE MIGHT CLOSE ISSUES/PULL REQUESTS IF THEY DON'T FULLFIL OUR WRITTEN GUIDELINES WRITTEN INSIDE THIS DOCUMENT**. So please check this guidelines before you propose a Issue/Pull Request.
+**PLEASE NOTE, THAT WE WILL CLOSE ISSUES/PULL REQUESTS IF THEY DON'T FULFILL OUR WRITTEN GUIDELINES WRITTEN INSIDE THIS DOCUMENT**. So please check this guidelines before you propose a Issue/Pull Request.
 
 ## Topics
 
@@ -27,14 +27,18 @@ However, please note the following regarding pull requests:
 6. Please **ALWAYS** create the actual pull request against the staging branch and **NEVER** directly against the master branch. *If you forget to do this, our moobot will remind you to switch the branch to staging.*
 7. Wait for a merge commit: It may happen that we do not accept your pull request immediately or sometimes not at all for various reasons. Please do not be disappointed if this is the case. We always endeavor to incorporate any meaningful changes from the community into the mailcow project.
 8. If you are planning larger and therefore more complex pull requests, it would be advisable to first announce this in a separate issue and then start implementing it after the idea has been accepted in order to avoid unnecessary frustration and effort!
+9. If your PR requires a Docker image rebuild (changes to Dockerfiles or files in data/Dockerfiles/), update the image tag in docker-compose.yml. Use the base-image versioning (e.g. ghcr.io/mailcow/sogo:5.12.4 → :5.12.5 for version bumps; append a letter for patch fixes, e.g. :5.12.4a). Follow this scheme.
 
 ---
 
 ## Issue Reporting
-**_Last modified on 15th August 2024_**
+**_Last modified on 12th November 2025_**
 
 If you plan to report a issue within mailcow please read and understand the following rules:
 
+### Security disclosures / Security-related fixes
+- Security vulnerabilities and security fixes must always be reported confidentially first to the contact address specified in SECURITY.md before they are integrated, published, or publicly disclosed in issues/PRs. Please wait for a response from the specified contact to ensure coordinated and responsible disclosure.
+
 ### Issue Reporting Guidelines
 
 1. **ONLY** use the issue tracker for bug reports or improvement requests and NOT for support questions. For support questions you can either contact the [mailcow community on Telegram](https://docs.mailcow.email/#community-support-and-chat) or the mailcow team directly in exchange for a [support fee](https://docs.mailcow.email/#commercial-support).

README.md

@@ -23,10 +23,15 @@ A big thank you to everyone supporting us on GitHub Sponsors—your contribution
   <a href="https://www.maehdros.com/" target=_blank><img
     src="https://avatars.githubusercontent.com/u/173894712" height="58"
   /></a>
+  <a href="https://garske-systems.de/" target=_blank><img
+    src="https://garske-systems.de/img/logo_min.png" height="58"
+  /></a>
 
 ### 50$/Month Sponsors
   <a href="https://github.com/vnukhr" target=_blank><img
     src="https://avatars.githubusercontent.com/u/7805987?s=52&v=4" height="58"
+  /></a><a href="https://github.com/humiico" target=_blank><img
+    src="https://avatars.githubusercontent.com/u/110134705?s=52&v=4" height="58"
   /></a>
 
 ## Info, documentation and support

_modules/scripts/core.sh

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+# _modules/scripts/core.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+# ANSI color for red errors
+RED='\e[31m'
+GREEN='\e[32m'
+YELLOW='\e[33m'
+BLUE='\e[34m'
+MAGENTA='\e[35m'
+LIGHT_RED='\e[91m'
+LIGHT_GREEN='\e[92m'
+NC='\e[0m'
+
+caller="${BASH_SOURCE[1]##*/}"
+
+get_installed_tools(){
+    for bin in openssl curl docker git awk sha1sum grep cut jq; do
+        if [[ -z $(command -v ${bin}) ]]; then
+          echo "Error: Cannot find command '${bin}'. Cannot proceed."
+          echo "Solution: Please review system requirements and install requirements. Then, re-run the script."
+          echo "See System Requirements: https://docs.mailcow.email/getstarted/install/"
+          echo "Exiting..."
+          exit 1
+        fi
+    done
+
+    if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo -e "${LIGHT_RED}BusyBox grep detected, please install gnu grep, \"apk add --no-cache --upgrade grep\"${NC}"; exit 1; fi
+    # This will also cover sort
+    if cp --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo -e "${LIGHT_RED}BusyBox cp detected, please install coreutils, \"apk add --no-cache --upgrade coreutils\"${NC}"; exit 1; fi
+    if sed --help 2>&1 | head -n 1 | grep -q -i "busybox"; then echo -e "${LIGHT_RED}BusyBox sed detected, please install gnu sed, \"apk add --no-cache --upgrade sed\"${NC}"; exit 1; fi
+}
+
+get_docker_version(){
+    # Check Docker Version (need at least 24.X)
+    docker_version=$(docker version --format '{{.Server.Version}}' | cut -d '.' -f 1)
+}
+
+get_compose_type(){
+  if docker compose > /dev/null 2>&1; then
+    if docker compose version --short | grep -e "^[2-9]\." -e "^v[2-9]\." -e "^[1-9][0-9]\." -e "^v[1-9][0-9]\." > /dev/null 2>&1; then
+      COMPOSE_VERSION=native
+      COMPOSE_COMMAND="docker compose"
+      if [[ "$caller" == "update.sh" ]]; then
+        sed -i 's/^DOCKER_COMPOSE_VERSION=.*/DOCKER_COMPOSE_VERSION=native/' "$SCRIPT_DIR/mailcow.conf"
+      fi
+      echo -e "\e[33mFound Docker Compose Plugin (native).\e[0m"
+      echo -e "\e[33mSetting the DOCKER_COMPOSE_VERSION Variable to native\e[0m"
+      sleep 2
+      echo -e "\e[33mNotice: You'll have to update this Compose Version via your Package Manager manually!\e[0m"
+    else
+      echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m"
+      echo -e "\e[31mPlease update/install it manually regarding to this doc site: https://docs.mailcow.email/install/\e[0m"
+      exit 1
+    fi
+  elif docker-compose > /dev/null 2>&1; then
+  if ! [[ $(alias docker-compose 2> /dev/null) ]] ; then
+    if docker-compose version --short | grep -e "^[2-9]\." -e "^[1-9][0-9]\." > /dev/null 2>&1; then
+      COMPOSE_VERSION=standalone
+      COMPOSE_COMMAND="docker-compose"
+      if [[ "$caller" == "update.sh" ]]; then
+        sed -i 's/^DOCKER_COMPOSE_VERSION=.*/DOCKER_COMPOSE_VERSION=standalone/' "$SCRIPT_DIR/mailcow.conf"
+      fi
+      echo -e "\e[33mFound Docker Compose Standalone.\e[0m"
+      echo -e "\e[33mSetting the DOCKER_COMPOSE_VERSION Variable to standalone\e[0m"
+      sleep 2
+      echo -e "\e[33mNotice: For an automatic update of docker-compose please use the update_compose.sh scripts located at the helper-scripts folder.\e[0m"
+    else
+      echo -e "\e[31mCannot find Docker Compose with a Version Higher than 2.X.X.\e[0m"
+      echo -e "\e[31mPlease update/install manually regarding to this doc site: https://docs.mailcow.email/install/\e[0m"
+      exit 1
+    fi
+  fi
+  else
+    echo -e "\e[31mCannot find Docker Compose.\e[0m"
+    echo -e "\e[31mPlease install it regarding to this doc site: https://docs.mailcow.email/install/\e[0m"
+    exit 1
+  fi
+}
+
+detect_bad_asn() {
+  echo -e "\e[33mDetecting if your IP is listed on Spamhaus Bad ASN List...\e[0m"
+  response=$(curl --connect-timeout 15 --max-time 30 -s -o /dev/null -w "%{http_code}" "https://asn-check.mailcow.email")
+  if [ "$response" -eq 503 ]; then
+    if [ -z "$SPAMHAUS_DQS_KEY" ]; then
+      echo -e "\e[33mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS public blocklists for Postfix.\e[0m"
+      echo -e "\e[33mmailcow did not detected a value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf!\e[0m"
+      sleep 2
+      echo ""
+      echo -e "\e[33mTo use the Spamhaus DNS Blocklists again, you will need to create a FREE account for their Data Query Service (DQS) at: https://www.spamhaus.com/free-trial/sign-up-for-a-free-data-query-service-account\e[0m"
+      echo -e "\e[33mOnce done, enter your DQS API key in mailcow.conf and mailcow will do the rest for you!\e[0m"
+      echo ""
+      sleep 2
+    else
+      echo -e "\e[33mYour server's public IP uses an AS that is blocked by Spamhaus to use their DNS public blocklists for Postfix.\e[0m"
+      echo -e "\e[32mmailcow detected a Value for the variable SPAMHAUS_DQS_KEY inside mailcow.conf. Postfix will use DQS with the given API key...\e[0m"
+    fi
+  elif [ "$response" -eq 200 ]; then
+    echo -e "\e[33mCheck completed! Your IP is \e[32mclean\e[0m"
+  elif [ "$response" -eq 429 ]; then
+    echo -e "\e[33mCheck completed! \e[31mYour IP seems to be rate limited on the ASN Check service... please try again later!\e[0m"
+  else
+    echo -e "\e[31mCheck failed! \e[0mMaybe a DNS or Network problem?\e[0m"
+  fi
+}
+
+check_online_status() {
+  CHECK_ONLINE_DOMAINS=('https://github.com' 'https://hub.docker.com')
+  for domain in "${CHECK_ONLINE_DOMAINS[@]}"; do
+    if timeout 6 curl --head --silent --output /dev/null ${domain}; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+prefetch_images() {
+  [[ -z ${BRANCH} ]] && { echo -e "\e[33m\nUnknown branch...\e[0m"; exit 1; }
+  git fetch origin #${BRANCH}
+  while read image; do
+    RET_C=0
+    until docker pull "${image}"; do
+      RET_C=$((RET_C + 1))
+      echo -e "\e[33m\nError pulling $image, retrying...\e[0m"
+      [ ${RET_C} -gt 3 ] && { echo -e "\e[31m\nToo many failed retries, exiting\e[0m"; exit 1; }
+      sleep 1
+    done
+  done < <(git show "origin/${BRANCH}:docker-compose.yml" | grep "image:" | awk '{ gsub("image:","", $3); print $2 }')
+}
+
+docker_garbage() {
+  SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
+  IMGS_TO_DELETE=()
+
+  declare -A IMAGES_INFO
+  COMPOSE_IMAGES=($(grep -oP "image: \K(ghcr\.io/)?mailcow.+" "${SCRIPT_DIR}/docker-compose.yml"))
+
+  for existing_image in $(docker images --format "{{.ID}}:{{.Repository}}:{{.Tag}}" | grep -E '(mailcow/|ghcr\.io/mailcow/)'); do
+      ID=$(echo "$existing_image" | cut -d ':' -f 1)
+      REPOSITORY=$(echo "$existing_image" | cut -d ':' -f 2)
+      TAG=$(echo "$existing_image" | cut -d ':' -f 3)
+
+      if [[ "$REPOSITORY" == "mailcow/backup" || "$REPOSITORY" == "ghcr.io/mailcow/backup" ]]; then
+          if [[ "$TAG" != "<none>" ]]; then
+              continue
+          fi
+      fi
+
+      if [[ " ${COMPOSE_IMAGES[@]} " =~ " ${REPOSITORY}:${TAG} " ]]; then
+          continue
+      else
+          IMGS_TO_DELETE+=("$ID")
+          IMAGES_INFO["$ID"]="$REPOSITORY:$TAG"
+      fi
+  done
+
+  if [[ ! -z ${IMGS_TO_DELETE[*]} ]]; then
+      echo "The following unused mailcow images were found:"
+      for id in "${IMGS_TO_DELETE[@]}"; do
+          echo "    ${IMAGES_INFO[$id]} ($id)"
+      done
+
+      if [ -z "$FORCE" ]; then
+          read -r -p "Do you want to delete them to free up some space? [y/N] " response
+          if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+              docker rmi ${IMGS_TO_DELETE[*]}
+          else
+              echo "OK, skipped."
+          fi
+      else
+          echo "Running in forced mode! Force removing old mailcow images..."
+          docker rmi ${IMGS_TO_DELETE[*]}
+      fi
+      echo -e "\e[32mFurther cleanup...\e[0m"
+      echo "If you want to cleanup further garbage collected by Docker, please make sure all containers are up and running before cleaning your system by executing \"docker system prune\""
+  fi
+}
+
+in_array() {
+  local e match="$1"
+  shift
+  for e; do [[ "$e" == "$match" ]] && return 0; done
+  return 1
+}
+
+detect_major_update() {
+  if [ ${BRANCH} == "master" ]; then
+    # Array with major versions
+    # Add major versions here
+    MAJOR_VERSIONS=(
+      "2025-02"
+      "2025-03"
+      "2025-09"
+    )
+
+    current_version=""
+    if [[ -f "${SCRIPT_DIR}/data/web/inc/app_info.inc.php" ]]; then
+      current_version=$(grep 'MAILCOW_GIT_VERSION' ${SCRIPT_DIR}/data/web/inc/app_info.inc.php | sed -E 's/.*MAILCOW_GIT_VERSION="([^"]+)".*/\1/')
+    fi
+    if [[ -z "$current_version" ]]; then
+      return 1
+    fi
+    release_url="https://github.com/mailcow/mailcow-dockerized/releases/tag"
+
+    updates_to_apply=()
+
+    for version in "${MAJOR_VERSIONS[@]}"; do
+      if [[ "$current_version" < "$version" ]]; then
+        updates_to_apply+=("$version")
+      fi
+    done
+
+    if [[ ${#updates_to_apply[@]} -gt 0 ]]; then
+      echo -e "\e[33m\nMAJOR UPDATES to be applied:\e[0m"
+      for update in "${updates_to_apply[@]}"; do
+        echo "$update - $release_url/$update"
+      done
+
+      echo -e "\nPlease read the release notes before proceeding."
+      read -p "Do you want to proceed with the update? [y/n] " response
+      if [[ "${response}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+        echo "Proceeding with the update..."
+      else
+        echo "Update canceled. Exiting."
+        exit 1
+      fi
+    fi
+  fi
+}

_modules/scripts/ipv6_controller.sh

@@ -0,0 +1,239 @@
+#!/usr/bin/env bash
+# _modules/scripts/ipv6_controller.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+# 1) Check if the host supports IPv6
+get_ipv6_support() {
+  # ---- helper: probe external IPv6 connectivity without DNS ----
+  _probe_ipv6_connectivity() {
+    # Use literal, always-on IPv6 echo responders (no DNS required)
+    local PROBE_IPS=("2001:4860:4860::8888" "2606:4700:4700::1111")
+    local ip rc=1
+
+    for ip in "${PROBE_IPS[@]}"; do
+      if command -v ping6 &>/dev/null; then
+        ping6 -c1 -W2 "$ip" &>/dev/null || ping6 -c1 -w2 "$ip" &>/dev/null
+        rc=$?
+      elif command -v ping &>/dev/null; then
+        ping -6 -c1 -W2 "$ip" &>/dev/null || ping -6 -c1 -w2 "$ip" &>/dev/null
+        rc=$?
+      else
+        rc=1
+      fi
+      [[ $rc -eq 0 ]] && return 0
+    done
+    return 1
+  }
+
+  if [[ ! -f /proc/net/if_inet6 ]] || grep -qs '^1' /proc/sys/net/ipv6/conf/all/disable_ipv6 2>/dev/null; then
+    DETECTED_IPV6=false
+    echo -e "${YELLOW}IPv6 not detected on host – ${LIGHT_RED}IPv6 is administratively disabled${YELLOW}.${NC}"
+    return
+  fi
+
+  if ip -6 route show default 2>/dev/null | grep -qE '^default'; then
+    echo -e "${YELLOW}Default IPv6 route found – testing external IPv6 connectivity...${NC}"
+    if _probe_ipv6_connectivity; then
+      DETECTED_IPV6=true
+      echo -e "IPv6 detected on host – ${LIGHT_GREEN}leaving IPv6 support enabled${YELLOW}.${NC}"
+    else
+      DETECTED_IPV6=false
+      echo -e "${YELLOW}Default IPv6 route present but external IPv6 connectivity failed – ${LIGHT_RED}disabling IPv6 support${YELLOW}.${NC}"
+    fi
+    return
+  fi
+
+  if ip -6 addr show scope global 2>/dev/null | grep -q 'inet6'; then
+    DETECTED_IPV6=false
+    echo -e "${YELLOW}Global IPv6 address present but no default route – ${LIGHT_RED}disabling IPv6 support${YELLOW}.${NC}"
+    return
+  fi
+
+  if ip -6 addr show scope link 2>/dev/null | grep -q 'inet6'; then
+    echo -e "${YELLOW}Only link-local IPv6 addresses found – testing external IPv6 connectivity...${NC}"
+    if _probe_ipv6_connectivity; then
+      DETECTED_IPV6=true
+      echo -e "External IPv6 connectivity available – ${LIGHT_GREEN}leaving IPv6 support enabled${YELLOW}.${NC}"
+    else
+      DETECTED_IPV6=false
+      echo -e "${YELLOW}Only link-local IPv6 present and no external connectivity – ${LIGHT_RED}disabling IPv6 support${YELLOW}.${NC}"
+    fi
+    return
+  fi
+
+  DETECTED_IPV6=false
+  echo -e "${YELLOW}IPv6 not detected on host – ${LIGHT_RED}disabling IPv6 support${YELLOW}.${NC}"
+}
+
+# 2) Ensure Docker daemon.json has (or create) the required IPv6 settings
+docker_daemon_edit(){
+  DOCKER_DAEMON_CONFIG="/etc/docker/daemon.json"
+  DOCKER_MAJOR=$(docker version --format '{{.Server.Version}}' 2>/dev/null | cut -d. -f1)
+  MISSING=()
+
+  _has_kv() { grep -Eq "\"$1\"[[:space:]]*:[[:space:]]*$2" "$DOCKER_DAEMON_CONFIG" 2>/dev/null; }
+
+  if [[ -f "$DOCKER_DAEMON_CONFIG" ]]; then
+
+    # reject empty or whitespace-only file immediately
+    if [[ ! -s "$DOCKER_DAEMON_CONFIG" ]] || ! grep -Eq '[{}]' "$DOCKER_DAEMON_CONFIG"; then
+      echo -e "${RED}ERROR: $DOCKER_DAEMON_CONFIG exists but is empty or contains no JSON braces – please initialize it with valid JSON (e.g. {}).${NC}"
+      exit 1
+    fi
+
+    # Validate JSON if jq is present
+    if command -v jq &>/dev/null && ! jq empty "$DOCKER_DAEMON_CONFIG" &>/dev/null; then
+      echo -e "${RED}ERROR: Invalid JSON in $DOCKER_DAEMON_CONFIG – please correct manually.${NC}"
+      exit 1
+    fi
+
+    # Gather missing keys
+    ! _has_kv ipv6 true && MISSING+=("ipv6: true")
+
+    # For Docker < 28, keep requiring fixed-cidr-v6 (default bridge needs it on old engines)
+    if [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 28 ]]; then
+      ! grep -Eq '"fixed-cidr-v6"[[:space:]]*:[[:space:]]*".+"' "$DOCKER_DAEMON_CONFIG" \
+                                && MISSING+=('fixed-cidr-v6: "fd00:dead:beef:c0::/80"')
+    fi
+
+    # For Docker < 27, ip6tables needed and was tied to experimental in older releases
+    if [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 27 ]]; then
+      _has_kv ipv6 true && ! _has_kv ip6tables true && MISSING+=("ip6tables: true")
+      ! _has_kv experimental true && MISSING+=("experimental: true")
+    fi
+
+    # Fix if needed
+    if ((${#MISSING[@]}>0)); then
+      echo -e "${MAGENTA}Your daemon.json is missing: ${YELLOW}${MISSING[*]}${NC}"
+      if [[ -n "$FORCE" ]]; then
+        ans=Y
+      else
+        read -p "Would you like to update $DOCKER_DAEMON_CONFIG now? [Y/n] " ans
+        ans=${ans:-Y}
+      fi
+
+      if [[ $ans =~ ^[Yy]$ ]]; then
+        cp "$DOCKER_DAEMON_CONFIG" "${DOCKER_DAEMON_CONFIG}.bak"
+        if command -v jq &>/dev/null; then
+          TMP=$(mktemp)
+          # Base filter: ensure ipv6 = true
+          JQ_FILTER='.ipv6 = true'
+
+          # Add fixed-cidr-v6 only for Docker < 28
+          if [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 28 ]]; then
+            JQ_FILTER+=' | .["fixed-cidr-v6"] = (.["fixed-cidr-v6"] // "fd00:dead:beef:c0::/80")'
+          fi
+
+          # Add ip6tables/experimental only for Docker < 27
+          if [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 27 ]]; then
+            JQ_FILTER+=' | .ip6tables = true | .experimental = true'
+          fi
+
+          jq "$JQ_FILTER" "$DOCKER_DAEMON_CONFIG" >"$TMP" && mv "$TMP" "$DOCKER_DAEMON_CONFIG"
+          echo -e "${LIGHT_GREEN}daemon.json updated. Restarting Docker...${NC}"
+          (command -v systemctl &>/dev/null && systemctl restart docker) || service docker restart
+          echo -e "${YELLOW}Docker restarted.${NC}"
+        else
+          echo -e "${RED}Please install jq or manually update daemon.json and restart Docker.${NC}"
+          exit 1
+        fi
+      else
+        echo -e "${YELLOW}User declined Docker update – please insert these changes manually:${NC}"
+        echo "${MISSING[*]}"
+        exit 1
+      fi
+    fi
+
+  else
+    # Create new daemon.json if missing
+    if [[ -n "$FORCE" ]]; then
+      ans=Y
+    else
+      read -p "$DOCKER_DAEMON_CONFIG not found. Create it with IPv6 settings? [Y/n] " ans
+      ans=${ans:-Y}
+    fi
+
+    if [[ $ans =~ ^[Yy]$ ]]; then
+      mkdir -p "$(dirname "$DOCKER_DAEMON_CONFIG")"
+      if [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 27 ]]; then
+        cat > "$DOCKER_DAEMON_CONFIG" <<EOF
+{
+  "ipv6": true,
+  "fixed-cidr-v6": "fd00:dead:beef:c0::/80",
+  "ip6tables": true,
+  "experimental": true
+}
+EOF
+      elif [[ -n "$DOCKER_MAJOR" && "$DOCKER_MAJOR" -lt 28 ]]; then
+        cat > "$DOCKER_DAEMON_CONFIG" <<EOF
+{
+  "ipv6": true,
+  "fixed-cidr-v6": "fd00:dead:beef:c0::/80"
+}
+EOF
+      else
+        # Docker 28+: ipv6 works without fixed-cidr-v6
+        cat > "$DOCKER_DAEMON_CONFIG" <<EOF
+{
+  "ipv6": true
+}
+EOF
+      fi
+      echo -e "${GREEN}Created $DOCKER_DAEMON_CONFIG with IPv6 settings.${NC}"
+      echo "Restarting Docker..."
+      (command -v systemctl &>/dev/null && systemctl restart docker) || service docker restart
+      echo "Docker restarted."
+    else
+      echo "User declined to create daemon.json – please manually merge the docker daemon with these configs:"
+      echo "${MISSING[*]}"
+      exit 1
+    fi
+  fi
+}
+
+# 3) Main wrapper for generate_config.sh and update.sh
+configure_ipv6() {
+  # detect manual override if mailcow.conf is present
+  if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]] && grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
+    MANUAL_SETTING=$(grep '^ENABLE_IPV6=' "$MAILCOW_CONF" | cut -d= -f2)
+  elif [[ -z "$MAILCOW_CONF" ]] && [[ -n "${ENABLE_IPV6:-}" ]]; then
+    MANUAL_SETTING="$ENABLE_IPV6"
+  else
+    MANUAL_SETTING=""
+  fi
+
+  get_ipv6_support
+
+  # if user manually set it, check for mismatch
+  if [[ "$DETECTED_IPV6" != "true" ]]; then
+    if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]]; then
+      if grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
+        sed -i 's/^ENABLE_IPV6=.*/ENABLE_IPV6=false/' "$MAILCOW_CONF"
+      else
+        echo "ENABLE_IPV6=false" >> "$MAILCOW_CONF"
+      fi
+    else
+      export IPV6_BOOL=false
+    fi
+    echo "Skipping Docker IPv6 configuration because host does not support IPv6."
+    echo "Make sure to check if your docker daemon.json does not include \"enable_ipv6\": true if you do not want IPv6."
+    echo "IPv6 configuration complete: ENABLE_IPV6=false"
+    sleep 2
+    return
+  fi
+
+  docker_daemon_edit
+
+  if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]]; then
+    if grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
+      sed -i 's/^ENABLE_IPV6=.*/ENABLE_IPV6=true/' "$MAILCOW_CONF"
+    else
+      echo "ENABLE_IPV6=true" >> "$MAILCOW_CONF"
+    fi
+  else
+    export IPV6_BOOL=true
+  fi
+
+  echo "IPv6 configuration complete: ENABLE_IPV6=true"
+}

_modules/scripts/migrate_options.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# _modules/scripts/migrate_options.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+migrate_config_options() {
+
+  sed -i --follow-symlinks '$a\' mailcow.conf
+
+  KEYS=(
+    SOLR_HEAP
+    SKIP_SOLR
+    SOLR_PORT
+    FLATCURVE_EXPERIMENTAL
+    DISABLE_IPv6
+    ACME_CONTACT
+  )
+
+  for key in "${KEYS[@]}"; do
+    if grep -q "${key}" mailcow.conf; then
+      case "${key}" in
+        SOLR_HEAP)
+          echo "Removing ${key} in mailcow.conf"
+          sed -i '/# Solr heap size in MB\b/d' mailcow.conf
+          sed -i '/# Solr is a prone to run\b/d' mailcow.conf
+          sed -i '/SOLR_HEAP\b/d' mailcow.conf
+          ;;
+        SKIP_SOLR)
+          echo "Removing ${key} in mailcow.conf"
+          sed -i '/\bSkip Solr on low-memory\b/d' mailcow.conf
+          sed -i '/\bSolr is disabled by default\b/d' mailcow.conf
+          sed -i '/\bDisable Solr or\b/d' mailcow.conf
+          sed -i '/\bSKIP_SOLR\b/d' mailcow.conf
+          ;;
+        SOLR_PORT)
+          echo "Removing ${key} in mailcow.conf"
+          sed -i '/\bSOLR_PORT\b/d' mailcow.conf
+          ;;
+        FLATCURVE_EXPERIMENTAL)
+          echo "Removing ${key} in mailcow.conf"
+          sed -i '/\bFLATCURVE_EXPERIMENTAL\b/d' mailcow.conf
+          ;;
+        DISABLE_IPv6)
+          echo "Migrating ${key} to ENABLE_IPv6 in mailcow.conf"
+          local old=$(grep '^DISABLE_IPv6=' "mailcow.conf" | cut -d'=' -f2)
+          local new
+          if [[ "$old" == "y" ]]; then
+            new="false"
+          else
+            new="true"
+          fi
+          sed -i '/^DISABLE_IPv6=/d' "mailcow.conf"
+          echo "ENABLE_IPV6=$new" >> "mailcow.conf"
+          ;;
+        ACME_CONTACT)
+          echo "Deleting obsoleted ${key} in mailcow.conf"
+          sed -i '/^# Lets Encrypt registration contact information/d' mailcow.conf
+          sed -i '/^# Optional: Leave empty for none/d' mailcow.conf
+          sed -i '/^# This value is only used on first order!/d' mailcow.conf
+          sed -i '/^# Setting it at a later point will require the following steps:/d' mailcow.conf
+          sed -i '/^# https:\/\/docs.mailcow.email\/troubleshooting\/debug-reset_tls\//d' mailcow.conf
+          sed -i '/^ACME_CONTACT=.*/d' mailcow.conf
+          sed -i '/^#ACME_CONTACT=.*/d' mailcow.conf
+          ;;
+      esac
+    fi
+  done
+
+  solr_volume=$(docker volume ls -qf name=^${COMPOSE_PROJECT_NAME}_solr-vol-1)
+  if [[ -n $solr_volume ]]; then
+    echo -e "\e[34mSolr has been replaced within mailcow since 2025-01.\nThe volume $solr_volume is unused.\e[0m"
+    sleep 1
+    if [ ! "$FORCE" ]; then
+      read -r -p "Remove $solr_volume? [y/N] " response
+      if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+        echo -e "\e[33mRemoving $solr_volume...\e[0m"
+        docker volume rm $solr_volume || echo -e "\e[31mFailed to remove. Remove it manually!\e[0m"
+        echo -e "\e[32mSuccessfully removed $solr_volume!\e[0m"
+      else
+        echo -e "Not removing $solr_volume. Run \`docker volume rm $solr_volume\` manually if needed."
+      fi
+    else
+      echo -e "\e[33mForce removing $solr_volume...\e[0m"
+      docker volume rm $solr_volume || echo -e "\e[31mFailed to remove. Remove it manually!\e[0m"
+      echo -e "\e[32mSuccessfully removed $solr_volume!\e[0m"
+    fi
+  fi
+
+  # Delete old fts.conf before forced switch to flatcurve to ensure update is working properly
+  FTS_CONF_PATH="${SCRIPT_DIR}/data/conf/dovecot/conf.d/fts.conf"
+  if [[ -f "$FTS_CONF_PATH" ]]; then
+    if grep -q "Autogenerated by mailcow" "$FTS_CONF_PATH"; then
+      rm -rf $FTS_CONF_PATH
+    fi
+  fi
+}

_modules/scripts/new_options.sh

@@ -0,0 +1,317 @@
+#!/usr/bin/env bash
+# _modules/scripts/new_options.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+adapt_new_options() {
+
+  CONFIG_ARRAY=(
+  "AUTODISCOVER_SAN"
+  "SKIP_LETS_ENCRYPT"
+  "SKIP_SOGO"
+  "USE_WATCHDOG"
+  "WATCHDOG_NOTIFY_EMAIL"
+  "WATCHDOG_NOTIFY_WEBHOOK"
+  "WATCHDOG_NOTIFY_WEBHOOK_BODY"
+  "WATCHDOG_NOTIFY_BAN"
+  "WATCHDOG_NOTIFY_START"
+  "WATCHDOG_EXTERNAL_CHECKS"
+  "WATCHDOG_SUBJECT"
+  "SKIP_CLAMD"
+  "SKIP_OLEFY"
+  "SKIP_IP_CHECK"
+  "ADDITIONAL_SAN"
+  "DOVEADM_PORT"
+  "IPV4_NETWORK"
+  "IPV6_NETWORK"
+  "LOG_LINES"
+  "SNAT_TO_SOURCE"
+  "SNAT6_TO_SOURCE"
+  "COMPOSE_PROJECT_NAME"
+  "DOCKER_COMPOSE_VERSION"
+  "SQL_PORT"
+  "API_KEY"
+  "API_KEY_READ_ONLY"
+  "API_ALLOW_FROM"
+  "MAILDIR_GC_TIME"
+  "MAILDIR_SUB"
+  "ACL_ANYONE"
+  "FTS_HEAP"
+  "FTS_PROCS"
+  "SKIP_FTS"
+  "ENABLE_SSL_SNI"
+  "ALLOW_ADMIN_EMAIL_LOGIN"
+  "SKIP_HTTP_VERIFICATION"
+  "SOGO_EXPIRE_SESSION"
+  "SOGO_URL_ENCRYPTION_KEY"
+  "REDIS_PORT"
+  "REDISPASS"
+  "DOVECOT_MASTER_USER"
+  "DOVECOT_MASTER_PASS"
+  "MAILCOW_PASS_SCHEME"
+  "ADDITIONAL_SERVER_NAMES"
+  "WATCHDOG_VERBOSE"
+  "WEBAUTHN_ONLY_TRUSTED_VENDORS"
+  "SPAMHAUS_DQS_KEY"
+  "SKIP_UNBOUND_HEALTHCHECK"
+  "DISABLE_NETFILTER_ISOLATION_RULE"
+  "HTTP_REDIRECT"
+  "ENABLE_IPV6"
+  "ACME_DNS_CHALLENGE"
+  "ACME_DNS_PROVIDER"
+  "ACME_ACCOUNT_EMAIL"
+  )
+
+  sed -i --follow-symlinks '$a\' mailcow.conf
+  for option in ${CONFIG_ARRAY[@]}; do
+    if grep -q "^#\?${option}=" mailcow.conf; then
+      continue
+    fi
+
+    echo "Adding new option \"${option}\" to mailcow.conf"
+
+    case "${option}" in
+        AUTODISCOVER_SAN)
+            echo '# Obtain certificates for autodiscover.* and autoconfig.* domains.' >> mailcow.conf
+            echo '# This can be useful to switch off in case you are in a scenario where a reverse proxy already handles those.' >> mailcow.conf
+            echo '# There are mixed scenarios where ports 80,443 are occupied and you do not want to share certs' >> mailcow.conf
+            echo '# between services. So acme-mailcow obtains for maildomains and all web-things get handled' >> mailcow.conf
+            echo '# in the reverse proxy.' >> mailcow.conf
+            echo 'AUTODISCOVER_SAN=y' >> mailcow.conf
+            ;;
+
+        DOCKER_COMPOSE_VERSION)
+            echo "# Used Docker Compose version" >> mailcow.conf
+            echo "# Switch here between native (compose plugin) and standalone" >> mailcow.conf
+            echo "# For more informations take a look at the mailcow docs regarding the configuration options." >> mailcow.conf
+            echo "# Normally this should be untouched but if you decided to use either of those you can switch it manually here." >> mailcow.conf
+            echo "# Please be aware that at least one of those variants should be installed on your machine or mailcow will fail." >> mailcow.conf
+            echo "" >> mailcow.conf
+            echo "DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION}" >> mailcow.conf
+            ;;
+
+        DOVEADM_PORT)
+            echo "DOVEADM_PORT=127.0.0.1:19991" >> mailcow.conf
+            ;;
+
+        LOG_LINES)
+            echo '# Max log lines per service to keep in Redis logs' >> mailcow.conf
+            echo "LOG_LINES=9999" >> mailcow.conf
+            ;;
+        IPV4_NETWORK)
+            echo '# Internal IPv4 /24 subnet, format n.n.n. (expands to n.n.n.0/24)' >> mailcow.conf
+            echo "IPV4_NETWORK=172.22.1" >> mailcow.conf
+            ;;
+        IPV6_NETWORK)
+            echo '# Internal IPv6 subnet in fc00::/7' >> mailcow.conf
+            echo "IPV6_NETWORK=fd4d:6169:6c63:6f77::/64" >> mailcow.conf
+            ;;
+        SQL_PORT)
+            echo '# Bind SQL to 127.0.0.1 on port 13306' >> mailcow.conf
+            echo "SQL_PORT=127.0.0.1:13306" >> mailcow.conf
+            ;;
+        API_KEY)
+            echo '# Read-write service API key, shown read-only in the web UI' >> mailcow.conf
+            echo "#API_KEY=" >> mailcow.conf
+            ;;
+        API_KEY_READ_ONLY)
+            echo '# Read-only service API key, shown read-only in the web UI' >> mailcow.conf
+            echo "#API_KEY_READ_ONLY=" >> mailcow.conf
+            ;;
+        API_ALLOW_FROM)
+            echo '# Must be set for API_KEY and API_KEY_READ_ONLY to be active' >> mailcow.conf
+            echo '# Comma separated list of IPs/networks' >> mailcow.conf
+            echo "#API_ALLOW_FROM=" >> mailcow.conf
+            ;;
+        SNAT_TO_SOURCE)
+            echo '# Use this IPv4 for outgoing connections (SNAT)' >> mailcow.conf
+            echo "#SNAT_TO_SOURCE=" >> mailcow.conf
+            ;;
+        SNAT6_TO_SOURCE)
+            echo '# Use this IPv6 for outgoing connections (SNAT)' >> mailcow.conf
+            echo "#SNAT6_TO_SOURCE=" >> mailcow.conf
+            ;;
+        MAILDIR_GC_TIME)
+            echo '# Garbage collector cleanup' >> mailcow.conf
+            echo '# Deleted domains and mailboxes are moved to /var/vmail/_garbage/timestamp_sanitizedstring' >> mailcow.conf
+            echo '# How long should objects remain in the garbage until they are being deleted? (value in minutes)' >> mailcow.conf
+            echo '# Check interval is hourly' >> mailcow.conf
+            echo 'MAILDIR_GC_TIME=1440' >> mailcow.conf
+            ;;
+        ACL_ANYONE)
+            echo '# Set this to "allow" to enable the anyone pseudo user. Disabled by default.' >> mailcow.conf
+            echo '# When enabled, ACL can be created, that apply to "All authenticated users"' >> mailcow.conf
+            echo '# This should probably only be activated on mail hosts, that are used exclusively by one organisation.' >> mailcow.conf
+            echo '# Otherwise a user might share data with too many other users.' >> mailcow.conf
+            echo 'ACL_ANYONE=disallow' >> mailcow.conf
+            ;;
+        FTS_HEAP)
+            echo '# Dovecot Indexing (FTS) Process maximum heap size in MB, there is no recommendation, please see Dovecot docs.' >> mailcow.conf
+            echo '# Flatcurve is used as FTS Engine. It is supposed to be pretty efficient in CPU and RAM consumption.' >> mailcow.conf
+            echo '# Please always monitor your Resource consumption!' >> mailcow.conf
+            echo "FTS_HEAP=128" >> mailcow.conf
+            ;;
+        SKIP_FTS)
+            echo '# Skip FTS (Fulltext Search) for Dovecot on low-memory, low-threaded systems or if you simply want to disable it.' >> mailcow.conf
+            echo "# Dovecot inside mailcow use Flatcurve as FTS Backend." >> mailcow.conf
+            echo "SKIP_FTS=y" >> mailcow.conf
+            ;;
+        FTS_PROCS)
+            echo '# Controls how many processes the Dovecot indexing process can spawn at max.' >> mailcow.conf
+            echo '# Too many indexing processes can use a lot of CPU and Disk I/O' >> mailcow.conf
+            echo '# Please visit: https://doc.dovecot.org/configuration_manual/service_configuration/#indexer-worker for more informations' >> mailcow.conf
+            echo "FTS_PROCS=1" >> mailcow.conf
+            ;;
+        ENABLE_SSL_SNI)
+            echo '# Create seperate certificates for all domains - y/n' >> mailcow.conf
+            echo '# this will allow adding more than 100 domains, but some email clients will not be able to connect with alternative hostnames' >> mailcow.conf
+            echo '# see https://wiki.dovecot.org/SSL/SNIClientSupport' >> mailcow.conf
+            echo "ENABLE_SSL_SNI=n" >> mailcow.conf
+            ;;
+        SKIP_SOGO)
+            echo '# Skip SOGo: Will disable SOGo integration and therefore webmail, DAV protocols and ActiveSync support (experimental, unsupported, not fully implemented) - y/n' >> mailcow.conf
+            echo "SKIP_SOGO=n" >> mailcow.conf
+            ;;
+        MAILDIR_SUB)
+            echo '# MAILDIR_SUB defines a path in a users virtual home to keep the maildir in. Leave empty for updated setups.' >> mailcow.conf
+            echo "#MAILDIR_SUB=Maildir" >> mailcow.conf
+            echo "MAILDIR_SUB=" >> mailcow.conf
+            ;;
+        WATCHDOG_NOTIFY_WEBHOOK)
+            echo '# Send notifications to a webhook URL that receives a POST request with the content type "application/json".' >> mailcow.conf
+            echo '# You can use this to send notifications to services like Discord, Slack and others.' >> mailcow.conf
+            echo '#WATCHDOG_NOTIFY_WEBHOOK=https://discord.com/api/webhooks/XXXXXXXXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' >> mailcow.conf
+            ;;
+        WATCHDOG_NOTIFY_WEBHOOK_BODY)
+            echo '# JSON body included in the webhook POST request. Needs to be in single quotes.' >> mailcow.conf
+            echo '# Following variables are available: SUBJECT, BODY' >> mailcow.conf
+            WEBHOOK_BODY='{"username": "mailcow Watchdog", "content": "**${SUBJECT}**\n${BODY}"}'
+            echo "#WATCHDOG_NOTIFY_WEBHOOK_BODY='${WEBHOOK_BODY}'" >> mailcow.conf
+            ;;
+        WATCHDOG_NOTIFY_BAN)
+            echo '# Notify about banned IP. Includes whois lookup.' >> mailcow.conf
+            echo "WATCHDOG_NOTIFY_BAN=y" >> mailcow.conf
+            ;;
+        WATCHDOG_NOTIFY_START)
+            echo '# Send a notification when the watchdog is started.' >> mailcow.conf
+            echo "WATCHDOG_NOTIFY_START=y" >> mailcow.conf
+            ;;
+        WATCHDOG_SUBJECT)
+            echo '# Subject for watchdog mails. Defaults to "Watchdog ALERT" followed by the error message.' >> mailcow.conf
+            echo "#WATCHDOG_SUBJECT=" >> mailcow.conf
+            ;;
+        WATCHDOG_EXTERNAL_CHECKS)
+            echo '# Checks if mailcow is an open relay. Requires a SAL. More checks will follow.' >> mailcow.conf
+            echo '# No data is collected. Opt-in and anonymous.' >> mailcow.conf
+            echo '# Will only work with unmodified mailcow setups.' >> mailcow.conf
+            echo "WATCHDOG_EXTERNAL_CHECKS=n" >> mailcow.conf
+            ;;
+        SOGO_EXPIRE_SESSION)
+            echo '# SOGo session timeout in minutes' >> mailcow.conf
+            echo "SOGO_EXPIRE_SESSION=480" >> mailcow.conf
+            ;;
+        REDIS_PORT)
+            echo "REDIS_PORT=127.0.0.1:7654" >> mailcow.conf
+            ;;
+        DOVECOT_MASTER_USER)
+            echo '# DOVECOT_MASTER_USER and _PASS must _both_ be provided. No special chars.' >> mailcow.conf
+            echo '# Empty by default to auto-generate master user and password on start.' >> mailcow.conf
+            echo '# User expands to DOVECOT_MASTER_USER@mailcow.local' >> mailcow.conf
+            echo '# LEAVE EMPTY IF UNSURE' >> mailcow.conf
+            echo "DOVECOT_MASTER_USER=" >> mailcow.conf
+            ;;
+        DOVECOT_MASTER_PASS)
+            echo '# LEAVE EMPTY IF UNSURE' >> mailcow.conf
+            echo "DOVECOT_MASTER_PASS=" >> mailcow.conf
+            ;;
+        MAILCOW_PASS_SCHEME)
+            echo '# Password hash algorithm' >> mailcow.conf
+            echo '# Only certain password hash algorithm are supported. For a fully list of supported schemes,' >> mailcow.conf
+            echo '# see https://docs.mailcow.email/models/model-passwd/' >> mailcow.conf
+            echo "MAILCOW_PASS_SCHEME=BLF-CRYPT" >> mailcow.conf
+            ;;
+        ADDITIONAL_SERVER_NAMES)
+            echo '# Additional server names for mailcow UI' >> mailcow.conf
+            echo '#' >> mailcow.conf
+            echo '# Specify alternative addresses for the mailcow UI to respond to' >> mailcow.conf
+            echo '# This is useful when you set mail.* as ADDITIONAL_SAN and want to make sure mail.maildomain.com will always point to the mailcow UI.' >> mailcow.conf
+            echo '# If the server name does not match a known site, Nginx decides by best-guess and may redirect users to the wrong web root.' >> mailcow.conf
+            echo '# You can understand this as server_name directive in Nginx.' >> mailcow.conf
+            echo '# Comma separated list without spaces! Example: ADDITIONAL_SERVER_NAMES=a.b.c,d.e.f' >> mailcow.conf
+            echo 'ADDITIONAL_SERVER_NAMES=' >> mailcow.conf
+            ;;
+        WEBAUTHN_ONLY_TRUSTED_VENDORS)
+            echo "# WebAuthn device manufacturer verification" >> mailcow.conf
+            echo '# After setting WEBAUTHN_ONLY_TRUSTED_VENDORS=y only devices from trusted manufacturers are allowed' >> mailcow.conf
+            echo '# root certificates can be placed for validation under mailcow-dockerized/data/web/inc/lib/WebAuthn/rootCertificates' >> mailcow.conf
+            echo 'WEBAUTHN_ONLY_TRUSTED_VENDORS=n' >> mailcow.conf
+            ;;
+        SPAMHAUS_DQS_KEY)
+            echo "# Spamhaus Data Query Service Key" >> mailcow.conf
+            echo '# Optional: Leave empty for none' >> mailcow.conf
+            echo '# Enter your key here if you are using a blocked ASN (OVH, AWS, Cloudflare e.g) for the unregistered Spamhaus Blocklist.' >> mailcow.conf
+            echo '# If empty, it will completely disable Spamhaus blocklists if it detects that you are running on a server using a blocked AS.' >> mailcow.conf
+            echo '# Otherwise it will work as usual.' >> mailcow.conf
+            echo 'SPAMHAUS_DQS_KEY=' >> mailcow.conf
+            ;;
+        WATCHDOG_VERBOSE)
+            echo '# Enable watchdog verbose logging' >> mailcow.conf
+            echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
+            ;;
+        SKIP_UNBOUND_HEALTHCHECK)
+            echo '# Skip Unbound (DNS Resolver) Healthchecks (NOT Recommended!) - y/n' >> mailcow.conf
+            echo 'SKIP_UNBOUND_HEALTHCHECK=n' >> mailcow.conf
+            ;;
+        DISABLE_NETFILTER_ISOLATION_RULE)
+            echo '# Prevent netfilter from setting an iptables/nftables rule to isolate the mailcow docker network - y/n' >> mailcow.conf
+            echo '# CAUTION: Disabling this may expose container ports to other neighbors on the same subnet, even if the ports are bound to localhost' >> mailcow.conf
+            echo 'DISABLE_NETFILTER_ISOLATION_RULE=n' >> mailcow.conf
+            ;;
+        HTTP_REDIRECT)
+            echo '# Redirect HTTP connections to HTTPS - y/n' >> mailcow.conf
+            echo 'HTTP_REDIRECT=n' >> mailcow.conf
+            ;;
+        ENABLE_IPV6)
+            echo '# IPv6 Controller Section' >> mailcow.conf
+            echo '# This variable controls the usage of IPv6 within mailcow.' >> mailcow.conf
+            echo '# Can either be true or false | Defaults to true' >> mailcow.conf
+            echo '# WARNING: MAKE SURE TO PROPERLY CONFIGURE IPv6 ON YOUR HOST FIRST BEFORE ENABLING THIS AS FAULTY CONFIGURATIONS CAN LEAD TO OPEN RELAYS!' >> mailcow.conf
+            echo '# A COMPLETE DOCKER STACK REBUILD (compose down && compose up -d) IS NEEDED TO APPLY THIS.' >> mailcow.conf
+            echo ENABLE_IPV6=${IPV6_BOOL} >> mailcow.conf
+            ;;
+        SKIP_CLAMD)
+            echo '# Skip ClamAV (clamd-mailcow) anti-virus (Rspamd will auto-detect a missing ClamAV container) - y/n' >> mailcow.conf
+            echo 'SKIP_CLAMD=n' >> mailcow.conf
+            ;;
+        SKIP_OLEFY)
+            echo '# Skip Olefy (olefy-mailcow) anti-virus for Office documents (Rspamd will auto-detect a missing Olefy container) - y/n' >> mailcow.conf
+            echo 'SKIP_OLEFY=n' >> mailcow.conf
+            ;;
+        REDISPASS)
+            echo "REDISPASS=$(LC_ALL=C </dev/urandom tr -dc A-Za-z0-9 2>/dev/null | head -c 28)" >> mailcow.conf
+            ;;
+        SOGO_URL_ENCRYPTION_KEY)
+            echo '# SOGo URL encryption key (exactly 16 characters, limited to A–Z, a–z, 0–9)' >> mailcow.conf
+            echo '# This key is used to encrypt email addresses within SOGo URLs' >> mailcow.conf
+            echo "SOGO_URL_ENCRYPTION_KEY=$(LC_ALL=C </dev/urandom tr -dc A-Za-z0-9 2>/dev/null | head -c 16)" >> mailcow.conf
+            ;;
+        ACME_DNS_CHALLENGE)
+            echo '# Enable DNS-01 challenge for ACME (acme-mailcow) - y/n' >> mailcow.conf
+            echo '# This requires you to set ACME_DNS_PROVIDER and ACME_ACCOUNT_EMAIL below' >> mailcow.conf
+            echo 'ACME_DNS_CHALLENGE=n' >> mailcow.conf
+            ;;
+        ACME_DNS_PROVIDER)
+            echo '# DNS provider for DNS-01 challenge (e.g. dns_cf, dns_azure, dns_gd, etc.)' >> mailcow.conf
+            echo '# See the dns-01 provider documentation for more information.' >> mailcow.conf
+            echo 'ACME_DNS_PROVIDER=dns_xxx' >> mailcow.conf
+            ;;
+        ACME_ACCOUNT_EMAIL)
+            echo '# Account email for ACME DNS-01 challenge registration' >> mailcow.conf
+            echo 'ACME_ACCOUNT_EMAIL=me@example.com' >> mailcow.conf
+            ;;
+        *)
+            echo "${option}=" >> mailcow.conf
+            ;;
+    esac
+  done
+}

data/Dockerfiles/acme/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.23
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
@@ -14,11 +14,22 @@ RUN apk upgrade --no-cache \
   tini \
   tzdata \
   python3 \
-  acme-tiny
+  acme-tiny \
+  git \
+  socat \
+  && git clone --depth 1 https://github.com/acmesh-official/acme.sh.git /opt/acme.sh \
+  && chmod +x /opt/acme.sh/acme.sh \
+  && mkdir -p /var/lib/acme/acme-sh
+
+ENV ACME_SH_BIN=/opt/acme.sh/acme.sh \
+    ACME_SH_HOME=/opt/acme.sh \
+    ACME_SH_CONFIG_HOME=/var/lib/acme/acme-sh
 
 COPY acme.sh /srv/acme.sh
 COPY functions.sh /srv/functions.sh
 COPY obtain-certificate.sh /srv/obtain-certificate.sh
+COPY obtain-certificate-dns.sh /srv/obtain-certificate-dns.sh
+COPY load-dns-config.sh /srv/load-dns-config.sh
 COPY reload-configurations.sh /srv/reload-configurations.sh
 COPY expand6.sh /srv/expand6.sh
 

data/Dockerfiles/acme/acme.sh

@@ -14,6 +14,17 @@ until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
   sleep 2
 done
 
+# Create DNS-01 configuration template if it doesn't exist
+if [[ ! -f /etc/acme/dns-01.conf ]]; then
+  mkdir -p /etc/acme
+  cat > /etc/acme/dns-01.conf <<'EOF'
+# Add here your DNS-01 challenge configuration
+# For more information, visit the acme.sh documentation:
+# https://github.com/acmesh-official/acme.sh/wiki/dnsapi
+EOF
+  echo "Created DNS-01 configuration template at /etc/acme/dns-01.conf"
+fi
+
 source /srv/functions.sh
 # Thanks to https://github.com/cvmiller -> https://github.com/cvmiller/expand6
 source /srv/expand6.sh
@@ -42,6 +53,10 @@ if [[ "${ENABLE_SSL_SNI}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   ENABLE_SSL_SNI=y
 fi
 
+if [[ "${ACME_DNS_CHALLENGE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  ACME_DNS_CHALLENGE=y
+fi
+
 if [[ "${SKIP_LETS_ENCRYPT}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   log_f "SKIP_LETS_ENCRYPT=y, skipping Let's Encrypt..."
   sleep 365d
@@ -159,18 +174,6 @@ while true; do
   fi
   if [[ ! -f ${ACME_BASE}/acme/account.pem ]]; then
     log_f "Generating missing Lets Encrypt account key..."
-    if [[ ! -z ${ACME_CONTACT} ]]; then
-      if ! verify_email "${ACME_CONTACT}"; then
-        log_f "Invalid email address, will not start registration!"
-        sleep 365d
-        exec $(readlink -f "$0")
-      else
-        ACME_CONTACT_PARAMETER="--contact mailto:${ACME_CONTACT}"
-        log_f "Valid email address, using ${ACME_CONTACT} for registration"
-      fi
-    else
-      ACME_CONTACT_PARAMETER=""
-    fi
     openssl genrsa 4096 > ${ACME_BASE}/acme/account.pem
   else
     log_f "Using existing Lets Encrypt account key ${ACME_BASE}/acme/account.pem"
@@ -218,7 +221,7 @@ while true; do
 
   if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
   # Fetch certs for autoconfig and autodiscover subdomains
-  ADDITIONAL_WC_ARR+=('autodiscover' 'autoconfig')
+  ADDITIONAL_WC_ARR+=('autodiscover' 'autoconfig' 'mta-sts')
   fi
 
   if [[ ${SKIP_IP_CHECK} != "y" ]]; then
@@ -246,18 +249,71 @@ while true; do
   done <<< "${SQL_DOMAINS}"
 
   if [[ ${ONLY_MAILCOW_HOSTNAME} != "y" ]]; then
+  # Fetch all domains with an active MTA-STS policy once.
+  unset MTA_STS_ACTIVE_DOMAINS
+  declare -A MTA_STS_ACTIVE_DOMAINS
+  if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
+    SQL_MTA_STS_DOMAINS=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT domain FROM mta_sts WHERE active = 1" -Bs)
+    if [[ $? -eq 0 ]]; then
+      while read mta_sts_domain; do
+        if [[ -z "${mta_sts_domain}" ]]; then
+          # ignore empty lines
+          continue
+        fi
+        MTA_STS_ACTIVE_DOMAINS["${mta_sts_domain}"]=1
+      done <<< "${SQL_MTA_STS_DOMAINS}"
+    fi
+  fi
   for SQL_DOMAIN in "${SQL_DOMAIN_ARR[@]}"; do
     unset VALIDATED_CONFIG_DOMAINS_SUBDOMAINS
     declare -a VALIDATED_CONFIG_DOMAINS_SUBDOMAINS
     for SUBDOMAIN in "${ADDITIONAL_WC_ARR[@]}"; do
-      if [[  "${SUBDOMAIN}.${SQL_DOMAIN}" != "${MAILCOW_HOSTNAME}" ]]; then
-        if check_domain "${SUBDOMAIN}.${SQL_DOMAIN}"; then
-          VALIDATED_CONFIG_DOMAINS_SUBDOMAINS+=("${SUBDOMAIN}.${SQL_DOMAIN}")
-        fi
+      FULL_SUBDOMAIN="${SUBDOMAIN}.${SQL_DOMAIN}"
+
+      # Skip mta-sts subdomain unless MTA-STS is enabled (active) for this domain
+      if [[ "${SUBDOMAIN}" == "mta-sts" && -z "${MTA_STS_ACTIVE_DOMAINS[${SQL_DOMAIN}]}" ]]; then
+        log_f "MTA-STS is not enabled for ${SQL_DOMAIN} - skipping mta-sts subdomain certificate"
+        continue
+      fi
+
+      # Skip if subdomain matches MAILCOW_HOSTNAME
+      if [[ "${FULL_SUBDOMAIN}" == "${MAILCOW_HOSTNAME}" ]]; then
+        continue
+      fi
+      # Skip if subdomain is covered by a wildcard in ADDITIONAL_SAN
+      if is_covered_by_wildcard "${FULL_SUBDOMAIN}"; then
+        log_f "Subdomain '${FULL_SUBDOMAIN}' is covered by wildcard - skipping explicit subdomain"
+        continue
+      fi
+      # Validate and add subdomain
+      if check_domain "${FULL_SUBDOMAIN}"; then
+        VALIDATED_CONFIG_DOMAINS_SUBDOMAINS+=("${FULL_SUBDOMAIN}")
       fi
     done
     VALIDATED_CONFIG_DOMAINS+=("${VALIDATED_CONFIG_DOMAINS_SUBDOMAINS[*]}")
   done
+
+  # Fetch alias domains where target domain has MTA-STS enabled
+  if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
+    SQL_ALIAS_DOMAINS=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT ad.alias_domain FROM alias_domain ad INNER JOIN mta_sts m ON ad.target_domain = m.domain WHERE ad.active = 1 AND m.active = 1" -Bs)
+    if [[ $? -eq 0 ]]; then
+      while read alias_domain; do
+        if [[ -z "${alias_domain}" ]]; then
+          # ignore empty lines
+          continue
+        fi
+        # Only add mta-sts subdomain for alias domains
+        if [[ "mta-sts.${alias_domain}" != "${MAILCOW_HOSTNAME}" ]]; then
+          # Skip if mta-sts subdomain is covered by a wildcard
+          if is_covered_by_wildcard "mta-sts.${alias_domain}"; then
+            log_f "Alias domain mta-sts subdomain 'mta-sts.${alias_domain}' is covered by wildcard - skipping"
+          elif check_domain "mta-sts.${alias_domain}"; then
+            VALIDATED_CONFIG_DOMAINS+=("mta-sts.${alias_domain}")
+          fi
+        fi
+      done <<< "${SQL_ALIAS_DOMAINS}"
+    fi
+  fi
   fi
 
   if check_domain ${MAILCOW_HOSTNAME}; then
@@ -286,20 +342,38 @@ while true; do
   done
   fi
 
+  # Check if MAILCOW_HOSTNAME is covered by a wildcard in ADDITIONAL_SAN
+  MAILCOW_HOSTNAME_COVERED=0
+  if [[ ! -z ${VALIDATED_MAILCOW_HOSTNAME} ]]; then
+    if is_covered_by_wildcard "${VALIDATED_MAILCOW_HOSTNAME}"; then
+      MAILCOW_PARENT_DOMAIN=$(echo ${VALIDATED_MAILCOW_HOSTNAME} | cut -d. -f2-)
+      log_f "MAILCOW_HOSTNAME '${VALIDATED_MAILCOW_HOSTNAME}' is covered by wildcard '*.${MAILCOW_PARENT_DOMAIN}' - skipping explicit hostname"
+      MAILCOW_HOSTNAME_COVERED=1
+    fi
+  fi
+
   # Unique domains for server certificate
   if [[ ${ENABLE_SSL_SNI} == "y" ]]; then
     # create certificate for server name and fqdn SANs only
-    SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    if [[ ${MAILCOW_HOSTNAME_COVERED} == "1" ]]; then
+      SERVER_SAN_VALIDATED=($(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    else
+      SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    fi
   else
     # create certificate for all domains, including all subdomains from other domains [*]
-    SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    if [[ ${MAILCOW_HOSTNAME_COVERED} == "1" ]]; then
+      SERVER_SAN_VALIDATED=($(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    else
+      SERVER_SAN_VALIDATED=(${VALIDATED_MAILCOW_HOSTNAME} $(echo ${VALIDATED_CONFIG_DOMAINS[*]} ${ADDITIONAL_VALIDATED_SAN[*]} | xargs -n1 | sort -u | xargs))
+    fi
   fi
   if [[ ! -z ${SERVER_SAN_VALIDATED[*]} ]]; then
     CERT_NAME=${SERVER_SAN_VALIDATED[0]}
     VALIDATED_CERTIFICATES+=("${CERT_NAME}")
 
     # obtain server certificate if required
-    ACME_CONTACT_PARAMETER=${ACME_CONTACT_PARAMETER} DOMAINS=${SERVER_SAN_VALIDATED[@]} /srv/obtain-certificate.sh rsa
+    DOMAINS=${SERVER_SAN_VALIDATED[@]} /srv/obtain-certificate.sh rsa
     RETURN="$?"
     if [[ "$RETURN" == "0" ]]; then # 0 = cert created successfully
       CERT_AMOUNT_CHANGED=1

data/Dockerfiles/acme/functions.sh

@@ -80,6 +80,11 @@ check_domain(){
         return 1
       fi
     fi
+
+    if [[ ${ACME_DNS_CHALLENGE} == "y" ]]; then
+      log_f "ACME_DNS_CHALLENGE=y - skipping IP and HTTP validation for ${DOMAIN}"
+      return 0
+    fi
     # Check if CNAME without v6 enabled target
     if [[ ! -z ${AAAA_DOMAIN} ]] && [[ -z $(echo ${AAAA_DOMAIN} | grep "^\([0-9a-fA-F]\{0,4\}:\)\{1,7\}[0-9a-fA-F]\{0,4\}$") ]]; then
       AAAA_DOMAIN=
@@ -130,3 +135,32 @@ verify_challenge_path(){
     return 1
   fi
 }
+
+# Check if a domain is covered by a wildcard (*.example.com) in ADDITIONAL_SAN
+# Usage: is_covered_by_wildcard "subdomain.example.com"
+# Returns: 0 if covered, 1 if not covered
+# Note: Only returns 0 (covered) when DNS-01 challenge is enabled,
+#       as wildcards cannot be validated with HTTP-01 challenge
+is_covered_by_wildcard() {
+  local DOMAIN=$1
+
+  # Only skip if DNS challenge is enabled (wildcards require DNS-01)
+  if [[ ${ACME_DNS_CHALLENGE} != "y" ]]; then
+    return 1
+  fi
+
+  # Return early if no ADDITIONAL_SAN is set
+  if [[ -z ${ADDITIONAL_SAN} ]]; then
+    return 1
+  fi
+
+  # Extract parent domain (e.g., mail.example.com -> example.com)
+  local PARENT_DOMAIN=$(echo ${DOMAIN} | cut -d. -f2-)
+
+  # Check if ADDITIONAL_SAN contains a wildcard for this parent domain
+  if [[ "${ADDITIONAL_SAN}" == *"*.${PARENT_DOMAIN}"* ]]; then
+    return 0  # Covered by wildcard
+  fi
+
+  return 1  # Not covered
+}

data/Dockerfiles/acme/load-dns-config.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+SCRIPT_SOURCE="${BASH_SOURCE[0]:-${0}}"
+if [[ "${SCRIPT_SOURCE}" == "${0}" ]]; then
+  __dns_loader_standalone=1
+else
+  __dns_loader_standalone=0
+fi
+
+CONFIG_PATH="${ACME_DNS_CONFIG_FILE:-/etc/acme/dns-01.conf}"
+
+if [[ ! -f "${CONFIG_PATH}" ]]; then
+  if [[ $__dns_loader_standalone -eq 1 ]]; then
+    exit 0
+  else
+    return 0
+  fi
+fi
+
+source /srv/functions.sh
+
+log_f "Loading DNS-01 configuration from ${CONFIG_PATH}"
+
+LINE_NO=0
+while IFS= read -r line || [[ -n "${line}" ]]; do
+  LINE_NO=$((LINE_NO+1))
+  line="${line%$'\r'}"
+  line_trimmed="$(printf '%s' "${line}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  [[ -z "${line_trimmed}" ]] && continue
+  [[ "${line_trimmed:0:1}" == "#" ]] && continue
+  if [[ "${line_trimmed}" != *=* ]]; then
+    log_f "Skipping invalid DNS config line ${LINE_NO} (missing key=value)"
+    continue
+  fi
+  KEY="${line_trimmed%%=*}"
+  VALUE="${line_trimmed#*=}"
+  KEY="$(printf '%s' "${KEY}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  VALUE="$(printf '%s' "${VALUE}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+  if [[ -z "${KEY}" ]]; then
+    log_f "Skipping invalid DNS config line ${LINE_NO} (empty key)"
+    continue
+  fi
+  if [[ "${VALUE}" =~ ^\".*\"$ ]]; then
+    VALUE="${VALUE:1:-1}"
+  elif [[ "${VALUE}" =~ ^\'.*\'$ ]]; then
+    VALUE="${VALUE:1:-1}"
+  fi
+  export "${KEY}"="${VALUE}"
+  log_f "Exported DNS config key ${KEY}"
+
+done < "${CONFIG_PATH}"
+
+if [[ $__dns_loader_standalone -eq 1 ]]; then
+  exit 0
+else
+  return 0
+fi

data/Dockerfiles/acme/obtain-certificate-dns.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+
+# Return values / exit codes
+# 0 = cert created successfully
+# 1 = cert renewed successfully
+# 2 = cert not due for renewal
+# * = errors
+
+source /srv/functions.sh
+
+CERT_DOMAINS=(${DOMAINS[@]})
+CERT_DOMAIN=${CERT_DOMAINS[0]}
+ACME_BASE=/var/lib/acme
+
+# Load optional DNS provider secrets from /etc/acme/dns-01.conf
+if [[ -f /srv/load-dns-config.sh ]]; then
+  source /srv/load-dns-config.sh
+  if declare -F log_f >/dev/null; then
+    log_f "ACME_DNS_CHALLENGE is enabled, DNS provider secrets loaded"
+  fi
+fi
+
+TYPE=${1}
+PREFIX=""
+# only support rsa certificates for now
+if [[ "${TYPE}" != "rsa" ]]; then
+  log_f "Unknown certificate type '${TYPE}' requested"
+  exit 5
+fi
+
+if [[ -z "${ACME_DNS_PROVIDER}" ]]; then
+  log_f "ACME_DNS_PROVIDER is required when ACME_DNS_CHALLENGE is enabled"
+  exit 6
+fi
+
+DOMAINS_FILE=${ACME_BASE}/${CERT_DOMAIN}/domains
+CERT=${ACME_BASE}/${CERT_DOMAIN}/${PREFIX}cert.pem
+SHARED_KEY=${ACME_BASE}/acme/${PREFIX}key.pem  # must already exist
+KEY=${ACME_BASE}/${CERT_DOMAIN}/${PREFIX}key.pem
+CSR=${ACME_BASE}/${CERT_DOMAIN}/${PREFIX}acme.csr
+
+if [[ -z ${CERT_DOMAINS[*]} ]]; then
+  log_f "Missing CERT_DOMAINS to obtain a certificate"
+  exit 3
+fi
+
+if [[ "${LE_STAGING}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  if [[ ! -z "${DIRECTORY_URL}" ]]; then
+    log_f "Cannot use DIRECTORY_URL with LE_STAGING=y - ignoring DIRECTORY_URL"
+  fi
+  log_f "Using Let's Encrypt staging servers"
+  ACME_SH_SERVER_ARGS=("--staging")
+elif [[ ! -z "${DIRECTORY_URL}" ]]; then
+  log_f "Using custom directory URL ${DIRECTORY_URL}"
+  ACME_SH_SERVER_ARGS=("--server" "${DIRECTORY_URL}")
+else
+  log_f "Using Let's Encrypt production servers"
+  ACME_SH_SERVER_ARGS=("--server" "letsencrypt")
+fi
+
+if [[ -f ${DOMAINS_FILE} && "$(cat ${DOMAINS_FILE})" ==  "${CERT_DOMAINS[*]}" ]]; then
+  if [[ ! -f ${CERT} || ! -f "${KEY}" || -f "${ACME_BASE}/force_renew" ]]; then
+    log_f "Certificate ${CERT} doesn't exist yet or forced renewal - start obtaining"
+  elif ! openssl x509 -checkend 2592000 -noout -in ${CERT} > /dev/null; then
+    log_f "Certificate ${CERT} is due for renewal (< 30 days) - start renewing"
+  else
+    log_f "Certificate ${CERT} validation done, neither changed nor due for renewal."
+    exit 2
+  fi
+else
+  log_f "Certificate ${CERT} missing or changed domains '${CERT_DOMAINS[*]}' - start obtaining"
+fi
+
+# Make backup
+if [[ -f ${CERT} ]]; then
+  DATE=$(date +%Y-%m-%d_%H_%M_%S)
+  BACKUP_DIR=${ACME_BASE}/backups/${CERT_DOMAIN}/${PREFIX}${DATE}
+  log_f "Creating backups in ${BACKUP_DIR} ..."
+  mkdir -p ${BACKUP_DIR}/
+  [[ -f ${DOMAINS_FILE} ]] && cp ${DOMAINS_FILE} ${BACKUP_DIR}/
+  [[ -f ${CERT} ]] && cp ${CERT} ${BACKUP_DIR}/
+  [[ -f ${KEY} ]] && cp ${KEY} ${BACKUP_DIR}/
+  [[ -f ${CSR} ]] && cp ${CSR} ${BACKUP_DIR}/
+fi
+
+mkdir -p ${ACME_BASE}/${CERT_DOMAIN}
+if [[ ! -f ${KEY} ]]; then
+  log_f "Copying shared private key for this certificate..."
+  cp ${SHARED_KEY} ${KEY}
+  chmod 600 ${KEY}
+fi
+
+# Generating CSR to keep layout parity with HTTP challenge flow
+printf "[SAN]\nsubjectAltName=" > /tmp/_SAN
+printf "DNS:%s," "${CERT_DOMAINS[@]}" >> /tmp/_SAN
+sed -i '$s/,$//' /tmp/_SAN
+openssl req -new -sha256 -key ${KEY} -subj "/" -reqexts SAN -config <(cat "$(openssl version -d | sed 's/.*\"\(.*\)\"/\1/g')/openssl.cnf" /tmp/_SAN) > ${CSR}
+
+log_f "Checking resolver..."
+until dig letsencrypt.org +time=3 +tries=1 @unbound > /dev/null; do
+  sleep 2
+done
+log_f "Resolver OK"
+
+ACME_SH_BIN_PATH=${ACME_SH_BIN:-/opt/acme.sh/acme.sh}
+ACME_SH_WORK_HOME=${ACME_SH_CONFIG_HOME:-/var/lib/acme/acme-sh}
+mkdir -p ${ACME_SH_WORK_HOME}
+
+if [[ ! -x ${ACME_SH_BIN_PATH} ]]; then
+  log_f "acme.sh binary not found at ${ACME_SH_BIN_PATH}"
+  exit 7
+fi
+
+if [[ ! -f ${ACME_SH_WORK_HOME}/account.conf ]]; then
+  if [[ -z "${ACME_ACCOUNT_EMAIL}" ]]; then
+    log_f "ACME_ACCOUNT_EMAIL is required to register a new acme.sh account"
+    exit 8
+  fi
+  log_f "Registering acme.sh account for ${ACME_ACCOUNT_EMAIL}"
+  REGISTER_CMD=("${ACME_SH_BIN_PATH}" "--home" "${ACME_SH_WORK_HOME}" "--config-home" "${ACME_SH_WORK_HOME}" "--cert-home" "${ACME_SH_WORK_HOME}" "--register-account" "-m" "${ACME_ACCOUNT_EMAIL}")
+  REGISTER_CMD+=("${ACME_SH_SERVER_ARGS[@]}")
+  REGISTER_RESPONSE=$("${REGISTER_CMD[@]}" 2>&1)
+  if [[ $? -ne 0 ]]; then
+    log_f "Failed to register acme.sh account: ${REGISTER_RESPONSE}"
+    exit 9
+  fi
+fi
+
+TMP_CERT=$(mktemp /tmp/acme-cert.XXXXXX)
+TMP_FULLCHAIN=$(mktemp /tmp/acme-fullchain.XXXXXX)
+
+ACME_CMD=("${ACME_SH_BIN_PATH}" "--home" "${ACME_SH_WORK_HOME}" "--config-home" "${ACME_SH_WORK_HOME}" "--cert-home" "${ACME_SH_WORK_HOME}")
+ACME_CMD+=("${ACME_SH_SERVER_ARGS[@]}")
+ACME_CMD+=("--issue" "--dns" "${ACME_DNS_PROVIDER}" "--key-file" "${KEY}" "--cert-file" "${TMP_CERT}" "--fullchain-file" "${TMP_FULLCHAIN}" "--force")
+for domain in "${CERT_DOMAINS[@]}"; do
+  ACME_CMD+=("-d" "${domain}")
+done
+
+log_f "Using command ${ACME_CMD[*]}"
+if [[ -n "${ACME_DNS_PROVIDER}" ]]; then
+  log_f "DNS provider: ${ACME_DNS_PROVIDER}"
+fi
+if compgen -A variable | grep -Eq "^DNS_|^ACME_"; then
+  LOG_KEYS=$(env | grep -E "^(DNS_|ACME_)" | cut -d= -f1 | tr '\n' ' ')
+  log_f "Available DNS/ACME env keys: ${LOG_KEYS}" redis_only
+fi
+ACME_RESPONSE=$("${ACME_CMD[@]}" 2>&1 | tee /dev/fd/5; exit ${PIPESTATUS[0]})
+SUCCESS="$?"
+ACME_RESPONSE_B64=$(echo "${ACME_RESPONSE}" | openssl enc -e -A -base64)
+log_f "${ACME_RESPONSE_B64}" redis_only b64
+
+case "$SUCCESS" in
+  0)
+    log_f "Deploying certificate ${CERT}..."
+    if verify_hash_match ${TMP_FULLCHAIN} ${KEY}; then
+      RETURN=0
+      if [[ -f ${CERT} ]]; then
+        RETURN=1
+      fi
+      mv -f ${TMP_FULLCHAIN} ${CERT}
+      rm -f ${TMP_CERT}
+      echo -n ${CERT_DOMAINS[*]} > ${DOMAINS_FILE}
+      log_f "Certificate successfully obtained via DNS challenge"
+      exit ${RETURN}
+    else
+      log_f "Certificate was requested, but key and certificate hashes do not match"
+      rm -f ${TMP_CERT} ${TMP_FULLCHAIN}
+      exit 4
+    fi
+    ;;
+  *)
+    log_f "Failed to obtain certificate ${CERT} for domains '${CERT_DOMAINS[*]}' via DNS challenge"
+    redis-cli -h redis -a ${REDISPASS} --no-auth-warning SET ACME_FAIL_TIME "$(date +%s)"
+    rm -f ${TMP_CERT} ${TMP_FULLCHAIN}
+    exit 100${SUCCESS}
+    ;;
+esac

data/Dockerfiles/acme/obtain-certificate.sh

@@ -20,6 +20,10 @@ if [[ "${TYPE}" != "rsa" ]]; then
   log_f "Unknown certificate type '${TYPE}' requested"
   exit 5
 fi
+
+if [[ "${ACME_DNS_CHALLENGE}" == "y" ]]; then
+  exec /srv/obtain-certificate-dns.sh "$@"
+fi
 DOMAINS_FILE=${ACME_BASE}/${CERT_DOMAIN}/domains
 CERT=${ACME_BASE}/${CERT_DOMAIN}/${PREFIX}cert.pem
 SHARED_KEY=${ACME_BASE}/acme/${PREFIX}key.pem  # must already exist
@@ -93,8 +97,8 @@ until dig letsencrypt.org +time=3 +tries=1 @unbound > /dev/null; do
   sleep 2
 done
 log_f "Resolver OK"
-log_f "Using command acme-tiny ${DIRECTORY_URL} ${ACME_CONTACT_PARAMETER} --account-key ${ACME_BASE}/acme/account.pem --disable-check --csr ${CSR} --acme-dir /var/www/acme/"
-ACME_RESPONSE=$(acme-tiny ${DIRECTORY_URL} ${ACME_CONTACT_PARAMETER} \
+log_f "Using command acme-tiny ${DIRECTORY_URL} --account-key ${ACME_BASE}/acme/account.pem --disable-check --csr ${CSR} --acme-dir /var/www/acme/"
+ACME_RESPONSE=$(acme-tiny ${DIRECTORY_URL} \
   --account-key ${ACME_BASE}/acme/account.pem \
   --disable-check \
   --csr ${CSR} \

data/Dockerfiles/backup/Dockerfile

@@ -1,3 +1,3 @@
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
-RUN apt update && apt install pigz -y --no-install-recommends
+RUN apt update && apt install pigz zstd -y --no-install-recommends

data/Dockerfiles/clamd/Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.21 AS builder
+FROM alpine:3.24 AS builder
 
 WORKDIR /src
-ENV CLAMD_VERSION=1.4.2
+ENV CLAMD_VERSION=1.4.6
 
 RUN apk upgrade --no-cache \
   && apk add --update --no-cache \
@@ -68,7 +68,7 @@ RUN wget -P /src https://www.clamav.net/downloads/production/clamav-${CLAMD_VERS
   "/clamav/etc/clamav/clamav-milter.conf.sample" > "/clamav/etc/clamav/clamav-milter.conf" || exit 1
 
 
-FROM alpine:3.21
+FROM alpine:3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/clamd/clamd.sh

@@ -8,7 +8,7 @@ fi
 
 # Cleaning up garbage
 echo "Cleaning up tmp files..."
-rm -rf /var/lib/clamav/clamav-*.tmp
+rm -rf /var/lib/clamav/tmp.*
 
 # Prepare whitelist
 

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.23
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 

data/Dockerfiles/dockerapi/main.py

@@ -110,12 +110,12 @@ async def get_container(container_id : str):
     return Response(content=json.dumps(res, indent=4), media_type="application/json")
 
 @app.get("/containers/json")
-async def get_containers():
+async def get_containers(all: bool = False):
   global dockerapi
 
   containers = {}
   try:
-    for container in (await dockerapi.async_docker_client.containers.list()):
+    for container in (await dockerapi.async_docker_client.containers.list(all=all)):
       container_info = await container.show()
       containers.update({container_info['Id']: container_info})
     return Response(content=json.dumps(containers, indent=4), media_type="application/json")

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -412,6 +412,63 @@ class DockerApi:
 
         sogo_return = container.exec_run(["/bin/bash", "-c", f"sogo-tool rename-user '{old_username}' '{new_username}'"], user='sogo')
         return self.exec_run_handler('generic', sogo_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: subscribe
+  # Subscribes users to a folder of the owner, or unsubscribes them; values are passed as environment and
+  # arguments, never through the shell
+  def container_post__exec__sogo__subscribe(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      folder = str(request_json.get('folder', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or not re.match(r'^(Calendar|Contacts)/[A-Za-z0-9._-]+$', folder):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner, folder and users required' }, indent=4), media_type="application/json")
+      script = 'for user in "$@"; do sogo-tool manage-acl "$SOGO_ACTION" "$SOGO_OWNER" "$SOGO_FOLDER" "$user" || exit 1; done'
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe"] + users, user='sogo',
+        environment={ 'SOGO_ACTION': 'unsubscribe' if request_json.get('unsubscribe') else 'subscribe', 'SOGO_OWNER': owner, 'SOGO_FOLDER': folder })
+      return self.exec_run_handler('generic', subscribe_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: init_user
+  # Creates the personal folders of users that never logged in to SOGo (e.g. shared mailboxes)
+  def container_post__exec__sogo__init_user(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not users:
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
+      init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
+      return self.exec_run_handler('generic', init_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
+  # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
+  def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      owner = str(request_json.get('owner', ''))
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      if not owner or not users or any(c in owner for c in '/*%'):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'owner and users required' }, indent=4), media_type="application/json")
+      if request_json.get('unsubscribe'):
+        # Without access the user no longer sees the folders, their names come from the owner
+        script = ('set -o pipefail; for user in "$@"; do doveadm mailbox list -u "$SUB_OWNER" | while IFS= read -r folder; do '
+          'case "$folder" in Shared|Shared/*) continue;; esac; doveadm mailbox unsubscribe -u "$user" "Shared/$SUB_OWNER/$folder" || exit 1; '
+          'done || exit 1; done')
+      else:
+        # No visible folder means no access yet (ACL, auth cache): fail, so the subscription is retried
+        script = ('for user in "$@"; do folders=$(doveadm mailbox list -u "$user" "Shared/$SUB_OWNER/*") && [ -n "$folders" ] || exit 1; '
+          'while IFS= read -r folder; do doveadm mailbox subscribe -u "$user" "$folder" || exit 1; done <<< "$folders"; done')
+      subscribe_return = container.exec_run(["/bin/bash", "-c", script, "subscribe_shared"] + users, environment={ 'SUB_OWNER': owner })
+      return self.exec_run_handler('generic', subscribe_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: get_acl
   def container_post__exec__doveadm__get_acl(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -439,7 +496,8 @@ class DockerApi:
           if len(acls) >= 2:
             for acl in acls[1:]:
               user_id, rights = acl.split(maxsplit=1)
-              user_id = user_id.split('=')[1]
+              # user=<name> as plain name, other identifiers (group=...) keep their type
+              user_id = user_id.split('=', 1)[1] if user_id.startswith('user=') else user_id
               mailbox_seen.append(mailbox)
               formatted_acls.append({ 'user': id, 'id': user_id, 'mailbox': mailbox, 'rights': rights.split() })
         elif "Shared" in shared_folder and "/" in shared_folder:
@@ -476,7 +534,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' 'user={id}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -510,8 +568,50 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' 'user={id}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
+  # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
+  # environment, never through the shell
+  def container_post__exec__doveadm__set_acl_all(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      user = request_json.get('user', '')
+      id = self.acl_identifier(request_json.get('id', ''))
+      rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
+      if not user or id in ('user=', 'group='):
+        return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
+      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
+      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
+        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
+      acl_return = container.exec_run(["/bin/bash", "-c", script],
+        environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
+      return self.exec_run_handler('generic', acl_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: auth_cache_flush
+  # Drops cached userdb results (e.g. acl_groups) of the given users, all users if none are given
+  def container_post__exec__doveadm__auth_cache_flush(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      users = [str(user) for user in request_json.get('users', []) if str(user)]
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
+      return self.exec_run_handler('generic', flush_return)
+
+  # Rights doveadm acl set accepts from the API
+  acl_rights = ["admin", "create", "delete", "expunge", "insert", "lookup", "post", "read", "write", "write-deleted", "write-seen"]
+
+  # ACL identifier for doveadm: "user=<name>" for a plain name, "user=..." / "group=..." as given
+  def acl_identifier(self, id):
+    if id.startswith('user=') or id.startswith('group='):
+      return id
+    return 'user=' + id
 
 
   # Collect host stats

data/Dockerfiles/dovecot/Dockerfile

@@ -3,7 +3,7 @@ FROM alpine:3.21
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.16
+ARG GOSU_VERSION=1.19
 
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
@@ -68,7 +68,6 @@ RUN addgroup -g 5000 vmail \
   perl-package-stash-xs \
   perl-par-packer \
   perl-parse-recdescent \
-  perl-lockfile-simple \
   libproc2 \
   perl-readonly \
   perl-regexp-common \
@@ -120,7 +119,7 @@ COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
-COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
+COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -127,7 +127,8 @@ cat <<EOF > /etc/dovecot/sql/dovecot-dict-sql-userdb.conf
 # Autogenerated by mailcow
 driver = mysql
 connect = "host=/var/run/mysqld/mysqld.sock dbname=${DBNAME} user=${DBUSER} password=${DBPASS}"
-user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
+# acl_groups: active mailbox groups of the user, IMAP grants are ACLs on group=mcg_<id>
+user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format')), mailbox_path_prefix, '%d/%n/${MAILDIR_SUB}:VOLATILEDIR=/var/volatile/%u:INDEX=/var/vmail_index/%u') AS mail, '%s' AS protocol, 5000 AS uid, 5000 AS gid, concat('*:bytes=', quota) AS quota_rule, (SELECT GROUP_CONCAT(CONCAT('mcg_', gm.group_id)) FROM mailbox_group_member gm JOIN mailbox_group g ON g.id = gm.group_id AND g.active = 1 WHERE gm.username = mailbox.username) AS acl_groups FROM mailbox WHERE username = '%u' AND (active = '1' OR active = '2')
 iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
 EOF
 
@@ -204,16 +205,17 @@ EOF
 # Create random master Password for SOGo SSO
 RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
 echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
-# Creating additional creds file for SOGo notify crons (calendars, etc)
-echo -n ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/cron.creds
 cat <<EOF > /etc/dovecot/sogo-sso.conf
 # Autogenerated by mailcow
 passdb {
   driver = static
-  args = allow_real_nets=${IPV4_NETWORK}.248/32 password={plain}${RAND_PASS}
+  args = allow_nets=${IPV4_NETWORK}.248/32 password={plain}${RAND_PASS}
 }
 EOF
 
+# Creating additional creds file for SOGo notify crons (calendars, etc) (dummy user, sso password)
+echo -n ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/cron.creds
+
 if [[ "${MASTER}" =~ ^([nN][oO]|[nN])+$ ]]; then
   # Toggling MASTER will result in a rebuild of containers, so the quota script will be recreated
   cat <<'EOF' > /usr/local/bin/quota_notify.py
@@ -278,7 +280,7 @@ chmod g+rw /dev/console
 chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
-  /usr/local/bin/imapsync_runner.pl \
+  /usr/local/bin/imapsync_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -303,6 +305,15 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 # Envsubst maildir_gc
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+done
+echo "DB schema is ${DBV_NOW}"
+
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do
   if [[ ! -z $(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -B -e "SELECT 'OK' FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = \"${DBNAME}\" AND TABLE_NAME = 'versions'") ]]; then

data/Dockerfiles/dovecot/imapsync_runner.pl

@@ -1,196 +0,0 @@
-#!/usr/bin/perl
-
-use DBI;
-use LockFile::Simple qw(lock trylock unlock);
-use Proc::ProcessTable;
-use Data::Dumper qw(Dumper);
-use IPC::Run 'run';
-use File::Temp;
-use Try::Tiny;
-use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT);
-
-sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
-my $t = Proc::ProcessTable->new;
-my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table};
-if ($imapsync_running ge 1)
-{
-  print "imapsync is active, exiting...";
-  exit;
-}
-
-sub qqw($) {
-  my @params = ();
-  my @values = split(/(?=--)/, $_[0]);
-  foreach my $val (@values) {
-    my @tmpparam = split(/ /, $val, 2);
-    foreach my $tmpval (@tmpparam) {
-        if ($tmpval ne '') {
-          push @params, $tmpval;
-        }
-    }
-  }
-  foreach my $val (@params) {
-    $val=trim($val);
-  }
-  return @params;
-}
-
-$run_dir="/tmp";
-$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
-$lock_file = $run_dir . "/imapsync_busy";
-$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
-$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
-  mysql_auto_reconnect => 1,
-  mysql_enable_utf8mb4 => 1
-});
-$dbh->do("UPDATE imapsync SET is_running = 0");
-
-sub sig_handler {
-  # Send die to force exception in "run"
-  die "sig_handler received signal, preparing to exit...\n";
-};
-
-open my $file, '<', "/etc/sogo/sieve.creds"; 
-my $creds = <$file>; 
-close $file;
-my ($master_user, $master_pass) = split /:/, $creds;
-my $sth = $dbh->prepare("SELECT id,
-  user1,
-  user2,
-  host1,
-  authmech1,
-  password1,
-  exclude,
-  port1,
-  enc1,
-  delete2duplicates,
-  maxage,
-  subfolder2,
-  delete1,
-  delete2,
-  automap,
-  skipcrossduplicates,
-  maxbytespersecond,
-  custom_params,
-  subscribeall,
-  timeout1,
-  timeout2,
-  dry
-    FROM imapsync
-      WHERE active = 1
-        AND is_running = 0
-        AND (
-          UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
-          OR
-          last_run IS NULL)
-  ORDER BY last_run");
-
-$sth->execute();
-my $row;
-
-while ($row = $sth->fetchrow_arrayref()) {
-
-  $id                  = @$row[0];
-  $user1               = @$row[1];
-  $user2               = @$row[2];
-  $host1               = @$row[3];
-  $authmech1           = @$row[4];
-  $password1           = @$row[5];
-  $exclude             = @$row[6];
-  $port1               = @$row[7];
-  $enc1                = @$row[8];
-  $delete2duplicates   = @$row[9];
-  $maxage              = @$row[10];
-  $subfolder2          = @$row[11];
-  $delete1             = @$row[12];
-  $delete2             = @$row[13];
-  $automap             = @$row[14];
-  $skipcrossduplicates = @$row[15];
-  $maxbytespersecond   = @$row[16];
-  $custom_params       = @$row[17];
-  $subscribeall        = @$row[18];
-  $timeout1            = @$row[19];
-  $timeout2            = @$row[20];
-  $dry                 = @$row[21];
-
-  if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
-
-  my $template = $run_dir . '/imapsync.XXXXXXX';
-  my $passfile1 = File::Temp->new(TEMPLATE => $template);
-  my $passfile2 = File::Temp->new(TEMPLATE => $template);
-  
-  binmode( $passfile1, ":utf8" );
-  
-  print $passfile1 "$password1\n";
-  print $passfile2 trim($master_pass) . "\n";
-
-  my @custom_params_a = qqw($custom_params);
-  my $custom_params_ref = \@custom_params_a;
-
-  my $generated_cmds = [ "/usr/local/bin/imapsync",
-  "--tmpdir", "/tmp",
-  "--nofoldersizes",
-  "--addheader",
-  ($timeout1 gt "0" ? () : ('--timeout1', $timeout1)),
-  ($timeout2 gt "0" ? () : ('--timeout2', $timeout2)),
-  ($exclude eq "" ? () : ("--exclude", $exclude)),
-  ($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)),
-  ($maxage eq "0" ? () : ('--maxage', $maxage)),
-  ($maxbytespersecond eq "0" ? () : ('--maxbytespersecond', $maxbytespersecond)),
-  ($delete2duplicates ne "1" ? () : ('--delete2duplicates')),
-  ($subscribeall  ne "1" ? () : ('--subscribeall')),
-  ($delete1 ne "1" ? () : ('--delete')),
-  ($delete2 ne "1" ? () : ('--delete2')),
-  ($automap ne "1" ? () : ('--automap')),
-  ($skipcrossduplicates ne "1" ? () : ('--skipcrossduplicates')),
-  (!defined($enc1) ? () : ($enc1)),
-  "--host1", $host1,
-  "--user1", $user1,
-  "--passfile1", $passfile1->filename,
-  "--port1", $port1,
-  "--host2", "localhost",
-  "--user2", $user2 . '*' . trim($master_user),
-  "--passfile2", $passfile2->filename,
-  ($dry eq "1" ? ('--dry') : ()),
-  '--no-modulesversion',
-  '--noreleasecheck'];
-
-  try {
-    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
-    $is_running->bind_param( 1, ${id} );
-    $is_running->execute();
-
-    run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
-
-    # check exit code and status
-    ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
-
-    $success = 0;
-    if (defined $exit_code && $exit_code == 0) {
-      $success = 1;
-    }
-
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
-    $update->bind_param( 1, ${stdout} );
-    $update->bind_param( 2, ${success} );
-    $update->bind_param( 3, ${exit_status} );
-    $update->bind_param( 4, ${id} );
-    $update->execute();
-  } catch {
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
-    $update->execute();
-  } finally {
-    $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
-    $update->execute();
-  };
-
-
-}
-
-$sth->finish();
-$dbh->disconnect();
-
-$lockmgr->unlock($lock_file);

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -0,0 +1,331 @@
+#!/usr/bin/python3
+
+import fcntl
+import json
+import os
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import MySQLdb
+import MySQLdb.cursors
+
+run_dir = '/tmp'
+# Max size of imapsync output stored in returned_text (the tail holds the summary)
+max_output = 16 * 1024 * 1024
+lock_file = run_dir + '/imapsync_busy.lock'
+# An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
+token_min_valid = 900
+# Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
+token_sync_interval = 60
+
+# Next due job: higher prio first, never-run jobs first, then oldest run
+next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
+  i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
+  i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
+  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
+    FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
+      WHERE i.active = 1
+        AND s.active = 1
+        AND i.is_running = 0
+        AND (UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(i.last_run) > i.mins_interval * 60 OR i.last_run IS NULL)
+        {skip}
+  ORDER BY i.prio DESC, i.last_run IS NOT NULL, i.last_run ASC, i.id ASC
+  LIMIT 1"""
+# Atomic claim, a job is only started when this flips is_running 0 -> 1
+claim_sql = """UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL
+  WHERE id = %s AND is_running = 0 AND active = 1"""
+# Access token: client_credentials share the source token, authorization_code use the token of
+# source + local mailbox (user2) + remote user (user1). issued/error describe the last refresh.
+source_token_sql = """SELECT oauth_access_token AS access_token, oauth_token_expires AS token_expires,
+  UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued, oauth_last_refresh_error AS error
+  FROM imapsync_source WHERE id = %s"""
+user_token_sql = """SELECT access_token, token_expires, UNIX_TIMESTAMP(oauth_last_refresh_attempt) AS issued,
+  last_refresh_error AS error FROM imapsync_source_oauth_token
+  WHERE source_id = %s AND owner = %s AND username = %s"""
+
+conn = None
+# Popen => job of running workers
+running = {}
+# Jobs released without a run (token not ready), skipped for the rest of this runner round
+deferred = set()
+stopping = False
+
+def num(value):
+  try:
+    return int(value)
+  except (TypeError, ValueError):
+    return 0
+
+def unlink(files):
+  for path in files:
+    try:
+      os.unlink(path)
+    except OSError:
+      pass
+
+def imapsync_running():
+  for pid in filter(str.isdigit, os.listdir('/proc')):
+    try:
+      with open('/proc/' + pid + '/cmdline', 'rb') as f:
+        cmdline = f.read().replace(b'\0', b' ')
+    except OSError:
+      continue
+    if re.search(rb'imapsync\s', cmdline, re.I):
+      return True
+  return False
+
+def connect():
+  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
+    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
+
+# Reconnect when the server dropped the connection (syncs can outlast wait_timeout)
+def query(sql, args=None):
+  global conn
+  try:
+    conn.ping()
+  except MySQLdb.Error:
+    conn = connect()
+  cur = conn.cursor(MySQLdb.cursors.DictCursor)
+  cur.execute(sql, args)
+  return cur
+
+# Store a result and release the job
+def finish_job(job_id, text, success, exit_status):
+  try:
+    query("UPDATE imapsync SET returned_text = %s, success = %s, exit_status = %s, last_run = NOW(), is_running = 0 WHERE id = %s",
+      (text, success, exit_status, job_id))
+  except MySQLdb.Error:
+    query("UPDATE imapsync SET success = 0, last_run = NOW(), is_running = 0 WHERE id = %s", (job_id,))
+
+# Release a job without a run; last_run stays, so the next runner round retries it
+def defer_job(job_id, text, exit_status):
+  query("UPDATE imapsync SET returned_text = %s, success = 0, exit_status = %s, is_running = 0 WHERE id = %s",
+    (text, exit_status, job_id))
+  deferred.add(job_id)
+
+def oauth_token(j):
+  if j['oauth_flow'] == 'authorization_code':
+    return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
+  return query(source_token_sql, (j['source_id'],)).fetchone() or {}
+
+# Valid for token_min_valid, or half the lifetime of a short-lived token (known when its last refresh succeeded)
+def token_ready(row):
+  token, expires, issued = row.get('access_token'), row.get('token_expires'), row.get('issued')
+  if not token or expires is None:
+    return False
+  need = token_min_valid
+  if issued and not row.get('error') and expires > int(issued):
+    need = min(need, (expires - int(issued)) // 2)
+  return expires - time.time() >= need
+
+# Write a refreshed token over the token file of a running job (atomic, same path)
+def sync_token_file(oauth):
+  token = oauth_token(oauth['job']).get('access_token')
+  if not token or token == oauth['token']:
+    return
+  fd, tmp = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
+  try:
+    with os.fdopen(fd, 'w', encoding='utf-8') as f:
+      f.write(token + '\n')
+    os.replace(tmp, oauth['file'])
+    oauth['token'] = token
+  except OSError:
+    unlink([tmp])
+
+# Max concurrent imapsync processes and global per-process bandwidth cap in bytes/s (0 = off),
+# re-read before every claim round so changes apply to long runs
+def read_settings():
+  settings = {r['name']: r['value'] for r in query("SELECT name, value FROM imapsync_settings")}
+  return max(num(settings.get('max_parallel')), 1), max(num(settings.get('max_bytes_per_second')), 0)
+
+# Build temp files and argv for a claimed job. Returns None when the job was finished without running imapsync.
+def prepare_job(j, global_bw, files):
+  def tmpfile(content):
+    fd, path = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
+    files.append(path)
+    with os.fdopen(fd, 'w', encoding='utf-8') as f:
+      f.write(content)
+    return path
+
+  auth_type = j['auth_type'] or 'PLAIN'
+  oauth = None
+  if auth_type == 'XOAUTH2':
+    row = oauth_token(j)
+    if not token_ready(row):
+      defer_job(j['id'], "OAuth access token for the configured source is missing or expires too soon; refresh cron will retry.",
+        'OAUTH_TOKEN_NOT_READY')
+      return None
+    oauth = {'job': j, 'token': row['access_token'], 'file': tmpfile(row['access_token'] + '\n')}
+    auth_args = ['--authmech1', 'XOAUTH2', '--oauthaccesstoken1', oauth['file']]
+  else:
+    auth_args = ['--passfile1', tmpfile(j['password1'] + '\n')]
+    # imapsync auto-picks PLAIN, only pass --authmech1 for LOGIN/CRAM-MD5
+    if auth_type != 'PLAIN':
+      auth_args += ['--authmech1', auth_type]
+
+  # Custom params: JSON array of {o,v}, validated by the backend on write. One argv token per pair,
+  # "--opt" or "--opt=value" (value glued to its option, no shell)
+  custom_params = []
+  try:
+    pairs = json.loads(j['custom_params'] or '[]')
+  except ValueError:
+    pairs = []
+  for p in pairs if isinstance(pairs, list) else []:
+    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
+      continue
+    value = '' if p.get('v') is None else str(p['v'])
+    custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
+
+  # Effective per-process bandwidth cap (bytes/s): per-job value, capped by the global limit
+  eff_bw = num(j['maxbytespersecond'])
+  if global_bw > 0 and (eff_bw <= 0 or eff_bw > global_bw):
+    eff_bw = global_bw
+
+  # --nolog: the output is stored in returned_text, imapsync's own log dir would be the (read-only) cwd
+  cmd = ['/usr/local/bin/imapsync', '--tmpdir', '/tmp', '--nolog', '--nofoldersizes', '--addheader']
+  if num(j['timeout1']) > 0:
+    cmd += ['--timeout1', str(j['timeout1'])]
+  if num(j['timeout2']) > 0:
+    cmd += ['--timeout2', str(j['timeout2'])]
+  if j['exclude']:
+    cmd += ['--exclude', j['exclude']]
+  if j['subfolder2']:
+    cmd += ['--subfolder2', j['subfolder2']]
+  if num(j['maxage']) != 0:
+    cmd += ['--maxage', str(j['maxage'])]
+  if eff_bw > 0:
+    cmd += ['--maxbytespersecond', str(eff_bw)]
+  for col, opt in (('delete2duplicates', '--delete2duplicates'), ('subscribeall', '--subscribeall'),
+      ('delete1', '--delete1'), ('delete2', '--delete2'), ('automap', '--automap'),
+      ('skipcrossduplicates', '--skipcrossduplicates')):
+    if j[col] == 1:
+      cmd.append(opt)
+  enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
+  if enc1:
+    cmd.append(enc1)
+  cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
+  cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
+  if j['dry'] == 1:
+    cmd.append('--dry')
+  cmd += ['--no-modulesversion', '--noreleasecheck'] + custom_params
+
+  return {'id': j['id'], 'files': files, 'outfile': tmpfile(''), 'cmd': cmd, 'oauth': oauth}
+
+# Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
+def start_job(job):
+  with open(job['outfile'], 'wb') as out:
+    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
+
+# Store the result of a finished worker and remove its temp files
+def reap_job(job):
+  stdout = b''
+  try:
+    with open(job['outfile'], 'rb') as f:
+      size = os.fstat(f.fileno()).st_size
+      if size > max_output:
+        f.seek(size - max_output)
+        stdout = b'[output truncated, last %d bytes]\n' % max_output
+      stdout += f.read()
+  except OSError:
+    pass
+  unlink(job['files'])
+  text = stdout.decode('utf-8', 'replace')
+  # Last match: the echoed command line at the top may contain user controlled values
+  m = re.findall(r'Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)', text)
+  exit_code, exit_status = m[-1] if m else (None, None)
+  if exit_code is None:
+    text = 'Could not start or finish imapsync\n' + text
+  finish_job(job['id'], text, 1 if exit_code is not None and int(exit_code) == 0 else 0, exit_status)
+
+# Stop claiming new jobs, pass the signal on to the running imapsync processes
+def stop(signum, frame):
+  global stopping
+  stopping = True
+  for proc in list(running):
+    proc.send_signal(signal.SIGTERM)
+
+# Released by the kernel when the process ends, a crashed run leaves no stale lock
+lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+try:
+  fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+except OSError:
+  sys.exit("can't lock " + lock_file)
+
+if imapsync_running():
+  print('imapsync is active, exiting...')
+  sys.exit(0)
+
+for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
+  signal.signal(sig, stop)
+
+conn = connect()
+# Stale flags from crashed runs, safe only here: under the lock and with no imapsync running
+query("UPDATE imapsync SET is_running = 0")
+
+with open('/etc/sogo/sieve.creds') as f:
+  master_user, master_pass = [s.strip() for s in f.readline().split(':', 1)]
+
+# Worker pool: keep up to max_parallel workers busy, claim the next due job whenever a slot is free
+# (after every worker exit and at least every 10 seconds while workers run)
+last_token_sync = time.monotonic()
+try:
+  while True:
+    for proc, job in list(running.items()):
+      if proc.poll() is not None:
+        del running[proc]
+        reap_job(job)
+
+    if time.monotonic() - last_token_sync >= token_sync_interval:
+      last_token_sync = time.monotonic()
+      for job in running.values():
+        if job['oauth']:
+          try:
+            sync_token_file(job['oauth'])
+          except MySQLdb.Error as e:
+            print('imapsync job %d: %s' % (job['id'], e), file=sys.stderr)
+
+    max_parallel, global_bw = read_settings()
+    # Deferred and finished jobs drop out of next_sql, only failed claims are capped
+    failed_claims = 0
+    while not stopping and len(running) < max_parallel and failed_claims < 100:
+      skip = ('AND i.id NOT IN (%s)' % ', '.join(['%s'] * len(deferred))) if deferred else ''
+      j = query(next_sql.format(skip=skip), tuple(deferred) or None).fetchone()
+      if not j:
+        break
+      if query(claim_sql, (j['id'],)).rowcount != 1:
+        failed_claims += 1
+        continue
+
+      files = []
+      try:
+        job = prepare_job(j, global_bw, files)
+      except Exception as e:
+        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
+        finish_job(j['id'], 'Could not start or finish imapsync', 0, None)
+        job = None
+      if not job:
+        unlink(files)
+        continue
+      try:
+        running[start_job(job)] = job
+      except OSError as e:
+        # Release the claim, the job is picked up again later
+        print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
+        unlink(files)
+        query("UPDATE imapsync SET is_running = 0 WHERE id = %s", (j['id'],))
+        break
+
+    if not running:
+      break
+    # A worker exit ends the wait early
+    for _ in range(50):
+      if any(proc.poll() is not None for proc in running):
+        break
+      time.sleep(0.2)
+finally:
+  # Never leave password/token files behind
+  for job in running.values():
+    unlink(job['files'])

data/Dockerfiles/dovecot/quarantine_notify.py

@@ -8,7 +8,8 @@ from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import COMMASPACE, formatdate
 import jinja2
-from jinja2 import Template
+from jinja2 import TemplateError
+from jinja2.sandbox import SandboxedEnvironment
 import json
 import redis
 import time
@@ -46,7 +47,7 @@ try:
   if max_score == "":
     max_score = 9999.0
 
-  def query_mysql(query, headers = True, update = False):
+  def query_mysql(query, params = None, headers = True, update = False):
     while True:
       try:
         cnx = MySQLdb.connect(user=os.environ.get('DBUSER'), password=os.environ.get('DBPASS'), database=os.environ.get('DBNAME'), charset="utf8mb4", collation="utf8mb4_general_ci")
@@ -56,7 +57,10 @@ try:
       else:
         break
     cur = cnx.cursor()
-    cur.execute(query)
+    if params:
+      cur.execute(query, params)
+    else:
+      cur.execute(query)
     if not update:
       result = []
       columns = tuple( [d[0] for d in cur.description] )
@@ -75,22 +79,27 @@ try:
 
   def notify_rcpt(rcpt, msg_count, quarantine_acl, category):
     if category == "add_header": category = "add header"
-    meta_query = query_mysql('SELECT SHA2(CONCAT(id, qid), 256) AS qhash, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = "%s" AND score < %f AND (action = "%s" OR "all" = "%s")' % (rcpt, max_score, category, category))
+    meta_query = query_mysql('SELECT `qhash`, id, subject, score, sender, created, action FROM quarantine WHERE notified = 0 AND rcpt = %s AND score < %s AND (action = %s OR "all" = %s)', (rcpt, max_score, category, category))
     print("%s: %d of %d messages qualify for notification" % (rcpt, len(meta_query), msg_count))
     if len(meta_query) == 0:
       return
     msg_count = len(meta_query)
+    env = SandboxedEnvironment()
     if r.get('Q_HTML'):
-      try:
-        template = Template(r.get('Q_HTML'))
-      except:
-        print("Error: Cannot parse quarantine template, falling back to default template.")
-        with open('/templates/quarantine.tpl') as file_:
-          template = Template(file_.read())
+        try:
+            template = env.from_string(r.get('Q_HTML'))
+        except Exception:
+            print("Error: Cannot parse quarantine template, falling back to default template.")
+            with open('/templates/quarantine.tpl') as file_:
+                template = env.from_string(file_.read())
     else:
-      with open('/templates/quarantine.tpl') as file_:
-        template = Template(file_.read())
-    html = template.render(meta=meta_query, username=rcpt, counter=msg_count, hostname=mailcow_hostname, quarantine_acl=quarantine_acl)
+        with open('/templates/quarantine.tpl') as file_:
+            template = env.from_string(file_.read())
+    try:
+        html = template.render(meta=meta_query, username=rcpt, counter=msg_count, hostname=mailcow_hostname, quarantine_acl=quarantine_acl)
+    except (jinja2.exceptions.SecurityError, TemplateError) as ex:
+        print(f"SecurityError or TemplateError in template rendering: {ex}")
+        return
     text = html2text.html2text(html)
     count = 0
     while count < 15:
@@ -124,7 +133,7 @@ try:
             server.sendmail(msg['From'], [str(redirect)] + [str(bcc)], text)
         server.quit()
         for res in meta_query:
-         query_mysql('UPDATE quarantine SET notified = 1 WHERE id = "%d"' % (res['id']), update = True)
+         query_mysql('UPDATE quarantine SET notified = 1 WHERE id = %s', (res['id'],), update = True)
         r.hset('Q_LAST_NOTIFIED', record['rcpt'], time_now)
         break
       except Exception as ex:
@@ -132,7 +141,7 @@ try:
         print('%s'  % (ex))
         time.sleep(3)
 
-  records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND score < %f AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt' % (max_score))
+  records = query_mysql('SELECT IFNULL(user_acl.quarantine, 0) AS quarantine_acl, count(id) AS counter, rcpt FROM quarantine LEFT OUTER JOIN user_acl ON user_acl.username = rcpt WHERE notified = 0 AND score < %s AND rcpt in (SELECT username FROM mailbox) GROUP BY rcpt', (max_score,))
 
   for record in records:
     attrs = ''
@@ -150,7 +159,7 @@ try:
     except Exception as ex:
       print('Could not determine last notification for %s, assuming never' % (record['rcpt']))
       last_notification = 0
-    attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = "%s"' % (record['rcpt']))
+    attrs_json = query_mysql('SELECT attributes FROM mailbox WHERE username = %s', (record['rcpt'],))
     attrs = attrs_json[0]['attributes']
     if isinstance(attrs, str):
       # if attr is str then just load it

data/Dockerfiles/dovecot/quota_notify.py

@@ -6,7 +6,7 @@ from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import COMMASPACE, formatdate
 import jinja2
-from jinja2 import Template
+from jinja2.sandbox import SandboxedEnvironment
 import redis
 import time
 import json
@@ -33,16 +33,24 @@ while True:
 
 if r.get('QW_HTML'):
   try:
-    template = Template(r.get('QW_HTML'))
-  except:
-    print("Error: Cannot parse quarantine template, falling back to default template.")
+    env = SandboxedEnvironment()
+    template = env.from_string(r.get('QW_HTML'))
+  except Exception:
+    print("Error: Cannot parse quota template, falling back to default template.")
     with open('/templates/quota.tpl') as file_:
-      template = Template(file_.read())
+      env = SandboxedEnvironment()
+      template = env.from_string(file_.read())
 else:
   with open('/templates/quota.tpl') as file_:
-    template = Template(file_.read())
+    env = SandboxedEnvironment()
+    template = env.from_string(file_.read())
+
+try:
+  html = template.render(username=username, percent=percent)
+except (jinja2.exceptions.SecurityError, jinja2.TemplateError) as ex:
+  print(f"SecurityError or TemplateError in template rendering: {ex}")
+  sys.exit(1)
 
-html = template.render(username=username, percent=percent)
 text = html2text.html2text(html)
 
 try:

data/Dockerfiles/netfilter/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.23
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
@@ -40,4 +40,4 @@ COPY ./docker-entrypoint.sh /app/
 
 RUN chmod +x /app/docker-entrypoint.sh
 
-CMD ["/bin/sh", "-c", "/app/docker-entrypoint.sh"]
+CMD ["/bin/sh", "-c", "/app/docker-entrypoint.sh"]

data/Dockerfiles/netfilter/docker-entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-backend=iptables
+backend=nftables
 
 nft list table ip filter &>/dev/null
 nftables_found=$?
@@ -17,8 +17,8 @@ if [ $nftables_found -gt $iptables_found ]; then
 fi
 
 if [ $nftables_found -eq 0 ] && [ $nftables_found -eq $iptables_found ]; then
-  nftables_lines=$(nft list ruleset | wc -l)
-  iptables_lines=$(iptables-save | wc -l)
+  nftables_lines=$(nft list ruleset 2>/dev/null | wc -l)
+  iptables_lines=$(iptables-save 2>/dev/null | wc -l)
   if [ $nftables_lines -gt $iptables_lines ]; then
     backend=nftables
   else

data/Dockerfiles/netfilter/main.py

@@ -1,502 +1,57 @@
-#!/usr/bin/env python3
-
-import re
-import os
-import sys
-import time
-import atexit
-import signal
-import ipaddress
-from collections import Counter
-from random import randint
-from threading import Thread
-from threading import Lock
-import redis
-import json
-import dns.resolver
-import dns.exception
-import uuid
-from modules.Logger import Logger
-from modules.IPTables import IPTables
-from modules.NFTables import NFTables
-
-
-# globals
-WHITELIST = []
-BLACKLIST= []
-bans = {}
-quit_now = False
-exit_code = 0
-lock = Lock()
-chain_name = "MAILCOW"
-r = None
-pubsub = None
-clear_before_quit = False
-
-
-def refreshF2boptions():
-  global f2boptions
-  global quit_now
-  global exit_code
-
-  f2boptions = {}
-
-  if not r.get('F2B_OPTIONS'):
-    f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
-    f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
-    f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
-    f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
-    f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
-    f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
-    f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
-  else:
-    try:
-      f2boptions = json.loads(r.get('F2B_OPTIONS'))
-    except ValueError:
-      logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json')
-      quit_now = True
-      exit_code = 2
-
-  verifyF2boptions(f2boptions)
-  r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
-
-def verifyF2boptions(f2boptions):
-  verifyF2boption(f2boptions,'ban_time', 1800)
-  verifyF2boption(f2boptions,'max_ban_time', 10000)
-  verifyF2boption(f2boptions,'ban_time_increment', True)
-  verifyF2boption(f2boptions,'max_attempts', 10)
-  verifyF2boption(f2boptions,'retry_window', 600)
-  verifyF2boption(f2boptions,'netban_ipv4', 32)
-  verifyF2boption(f2boptions,'netban_ipv6', 128)
-  verifyF2boption(f2boptions,'banlist_id', str(uuid.uuid4()))
-  verifyF2boption(f2boptions,'manage_external', 0)
-
-def verifyF2boption(f2boptions, f2boption, f2bdefault):
-  f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault
-
-def refreshF2bregex():
-  global f2bregex
-  global quit_now
-  global exit_code
-  if not r.get('F2B_REGEX'):
-    f2bregex = {}
-    f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-    f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
-    f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
-    f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
-    f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
-    f2bregex[6] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[7] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)'
-    f2bregex[8] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-    f2bregex[9] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
-    r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
-  else:
-    try:
-      f2bregex = {}
-      f2bregex = json.loads(r.get('F2B_REGEX'))
-    except ValueError:
-      logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
-      quit_now = True
-      exit_code = 2
-
-def get_ip(address):
-  ip = ipaddress.ip_address(address)
-  if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped:
-    ip = ip.ipv4_mapped
-  if ip.is_private or ip.is_loopback:
-    return False
-
-  return ip
-
-def ban(address):
-  global f2boptions
-  global lock
-
-  refreshF2boptions()
-  MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-  RETRY_WINDOW = int(f2boptions['retry_window'])
-  NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
-  NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6'])
-
-  ip = get_ip(address)
-  if not ip: return
-  address = str(ip)
-  self_network = ipaddress.ip_network(address)
-
-  with lock:
-    temp_whitelist = set(WHITELIST)
-  if temp_whitelist:
-    for wl_key in temp_whitelist:
-      wl_net = ipaddress.ip_network(wl_key, False)
-      if wl_net.overlaps(self_network):
-        logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net))
-        return
-
-  net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False)
-  net = str(net)
-
-  if not net in bans:
-    bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0}
-
-  current_attempt = time.time()
-  if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW:
-    bans[net]['attempts'] = 0
-
-  bans[net]['attempts'] += 1
-  bans[net]['last_attempt'] = current_attempt
-
-  if bans[net]['attempts'] >= MAX_ATTEMPTS:
-    cur_time = int(round(time.time()))
-    NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
-    logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
-    if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
-      with lock:
-        tables.banIPv4(net)
-    elif int(f2boptions['manage_external']) != 1:
-      with lock:
-        tables.banIPv6(net)
-
-    r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
-  else:
-    logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
-
-def unban(net):
-  global lock
-
-  if not net in bans:
-   logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
-   r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-   return
-
-  logger.logInfo('Unbanning %s' % net)
-  if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network:
-    with lock:
-      tables.unbanIPv4(net)
-  else:
-    with lock:
-      tables.unbanIPv6(net)
-
-  r.hdel('F2B_ACTIVE_BANS', '%s' % net)
-  r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
-  if net in bans:
-    bans[net]['attempts'] = 0
-    bans[net]['ban_counter'] += 1
-
-def permBan(net, unban=False):
-  global f2boptions
-  global lock
-
-  is_unbanned = False
-  is_banned = False
-  if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv4(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv4(net)
-  else:
-    with lock:
-      if unban:
-        is_unbanned = tables.unbanIPv6(net)
-      elif int(f2boptions['manage_external']) != 1:
-        is_banned = tables.banIPv6(net)
-
-
-  if is_unbanned:
-    r.hdel('F2B_PERM_BANS', '%s' % net)
-    logger.logCrit('Removed host/network %s from blacklist' % net)
-  elif is_banned:
-    r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
-    logger.logCrit('Added host/network %s to blacklist' % net)
-
-def clear():
-  global lock
-  logger.logInfo('Clearing all bans')
-  for net in bans.copy():
-    unban(net)
-  with lock:
-    tables.clearIPv4Table()
-    tables.clearIPv6Table()
-    try:
-      if r is not None:
-        r.delete('F2B_ACTIVE_BANS')
-        r.delete('F2B_PERM_BANS')
-    except Exception as ex:
-      logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
-
-def watch():
-  global pubsub
-  global quit_now
-  global exit_code
-
-  logger.logInfo('Watching Redis channel F2B_CHANNEL')
-  pubsub.subscribe('F2B_CHANNEL')
-
-  while not quit_now:
-    try:
-      for item in pubsub.listen():
-        refreshF2bregex()
-        for rule_id, rule_regex in f2bregex.items():
-          if item['data'] and item['type'] == 'message':
-            try:
-              result = re.search(rule_regex, item['data'])
-            except re.error:
-              result = False
-            if result:
-              addr = result.group(1)
-              ip = ipaddress.ip_address(addr)
-              if ip.is_private or ip.is_loopback:
-                continue
-              logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data']))
-              ban(addr)
-    except Exception as ex:
-      logger.logWarn('Error reading log line from pubsub: %s' % ex)
-      pubsub = None
-      quit_now = True
-      exit_code = 2
-
-def snat4(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24')
-
-def snat6(snat_target):
-  global lock
-  global quit_now
-
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
-
-def autopurge():
-  global f2boptions
-
-  while not quit_now:
-    time.sleep(10)
-    refreshF2boptions()
-    MAX_ATTEMPTS = int(f2boptions['max_attempts'])
-    QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
-    if QUEUE_UNBAN:
-      for net in QUEUE_UNBAN:
-        unban(str(net))
-    for net in bans.copy():
-      if bans[net]['attempts'] >= MAX_ATTEMPTS:
-        NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
-        TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
-        if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME:
-          unban(net)
-
-def mailcowChainOrder():
-  global lock
-  global quit_now
-  global exit_code
-  while not quit_now:
-    time.sleep(10)
-    with lock:
-      quit_now, exit_code = tables.checkIPv4ChainOrder()
-      if quit_now: return
-      quit_now, exit_code = tables.checkIPv6ChainOrder()
-
-def calcNetBanTime(ban_counter):
-  global f2boptions
-
-  BAN_TIME = int(f2boptions['ban_time'])
-  MAX_BAN_TIME = int(f2boptions['max_ban_time'])
-  BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
-  NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
-  NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
-  return NET_BAN_TIME
-
-def isIpNetwork(address):
-  try:
-    ipaddress.ip_network(address, False)
-  except ValueError:
-    return False
-  return True
-
-def genNetworkList(list):
-  resolver = dns.resolver.Resolver()
-  hostnames = []
-  networks = []
-  for key in list:
-    if isIpNetwork(key):
-      networks.append(key)
-    else:
-      hostnames.append(key)
-  for hostname in hostnames:
-    hostname_ips = []
-    for rdtype in ['A', 'AAAA']:
-      try:
-        answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3)
-      except dns.exception.Timeout:
-        logger.logInfo('Hostname %s timedout on resolve' % hostname)
-        break
-      except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-        continue
-      except dns.exception.DNSException as dnsexception:
-        logger.logInfo('%s' % dnsexception)
-        continue
-      for rdata in answer:
-        hostname_ips.append(rdata.to_text())
-    networks.extend(hostname_ips)
-  return set(networks)
-
-def whitelistUpdate():
-  global lock
-  global quit_now
-  global WHITELIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_WHITELIST')
-    new_whitelist = []
-    if list:
-      new_whitelist = genNetworkList(list)
-    with lock:
-      if Counter(new_whitelist) != Counter(WHITELIST):
-        WHITELIST = new_whitelist
-        logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST))
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def blacklistUpdate():
-  global quit_now
-  global BLACKLIST
-  while not quit_now:
-    start_time = time.time()
-    list = r.hgetall('F2B_BLACKLIST')
-    new_blacklist = []
-    if list:
-      new_blacklist = genNetworkList(list)
-    if Counter(new_blacklist) != Counter(BLACKLIST):
-      addban = set(new_blacklist).difference(BLACKLIST)
-      delban = set(BLACKLIST).difference(new_blacklist)
-      BLACKLIST = new_blacklist
-      logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST))
-      if addban:
-        for net in addban:
-          permBan(net=net)
-      if delban:
-        for net in delban:
-          permBan(net=net, unban=True)
-    time.sleep(60.0 - ((time.time() - start_time) % 60.0))
-
-def sigterm_quit(signum, frame):
-  global clear_before_quit
-  clear_before_quit = True
-  sys.exit(exit_code)
-
-def berfore_quit():
-  if clear_before_quit:
-    clear()
-  if pubsub is not None:
-    pubsub.unsubscribe()
-
-
-if __name__ == '__main__':
-  atexit.register(berfore_quit)
-  signal.signal(signal.SIGTERM, sigterm_quit)
-
-  # init Logger
-  logger = Logger()
-
-  # init backend
-  backend = sys.argv[1]
-  if backend == "nftables":
-    logger.logInfo('Using NFTables backend')
-    tables = NFTables(chain_name, logger)
-  else:
-    logger.logInfo('Using IPTables backend')
-    tables = IPTables(chain_name, logger)
-
-  # In case a previous session was killed without cleanup
-  clear()
-
-  # Reinit MAILCOW chain
-  # Is called before threads start, no locking
-  logger.logInfo("Initializing mailcow netfilter chain")
-  tables.initChainIPv4()
-  tables.initChainIPv6()
-
-  if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE").lower() in ("y", "yes"):
-    logger.logInfo(f"Skipping {chain_name} isolation")
-  else:
-    logger.logInfo(f"Setting {chain_name} isolation")
-    tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
-
-  # connect to redis
-  while True:
-    try:
-      redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
-      redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
-      if "".__eq__(redis_slaveof_ip):
-        r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
-      else:
-        r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0, password=os.environ['REDISPASS'])
-      r.ping()
-      pubsub = r.pubsub()
-    except Exception as ex:
-      print('%s - trying again in 3 seconds'  % (ex))
-      time.sleep(3)
-    else:
-      break
-  logger.set_redis(r)
-
-  # rename fail2ban to netfilter
-  if r.exists('F2B_LOG'):
-    r.rename('F2B_LOG', 'NETFILTER_LOG')
-  # clear bans in redis
-  r.delete('F2B_ACTIVE_BANS')
-  r.delete('F2B_PERM_BANS')
-
-  refreshF2boptions()
-
-  watch_thread = Thread(target=watch)
-  watch_thread.daemon = True
-  watch_thread.start()
-
-  if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv4Address:
-        snat4_thread = Thread(target=snat4,args=(snat_ip,))
-        snat4_thread.daemon = True
-        snat4_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address')
-
-  if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n':
-    try:
-      snat_ip = os.getenv('SNAT6_TO_SOURCE')
-      snat_ipo = ipaddress.ip_address(snat_ip)
-      if type(snat_ipo) is ipaddress.IPv6Address:
-        snat6_thread = Thread(target=snat6,args=(snat_ip,))
-        snat6_thread.daemon = True
-        snat6_thread.start()
-    except ValueError:
-      print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address')
-
-  autopurge_thread = Thread(target=autopurge)
-  autopurge_thread.daemon = True
-  autopurge_thread.start()
-
-  mailcowchainwatch_thread = Thread(target=mailcowChainOrder)
-  mailcowchainwatch_thread.daemon = True
-  mailcowchainwatch_thread.start()
-
-  blacklistupdate_thread = Thread(target=blacklistUpdate)
-  blacklistupdate_thread.daemon = True
-  blacklistupdate_thread.start()
-
-  whitelistupdate_thread = Thread(target=whitelistUpdate)
-  whitelistupdate_thread.daemon = True
-  whitelistupdate_thread.start()
-
-  while not quit_now:
-    time.sleep(0.5)
-
-  sys.exit(exit_code)
+#!/usr/bin/env python3
+
+import os
+import sys
+import time
+import signal
+import redis
+from modules.Logger import Logger
+from modules.Netfilter import Netfilter
+
+chain_name = "MAILCOW"
+
+def connect_redis(logger):
+  if os.getenv('REDIS_SLAVEOF_IP', ''):
+    host, port = os.getenv('REDIS_SLAVEOF_IP'), os.getenv('REDIS_SLAVEOF_PORT', '')
+  else:
+    host, port = os.getenv('IPV4_NETWORK', '172.22.1') + '.249', 6379
+  while True:
+    try:
+      # log lines are not always valid UTF-8
+      r = redis.StrictRedis(host=host, port=port, db=0, password=os.environ['REDISPASS'], decode_responses=True,
+                            encoding_errors='replace', socket_connect_timeout=5, socket_timeout=30)
+      r.ping()
+      return r
+    except Exception as ex:
+      logger.logWarn('Redis connection failed: %s - trying again in 3 seconds' % ex)
+      time.sleep(3)
+
+if __name__ == '__main__':
+  logger = Logger()
+
+  backend = sys.argv[1]
+  if backend == "nftables":
+    from modules.NFTables import NFTables
+    logger.logInfo('Using NFTables backend')
+    tables = NFTables(chain_name, logger)
+  else:
+    from modules.IPTables import IPTables
+    logger.logInfo('Using IPTables backend')
+    logger.logWarn(
+        "DEPRECATION: iptables-legacy is deprecated and will be removed in future releases. "
+        "Please switch to nftables on your host to ensure complete compatibility."
+    )
+    time.sleep(5)
+    tables = IPTables(chain_name, logger)
+
+  r = connect_redis(logger)
+  logger.set_redis(r)
+
+  netfilter = Netfilter(r, tables, logger, os.environ)
+  signal.signal(signal.SIGTERM, lambda signum, frame: netfilter.stop.set())
+  signal.signal(signal.SIGINT, lambda signum, frame: netfilter.stop.set())
+
+  netfilter.setup()
+  netfilter.run()
+  netfilter.shutdown()
+  sys.exit(0)

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -0,0 +1,236 @@
+import ipaddress
+import time
+
+
+class BanManager:
+  """Failed attempts and automatic bans (Redis F2B_ACTIVE_BANS: network => expiry), the denylist (F2B_PERM_BANS)
+  and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
+  (chains, SNAT), because the backends must not run in parallel."""
+
+  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
+  EXCUSE_TIME = 60
+
+  def __init__(self, r, tables, options, logger, lock, clock=time.time):
+    self.r = r
+    self.tables = tables
+    self.options = options
+    self.logger = logger
+    self.lock = lock
+    self.clock = clock
+    self.counters = {}
+    self.banned = set()
+    self.denied = {}
+    self.allowlist = []
+    self.excused = {}
+
+  def excuse(self, ip):
+    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
+    with self.lock:
+      now = self.clock()
+      if self.excused.get(ip, 0) <= now:
+        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
+          ip, self.EXCUSE_TIME))
+      self.excused[ip] = now + self.EXCUSE_TIME
+
+  def attempt(self, ip):
+    """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
+    options = self.options.values
+    prefix = options['netban_ipv4'] if ip.version == 4 else options['netban_ipv6']
+    net = str(ipaddress.ip_network('%s/%d' % (ip, prefix), strict=False))
+    with self.lock:
+      now = self.clock()
+      allowed = self.allowed_by(ip)
+      if allowed is not None:
+        self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
+        return
+      if self.excused.get(ip, 0) > now:
+        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
+        return
+      if net in self.banned:
+        # log lines of connections from before the ban
+        return
+      counter = self.counter(net)
+      if now - counter['last_attempt'] > options['retry_window']:
+        counter['attempts'] = 0
+      counter['attempts'] += 1
+      counter['last_attempt'] = now
+      if counter['attempts'] < options['max_attempts']:
+        self.logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (
+          options['max_attempts'] - counter['attempts'], options['retry_window'], net))
+        return
+
+      ban_time = self.options.ban_time(counter['ban_counter'])
+      self.logger.logCrit('Banning %s for %d minutes' % (net, ban_time / 60))
+      # Redis first: a firewall rule without expiry would never be lifted
+      self.r.hset('F2B_ACTIVE_BANS', net, int(round(now + ban_time)))
+      self.banned.add(net)
+      if options['manage_external'] != 1:
+        self.apply([net])
+
+  def unban(self, net):
+    """Lifts the automatic ban of a network; the rule stays while the denylist bans the same network"""
+    with self.lock:
+      normalized = self.normalize(net)
+      if normalized is None:
+        self.logger.logWarn('Removing invalid network %s from the active bans' % net)
+      else:
+        self.logger.logInfo('Unbanning %s' % net)
+        if normalized not in self.denied:
+          self.remove([normalized])
+      self.r.hdel('F2B_ACTIVE_BANS', net)
+      self.r.hdel('F2B_QUEUE_UNBAN', net)
+      if net in self.banned:
+        self.banned.discard(net)
+        counter = self.counter(net)
+        counter['attempts'] = 0
+        counter['ban_counter'] += 1
+
+  def unban_queued(self, entry):
+    """Unban request of the UI: a banned network, or a network that contains banned ones (allowlisting)"""
+    network = self.network(entry)
+    if network is None:
+      self.logger.logWarn('Removing invalid network %s from the unban queue' % entry)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+      return
+    with self.lock:
+      banned = set(self.r.hkeys('F2B_ACTIVE_BANS')) | self.banned
+      targets = [net for net in banned if self.overlaps(net, network)]
+      for net in targets:
+        self.unban(net)
+      if not targets and str(network) not in self.denied:
+        # a rule without known ban, e.g. left by an older netfilter
+        if self.remove([str(network)]):
+          self.logger.logInfo('Unbanning %s' % network)
+        else:
+          self.logger.logInfo('%s is not banned, removing it from the unban queue' % network)
+      self.r.hdel('F2B_QUEUE_UNBAN', entry)
+
+  def purge(self):
+    """Processes the unban queue of the UI and lifts expired bans"""
+    for entry in self.r.hkeys('F2B_QUEUE_UNBAN'):
+      self.unban_queued(entry)
+    now = self.clock()
+    for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+      try:
+        expired = self.network(net) is None or float(expires) <= now
+      except (TypeError, ValueError):
+        expired = True
+      if expired:
+        self.unban(net)
+    with self.lock:
+      # bans whose Redis entry is gone, e.g. deleted by hand
+      for net in self.banned - set(self.r.hkeys('F2B_ACTIVE_BANS')):
+        self.unban(net)
+    self.forget_idle(now)
+
+  def restore(self):
+    """Applies the unexpired automatic bans of Redis again after a start of netfilter and drops the expired ones"""
+    now = self.clock()
+    restored = []
+    with self.lock:
+      for net, expires in self.r.hgetall('F2B_ACTIVE_BANS').items():
+        try:
+          valid = self.network(net) is not None and float(expires) > now
+        except (TypeError, ValueError):
+          valid = False
+        if not valid:
+          self.r.hdel('F2B_ACTIVE_BANS', net)
+          continue
+        self.banned.add(net)
+        self.counter(net)
+        restored.append(net)
+      if restored and self.options.values['manage_external'] != 1:
+        self.apply(restored)
+    if restored:
+      self.logger.logInfo('Restored %d active bans' % len(restored))
+
+  def reapply(self):
+    """Applies all bans again, e.g. after the MAILCOW chain had to be recreated"""
+    with self.lock:
+      if self.options.values['manage_external'] != 1:
+        self.apply(list(self.banned) + list(self.denied))
+
+  def set_denylist(self, entries):
+    """Bans the networks of the denylist permanently and lifts the bans of removed entries"""
+    networks = {}
+    for entry in entries:
+      net = self.normalize(entry)
+      if net is None:
+        self.logger.logWarn('Skipping invalid denylist entry %s' % entry)
+        continue
+      networks[net] = entry
+    with self.lock:
+      if networks.keys() == self.denied.keys():
+        return
+      self.logger.logInfo('Denylist was changed, it has %d entries' % len(networks))
+      added = [net for net in networks if net not in self.denied]
+      removed = [net for net in self.denied if net not in networks]
+      if added and self.options.values['manage_external'] != 1:
+        self.apply(added)
+      for net in added:
+        self.r.hset('F2B_PERM_BANS', networks[net], int(round(self.clock())))
+        self.logger.logCrit('Added host/network %s to denylist' % networks[net])
+      # an automatic ban of the same network keeps its rule
+      auto_banned = {self.normalize(net) for net in self.banned}
+      self.remove([net for net in removed if net not in auto_banned])
+      for net in removed:
+        self.r.hdel('F2B_PERM_BANS', self.denied[net])
+        self.logger.logCrit('Removed host/network %s from denylist' % self.denied[net])
+      self.denied = networks
+
+  def set_allowlist(self, entries):
+    networks = []
+    for entry in entries:
+      network = self.network(entry)
+      if network is None:
+        self.logger.logWarn('Skipping invalid allowlist entry %s' % entry)
+        continue
+      networks.append(network)
+    with self.lock:
+      if set(networks) != set(self.allowlist):
+        self.allowlist = networks
+        self.logger.logInfo('Allowlist was changed, it has %d entries' % len(networks))
+
+  def allowed_by(self, ip):
+    address = ipaddress.ip_network(ip)
+    for network in self.allowlist:
+      if network.version == address.version and network.overlaps(address):
+        return network
+    return None
+
+  def counter(self, net):
+    return self.counters.setdefault(net, {'attempts': 0, 'last_attempt': 0.0, 'ban_counter': 0})
+
+  def forget_idle(self, now):
+    """Drops counters of networks that were never banned and had no attempt within the retry window"""
+    window = self.options.values['retry_window']
+    with self.lock:
+      for net in [net for net, counter in self.counters.items()
+                  if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
+        del self.counters[net]
+      for ip in [ip for ip, until in self.excused.items() if until <= now]:
+        del self.excused[ip]
+
+  def apply(self, nets):
+    for net in self.tables.ban(nets):
+      self.logger.logWarn('Could not add the firewall rule for %s' % net)
+
+  def remove(self, nets):
+    return self.tables.unban(nets) if nets else []
+
+  @staticmethod
+  def network(entry):
+    try:
+      return ipaddress.ip_network(entry, strict=False)
+    except (TypeError, ValueError):
+      return None
+
+  @classmethod
+  def normalize(cls, entry):
+    network = cls.network(entry)
+    return str(network) if network is not None else None
+
+  @classmethod
+  def overlaps(cls, net, network):
+    other = cls.network(net)
+    return other is not None and other.version == network.version and other.overlaps(network)

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,6 +1,7 @@
 import iptc
 import time
-import os
+import ipaddress
+import contextlib
 
 class IPTables:
   def __init__(self, chain_name, logger):
@@ -8,62 +9,10 @@ class IPTables:
     self.logger = logger
 
   def initChainIPv4(self):
-    if not iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name) in iptc.Table(iptc.Table.FILTER).chains:
-      iptc.Table(iptc.Table.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), c)
-      rule = iptc.Rule()
-      rule.src = '0.0.0.0/0'
-      rule.dst = '0.0.0.0/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
+    self.init_chain(iptc.Table(iptc.Table.FILTER))
 
   def initChainIPv6(self):
-    if not iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name) in iptc.Table6(iptc.Table6.FILTER).chains:
-      iptc.Table6(iptc.Table6.FILTER).create_chain(self.chain_name)
-    for c in ['FORWARD', 'INPUT']:
-      chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), c)
-      rule = iptc.Rule6()
-      rule.src = '::/0'
-      rule.dst = '::/0'
-      target = iptc.Target(rule, self.chain_name)
-      rule.target = target
-      if rule not in chain.rules:
-        chain.insert_rule(rule)
-
-  def checkIPv4ChainOrder(self):
-    filter_table = iptc.Table(iptc.Table.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkIPv6ChainOrder(self):
-    filter_table = iptc.Table6(iptc.Table6.FILTER)
-    filter_table.refresh()
-    return self.checkChainOrder(filter_table)
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    forward_chain = iptc.Chain(filter_table, 'FORWARD')
-    input_chain = iptc.Chain(filter_table, 'INPUT')
-    for chain in [forward_chain, input_chain]:
-      target_found = False
-      for position, item in enumerate(chain.rules):
-        if item.target.name == self.chain_name:
-          target_found = True
-          if position > 2:
-            self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-            err = True
-            exit_code = 2
-      if not target_found:
-        self.logger.logCrit('Error in %s chain: %s target not found, restarting container' % (chain.name, self.chain_name))
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain(iptc.Table6(iptc.Table6.FILTER))
 
   def clearIPv4Table(self):
     self.clearTable(iptc.Table(iptc.Table.FILTER))
@@ -71,68 +20,133 @@ class IPTables:
   def clearIPv6Table(self):
     self.clearTable(iptc.Table6(iptc.Table6.FILTER))
 
-  def clearTable(self, filter_table):
-    filter_table.autocommit = False
-    forward_chain = iptc.Chain(filter_table, "FORWARD")
-    input_chain = iptc.Chain(filter_table, "INPUT")
-    mailcow_chain = iptc.Chain(filter_table, self.chain_name)
-    if mailcow_chain in filter_table.chains:
-      for rule in mailcow_chain.rules:
-        mailcow_chain.delete_rule(rule)
-      for rule in forward_chain.rules:
-        if rule.target.name == self.chain_name:
-          forward_chain.delete_rule(rule)
-      for rule in input_chain.rules:
-        if rule.target.name == self.chain_name:
-          input_chain.delete_rule(rule)
-      filter_table.delete_chain(self.chain_name)
-    filter_table.commit()
+  @contextlib.contextmanager
+  def transaction(self, table):
+    # One commit for all changes, autocommit rewrites the whole table for every change
+    table.autocommit = False
+    table.refresh()
+    try:
+      yield
+      table.commit()
+    finally:
+      table.refresh()
+      table.autocommit = True
+
+  def new_rule(self, table):
+    return iptc.Rule6() if isinstance(table, iptc.Table6) else iptc.Rule()
+
+  def family(self, table):
+    return 'IPv6' if isinstance(table, iptc.Table6) else 'IPv4'
+
+  def init_chain(self, filter_table):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
     filter_table.refresh()
-    filter_table.autocommit = True
+    if filter_table.is_chain(self.chain_name):
+      filter_table.flush_entries(self.chain_name)
+    else:
+      filter_table.create_chain(self.chain_name)
+    self.ensure_jumps(filter_table)
 
-  def banIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    recreated = False
+    for filter_table in [iptc.Table(iptc.Table.FILTER), iptc.Table6(iptc.Table6.FILTER)]:
+      filter_table.refresh()
+      if not filter_table.is_chain(self.chain_name):
+        filter_table.create_chain(self.chain_name)
+        self.logger.logWarn(f'MAILCOW {self.family(filter_table)} chain was missing and has been recreated')
+        recreated = True
+      self.ensure_jumps(filter_table)
+    return recreated
 
-  def banIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule in chain.rules:
-      return False
-    chain.insert_rule(rule)
-    return True
+  def ensure_jumps(self, filter_table):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jump (DOCKER-USER) at the top
+    for name in ['INPUT', 'FORWARD']:
+      chain = iptc.Chain(filter_table, name)
+      positions = [position for position, rule in enumerate(chain.rules) if rule.target.name == self.chain_name]
+      if positions == [0]: continue
 
-  def unbanIPv4(self, source):
-    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
-    rule = iptc.Rule()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules: 
-      return False
-    chain.delete_rule(rule)
-    return True
+      with self.transaction(filter_table):
+        for rule in [rule for rule in chain.rules if rule.target.name == self.chain_name]:
+          chain.delete_rule(rule)
+        rule = self.new_rule(filter_table)
+        rule.target = iptc.Target(rule, self.chain_name)
+        chain.insert_rule(rule)
+      if positions:
+        self.logger.logInfo(f'Moved MAILCOW jump to the top of the {self.family(filter_table)} {name} chain, it was at position {positions[0] + 1}')
+      else:
+        self.logger.logInfo(f'Added MAILCOW jump to the {self.family(filter_table)} {name} chain')
 
-  def unbanIPv6(self, source):
-    chain = iptc.Chain(iptc.Table6(iptc.Table6.FILTER), self.chain_name)
-    rule = iptc.Rule6()
-    rule.src = source
-    target = iptc.Target(rule, "REJECT")
-    rule.target = target
-    if rule not in chain.rules:
-      return False
-    chain.delete_rule(rule)
-    return True
+  def clearTable(self, filter_table):
+    try:
+      with self.transaction(filter_table):
+        if not filter_table.is_chain(self.chain_name):
+          return
+        for chain in filter_table.chains:
+          for rule in chain.rules:
+            if rule.target.name == self.chain_name:
+              chain.delete_rule(rule)
+        filter_table.flush_entries(self.chain_name)
+        filter_table.delete_chain(self.chain_name)
+      self.logger.logInfo(f'Clear completed: {self.family(filter_table)}')
+    except Exception as ex:
+      self.logger.logWarn(f'Error clearing the {self.family(filter_table)} MAILCOW chain: {ex}')
+
+  def ban_rule_network(self, rule):
+    # Source network of a ban rule, None for every other rule
+    if rule.target.name not in ('REJECT', 'DROP') or rule.matches or rule.in_interface or rule.out_interface:
+      return None
+    try:
+      return ipaddress.ip_network(rule.src, strict=False)
+    except ValueError:
+      return None
+
+  def split_tables(self, sources):
+    networks = {4: [], 6: []}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      networks[network.version].append(network)
+    tables = [(iptc.Table(iptc.Table.FILTER), networks[4]), (iptc.Table6(iptc.Table6.FILTER), networks[6])]
+    return [(table, table_networks) for table, table_networks in tables if table_networks]
+
+  def ban(self, sources):
+    # One REJECT rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for filter_table, networks in self.split_tables(sources):
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          existing = {self.ban_rule_network(rule) for rule in chain.rules}
+          for network in dict.fromkeys(networks):
+            if network in existing: continue
+            rule = self.new_rule(filter_table)
+            rule.src = str(network)
+            rule.target = iptc.Target(rule, "REJECT")
+            chain.insert_rule(rule)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        failed += networks
+    return [str(network) for network in failed]
+
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for filter_table, networks in self.split_tables(sources):
+      targets = set(networks)
+      try:
+        with self.transaction(filter_table):
+          chain = iptc.Chain(filter_table, self.chain_name)
+          found = set()
+          for rule in chain.rules:
+            network = self.ban_rule_network(rule)
+            if network in targets:
+              chain.delete_rule(rule)
+              found.add(network)
+      except Exception as ex:
+        self.logger.logWarn(f'iptables: {ex}')
+        continue
+      removed += found
+    return [str(network) for network in removed]
 
   def snat4(self, snat_target, source):
     try:
@@ -188,7 +202,7 @@ class IPTables:
           if item == new_rule:
             if position != 0:
               chain.delete_rule(new_rule)
-    
+
       table.commit()
       table.autocommit = True
     except:
@@ -215,7 +229,9 @@ class IPTables:
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
     try:
-      chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+      table = iptc.Table(iptc.Table.FILTER)
+      table.refresh()
+      chain = iptc.Chain(table, self.chain_name)
 
       # insert mailcow isolation rule
       rule = iptc.Rule()
@@ -231,7 +247,7 @@ class IPTables:
       chain.insert_rule(rule, position=0)
 
       # insert mailcow isolation exception rule
-      if _allow != "":
+      if _allow:
         rule = iptc.Rule()
         rule.src = _allow
         rule.in_interface = f'!{_interface}'
@@ -249,4 +265,4 @@ class IPTables:
       return True
     except Exception as e:
       self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
-      return False
+      return False

data/Dockerfiles/netfilter/modules/Logger.py

@@ -1,5 +1,6 @@
 import time
 import json
+import datetime
 
 class Logger:
   def __init__(self):
@@ -8,17 +9,28 @@ class Logger:
   def set_redis(self, redis):
     self.r = redis
 
+  def _format_timestamp(self):
+    # Local time with milliseconds
+    return datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
   def log(self, priority, message):
-    tolog = {}
-    tolog['time'] = int(round(time.time()))
-    tolog['priority'] = priority
-    tolog['message'] = message
-    print(message)
+    # build redis-friendly dict
+    tolog = {
+      'time': int(round(time.time())),  # keep raw timestamp for Redis
+      'priority': priority,
+      'message': message
+    }
+
+    # print human-readable message with timestamp
+    ts = self._format_timestamp()
+    print(f"{ts} {priority.upper()}: {message}", flush=True)
+
+    # also push JSON to Redis if connected
     if self.r is not None:
       try:
         self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
       except Exception as ex:
-        print('Failed logging to redis: %s'  % (ex))
+        print(f'{ts} WARN: Failed logging to redis: {ex}', flush=True)
 
   def logWarn(self, message):
     self.log('warn', message)
@@ -27,4 +39,4 @@ class Logger:
     self.log('crit', message)
 
   def logInfo(self, message):
-    self.log('info', message)
+    self.log('info', message)

data/Dockerfiles/netfilter/modules/NFTables.py

@@ -1,8 +1,9 @@
 import nftables
 import ipaddress
-import os
 
 class NFTables:
+  ISOLATION_COMMENTS = ('mailcow isolation', 'mailcow isolation allow')
+
   def __init__(self, chain_name, logger):
     self.chain_name = chain_name
     self.logger = logger
@@ -12,41 +13,15 @@ class NFTables:
     self.nft.set_handle_output(True)
     self.nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
                             'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
+    self.mailcow_chains = set()
 
     self.search_current_chains()
 
   def initChainIPv4(self):
-    self.insert_mailcow_chains("ip")
+    self.init_chain("ip")
 
   def initChainIPv6(self):
-    self.insert_mailcow_chains("ip6")
-
-  def checkIPv4ChainOrder(self):
-    return self.checkChainOrder("ip")
-
-  def checkIPv6ChainOrder(self):
-    return self.checkChainOrder("ip6")
-
-  def checkChainOrder(self, filter_table):
-    err = False
-    exit_code = None
-
-    for chain in ['input', 'forward']:
-      chain_position = self.check_mailcow_chains(filter_table, chain)
-      if chain_position is None: continue
-
-      if chain_position is False:
-        self.logger.logCrit(f'MAILCOW target not found in {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-      if chain_position > 0:
-        chain_position += 1
-        self.logger.logCrit(f'MAILCOW target is in position {chain_position} in the {filter_table} {chain} table, restarting container to fix it...')
-        err = True
-        exit_code = 2
-
-    return err, exit_code
+    self.init_chain("ip6")
 
   def clearIPv4Table(self):
     self.clearTable("ip")
@@ -54,84 +29,19 @@ class NFTables:
   def clearIPv6Table(self):
     self.clearTable("ip6")
 
-  def clearTable(self, _family):
-    is_empty_dict = True
-    json_command = self.get_base_dict()
-    chain_handle = self.get_chain_handle(_family, "filter", self.chain_name)
-    # if no handle, the chain doesn't exists
-    if chain_handle is not None:
-      is_empty_dict = False
-      # flush chain
-      mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-      flush_chain = {'flush': {'chain': mailcow_chain}}
-      json_command["nftables"].append(flush_chain)
-
-    # remove rule in forward chain
-    # remove rule in input chain
-    chains_family = [self.nft_chain_names[_family]['filter']['input'],
-                    self.nft_chain_names[_family]['filter']['forward'] ]
-
-    for chain_base in chains_family:
-      if not chain_base: continue
-
-      rules_handle = self.get_rules_handle(_family, "filter", chain_base)
-      if rules_handle is not None:
-        for r_handle in rules_handle:
-          is_empty_dict = False
-          mailcow_rule = {'family':_family,
-                          'table': 'filter',
-                          'chain': chain_base,
-                          'handle': r_handle }
-          delete_rules = {'delete': {'rule': mailcow_rule} }
-          json_command["nftables"].append(delete_rules)
-
-    # remove chain
-    # after delete all rules referencing this chain
-    if chain_handle is not None:
-      mc_chain_handle = {'family':_family,
-                        'table': 'filter',
-                        'name': self.chain_name,
-                        'handle': chain_handle }
-      delete_chain = {'delete': {'chain': mc_chain_handle} }
-      json_command["nftables"].append(delete_chain)
-
-    if is_empty_dict == False:
-      if self.nft_exec_dict(json_command):
-        self.logger.logInfo(f"Clear completed: {_family}")
-
-  def banIPv4(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip")
-    return self.nft_exec_dict(ban_dict)
-
-  def banIPv6(self, source):
-    ban_dict = self.get_ban_ip_dict(source, "ip6")
-    return self.nft_exec_dict(ban_dict)
-
-  def unbanIPv4(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
-  def unbanIPv6(self, source):
-    unban_dict = self.get_unban_ip_dict(source, "ip6")
-    if not unban_dict:
-      return False
-    return self.nft_exec_dict(unban_dict)
-
   def snat4(self, snat_target, source):
     self.snat_rule("ip", snat_target, source)
 
   def snat6(self, snat_target, source):
     self.snat_rule("ip6", snat_target, source)
 
-
-  def nft_exec_dict(self, query: dict):
+  def nft_exec_dict(self, query: dict, quiet: bool = False):
     if not query: return False
 
     rc, output, error = self.nft.json_cmd(query)
     if rc != 0:
-      #self.logger.logCrit(f"Nftables Error: {error}")
+      if not quiet:
+        self.logger.logWarn(f"nftables: {str(error).strip() or f'error {rc}'}")
       return False
 
     # Prevent returning False or empty string on commands that do not produce output
@@ -144,118 +54,165 @@ class NFTables:
     return {'nftables': [{ 'metainfo': { 'json_schema_version': 1} } ] }
 
   def search_current_chains(self):
+    nft_chain_names = {'ip': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} },
+                       'ip6': {'filter': {'input': '', 'forward': ''}, 'nat': {'postrouting': ''} } }
     nft_chain_priority = {'ip': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} },
-                      'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+                          'ip6': {'filter': {'input': None, 'forward': None}, 'nat': {'postrouting': None} } }
+    mailcow_chains = set()
 
     # Command: 'nft list chains'
     _list = {'list' : {'chains': 'null'} }
     command = self.get_base_dict()
     command['nftables'].append(_list)
     kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset['nftables']:
-        chain = _object.get("chain")
-        if not chain: continue
+    if not kernel_ruleset:
+      return
 
-        _family = chain['family']
-        _table = chain['table']
-        _hook = chain.get("hook")
-        _priority = chain.get("prio")
-        _name = chain['name']
-
-        if _family not in self.nft_chain_names: continue
-        if _table not in self.nft_chain_names[_family]: continue
-        if _hook not in self.nft_chain_names[_family][_table]: continue
-        if _priority is None: continue
-
-        _saved_priority = nft_chain_priority[_family][_table][_hook]
-        if _saved_priority is None or _priority < _saved_priority:
-          # at this point, we know the chain has:
-          # hook and priority set
-          # and it has the lowest priority
-          nft_chain_priority[_family][_table][_hook] = _priority
-          self.nft_chain_names[_family][_table][_hook] = _name
-
-  def search_for_chain(self, kernel_ruleset: dict, chain_name: str):
-    found = False
-    for _object in kernel_ruleset["nftables"]:
+    for _object in kernel_ruleset['nftables']:
       chain = _object.get("chain")
-      if not chain:
-        continue
-      ch_name = chain.get("name")
-      if ch_name == chain_name:
-        found = True
-        break
-    return found
+      if not chain: continue
+
+      _family = chain['family']
+      _table = chain['table']
+      _hook = chain.get("hook")
+      _priority = chain.get("prio")
+      _name = chain['name']
+
+      if _family not in nft_chain_names: continue
+      if _table not in nft_chain_names[_family]: continue
+      if _table == 'filter' and _name == self.chain_name:
+        mailcow_chains.add(_family)
+      if _hook not in nft_chain_names[_family][_table]: continue
+      if _priority is None: continue
+
+      _saved_priority = nft_chain_priority[_family][_table][_hook]
+      if _saved_priority is None or _priority < _saved_priority:
+        # at this point, we know the chain has:
+        # hook and priority set
+        # and it has the lowest priority
+        nft_chain_priority[_family][_table][_hook] = _priority
+        nft_chain_names[_family][_table][_hook] = _name
+
+    self.nft_chain_names = nft_chain_names
+    self.mailcow_chains = mailcow_chains
+
+  def has_filter_chains(self, _family: str):
+    # no base chains: the filter table does not exist, e.g. ip6 if docker has ipv6 disabled
+    return any(self.nft_chain_names[_family]['filter'].values())
+
+  def list_rules(self, _family: str, _table: str, _chain: str):
+    # Command: 'nft list chain {family} {table} {chain}', None if the chain does not exist
+    _list = {'list': {'chain': {'family': _family, 'table': _table, 'name': _chain}}}
+    command = self.get_base_dict()
+    command['nftables'].append(_list)
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return None
+    return [_object["rule"] for _object in kernel_ruleset["nftables"] if _object.get("rule")]
 
   def get_chain_dict(self, _family: str, _name: str):
-    # nft (add | create) chain [<family>] <table> <name> 
+    # nft (add | create) chain [<family>] <table> <name>
     _chain_opts = {'family': _family, 'table': 'filter', 'name': _name  }
     _add = {'add': {'chain': _chain_opts} }
     final_chain = self.get_base_dict()
     final_chain["nftables"].append(_add)
     return final_chain
 
-  def get_mailcow_jump_rule_dict(self, _family: str, _chain: str):
-    _jump_rule = self.get_base_dict()
-    _expr_opt=[]
-    _expr_counter = {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}
-    _counter_dict = {'counter': _expr_counter}
-    _expr_opt.append(_counter_dict)
-
-    _jump_opts = {'jump': {'target': self.chain_name} }
-
-    _expr_opt.append(_jump_opts)
-
+  def get_mailcow_jump_rule(self, _family: str, _chain: str):
+    _expr_opt = [{'counter': {'family': _family, 'table': 'filter', 'packets': 0, 'bytes': 0}},
+                 {'jump': {'target': self.chain_name}}]
     _rule_params = {'family': _family,
                     'table': 'filter',
                     'chain': _chain,
                     'expr': _expr_opt,
                     'comment': "mailcow" }
+    return {'insert': {'rule': _rule_params} }
 
-    _add_rule = {'insert': {'rule': _rule_params} }
+  def is_mailcow_jump(self, rule: dict):
+    # by target, jumps restored by iptables-restore carry no nft comment
+    return any(isinstance(expr, dict) and isinstance(expr.get('jump'), dict) and expr['jump'].get('target') == self.chain_name
+               for expr in rule.get('expr', []))
 
-    _jump_rule["nftables"].append(_add_rule)
+  def init_chain(self, _family: str):
+    # Empty MAILCOW chain with one jump at the top of the input and forward chains
+    if not self.has_filter_chains(_family): return
 
-    return _jump_rule
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+        self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    else:
+      flush_chain = {'flush': {'chain': {'family': _family, 'table': 'filter', 'name': self.chain_name}}}
+      command = self.get_base_dict()
+      command['nftables'].append(flush_chain)
+      self.nft_exec_dict(command)
 
-  def insert_mailcow_chains(self, _family: str):
-    nft_input_chain = self.nft_chain_names[_family]['filter']['input']
-    nft_forward_chain = self.nft_chain_names[_family]['filter']['forward']
-    # Command: 'nft list table <family> filter'
-    _table_opts = {'family': _family, 'name': 'filter'}
-    _list = {'list': {'table': _table_opts} }
+    self.ensure_jumps(_family)
+
+  def ensure_chains(self):
+    # Recreates missing MAILCOW chains and moves the jumps back to the top, True if a chain was recreated
+    self.search_current_chains()
+    recreated = False
+    for _family in ['ip', 'ip6']:
+      if not self.has_filter_chains(_family): continue
+
+      if _family not in self.mailcow_chains:
+        if self.nft_exec_dict(self.get_chain_dict(_family, self.chain_name)):
+          self.logger.logWarn(f"MAILCOW {_family} chain was missing and has been recreated")
+          recreated = True
+      self.ensure_jumps(_family)
+    return recreated
+
+  def ensure_jumps(self, _family: str):
+    # Exactly one jump to MAILCOW as first rule: docker inserts its jumps (DOCKER-USER, DOCKER-FORWARD) at the top
+    for hook in ['input', 'forward']:
+      base_chain = self.nft_chain_names[_family]['filter'][hook]
+      if not base_chain: continue
+
+      rules = self.list_rules(_family, 'filter', base_chain)
+      if rules is None: continue
+
+      positions = [position for position, rule in enumerate(rules) if self.is_mailcow_jump(rule)]
+      if positions == [0]: continue
+
+      # delete and insert in one transaction, the chain is never without jump
+      command = self.get_base_dict()
+      for position in positions:
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': base_chain, 'handle': rules[position]['handle']}
+        command['nftables'].append({'delete': {'rule': _rule_opts}})
+      command['nftables'].append(self.get_mailcow_jump_rule(_family, base_chain))
+      if self.nft_exec_dict(command):
+        if positions:
+          self.logger.logInfo(f"Moved MAILCOW jump to the top of the {_family} {base_chain} chain, it was at position {positions[0] + 1}")
+        else:
+          self.logger.logInfo(f"Added MAILCOW jump to the {_family} {base_chain} chain")
+
+  def clearTable(self, _family: str):
+    # Jumps first, the chain can only be deleted without references. Separate steps, so one failing step
+    # does not roll back the others
+    _list = {'list': {'table': {'family': _family, 'name': 'filter'}}}
     command = self.get_base_dict()
     command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      # chain
-      if not self.search_for_chain(kernel_ruleset, self.chain_name):
-        cadena = self.get_chain_dict(_family, self.chain_name)
-        if self.nft_exec_dict(cadena):
-          self.logger.logInfo(f"MAILCOW {_family} chain created successfully.")
+    kernel_ruleset = self.nft_exec_dict(command, quiet=True)
+    if not kernel_ruleset or not isinstance(kernel_ruleset, dict):
+      return
 
-      input_jump_found, forward_jump_found = False, False
+    delete_jumps = self.get_base_dict()
+    for _object in kernel_ruleset["nftables"]:
+      rule = _object.get("rule")
+      if rule and self.is_mailcow_jump(rule):
+        _rule_opts = {'family': _family, 'table': 'filter', 'chain': rule['chain'], 'handle': rule['handle']}
+        delete_jumps['nftables'].append({'delete': {'rule': _rule_opts}})
+    if len(delete_jumps['nftables']) > 1:
+      self.nft_exec_dict(delete_jumps)
 
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if nft_input_chain and rule["chain"] == nft_input_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            input_jump_found = True
-        if nft_forward_chain and rule["chain"] == nft_forward_chain:
-          if rule.get("comment") and rule["comment"] == "mailcow":
-            forward_jump_found = True
-
-      if not input_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_input_chain)
-        self.nft_exec_dict(command)
-
-      if not forward_jump_found:
-        command = self.get_mailcow_jump_rule_dict(_family, nft_forward_chain)
-        self.nft_exec_dict(command)
+    if self.list_rules(_family, 'filter', self.chain_name) is None:
+      return
+    mailcow_chain = {'family': _family, 'table': 'filter', 'name': self.chain_name}
+    command = self.get_base_dict()
+    command['nftables'].append({'flush': {'chain': mailcow_chain}})
+    command['nftables'].append({'delete': {'chain': mailcow_chain}})
+    if self.nft_exec_dict(command):
+      self.logger.logInfo(f"Clear completed: {_family}")
 
   def delete_nat_rule(self, _family:str, _chain: str, _handle:str):
     delete_command = self.get_base_dict()
@@ -285,23 +242,14 @@ class NFTables:
     # no postrouting chain, may occur if docker has ipv6 disabled.
     if not chain_name: return
 
-    # Command: nft list chain <family> nat <chain_name>
-    _chain_opts = {'family': _family, 'table': 'nat', 'name': chain_name}
-    _list = {'list':{'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if not kernel_ruleset:
+    rules = self.list_rules(_family, 'nat', chain_name)
+    if rules is None:
       return
 
     rule_position = 0
     rule_handle = None
     rule_found = False
-    for _object in kernel_ruleset["nftables"]:
-      if not _object.get("rule"):
-        continue
-
-      rule = _object["rule"]
+    for rule in rules:
       if not rule.get("comment") or not rule["comment"] == "mailcow":
         rule_position +=1
         continue
@@ -314,23 +262,28 @@ class NFTables:
     target_net = ipaddress.ip_network(snat_target, strict=False)
 
     if rule_found:
-      saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
-      saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
+      try:
+        saddr_ip = rule["expr"][0]["match"]["right"]["prefix"]["addr"]
+        saddr_len = int(rule["expr"][0]["match"]["right"]["prefix"]["len"])
 
-      daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
-      daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
+        daddr_ip = rule["expr"][1]["match"]["right"]["prefix"]["addr"]
+        daddr_len = int(rule["expr"][1]["match"]["right"]["prefix"]["len"])
 
-      target_ip = rule["expr"][3]["snat"]["addr"]
+        target_ip = rule["expr"][3]["snat"]["addr"]
 
-      saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
-      daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
-      current_target_net = ipaddress.ip_network(target_ip, strict=False)
+        saddr_net = ipaddress.ip_network(saddr_ip + '/' + str(saddr_len), strict=False)
+        daddr_net = ipaddress.ip_network(daddr_ip + '/' + str(daddr_len), strict=False)
+        current_target_net = ipaddress.ip_network(target_ip, strict=False)
 
-      match = all((
-                dest_net == saddr_net,
-                dest_net == daddr_net,
-                target_net == current_target_net
-              ))
+        match = all((
+                  dest_net == saddr_net,
+                  dest_net == daddr_net,
+                  target_net == current_target_net
+                ))
+      except (KeyError, IndexError, TypeError, ValueError):
+        # rule of another format, replace it
+        saddr_net = dest_net
+        match = False
       try:
         if rule_position == 0:
           if not match:
@@ -377,49 +330,13 @@ class NFTables:
       except:
         self.logger.logCrit(f"Error running SNAT on {_family}, retrying...")
 
-  def get_chain_handle(self, _family: str, _table: str, chain_name: str):
-    chain_handle = None
-    # Command: 'nft list chains {family}'
-    _list = {'list': {'chains': {'family': _family} } }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("chain"):
-          continue
-        chain = _object["chain"]
-        if chain["family"] == _family and chain["table"] == _table and chain["name"] == chain_name:
-          chain_handle = chain["handle"]
-          break
-    return chain_handle
-
   def get_rules_handle(self, _family: str, _table: str, chain_name: str, _comment_filter = "mailcow"):
-    rule_handle = []
-    # Command: 'nft list chain {family} {table} {chain_name}'
-    _chain_opts = {'family': _family, 'table': _table, 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-
-        rule = _object["rule"]
-        if rule["family"] == _family and rule["table"] == _table and rule["chain"] == chain_name:
-          if rule.get("comment") and rule["comment"] == _comment_filter:
-            rule_handle.append(rule["handle"])
-    return rule_handle
-
-  def get_ban_ip_dict(self, ipaddr: str, _family: str):
-    json_command = self.get_base_dict()
+    rules = self.list_rules(_family, _table, chain_name) or []
+    return [rule["handle"] for rule in rules if rule.get("comment") == _comment_filter]
 
+  def get_ban_rule(self, network, _family: str):
     expr_opt = []
-    ipaddr_net = ipaddress.ip_network(ipaddr, strict=False)
-    right_dict = {'prefix': {'addr': str(ipaddr_net.network_address), 'len': int(ipaddr_net.prefixlen) } }
+    right_dict = {'prefix': {'addr': str(network.network_address), 'len': int(network.prefixlen) } }
 
     left_dict = {'payload': {'protocol': _family, 'field': 'saddr'} }
     match_dict = {'op': '==', 'left': left_dict, 'right': right_dict }
@@ -432,228 +349,115 @@ class NFTables:
 
     rule_dict = {'family': _family, 'table': "filter", 'chain': self.chain_name, 'expr': expr_opt}
 
-    base_dict = {'insert': {'rule': rule_dict} }
-    json_command["nftables"].append(base_dict)
+    return {'insert': {'rule': rule_dict} }
 
-    return json_command
+  def ban_rule_network(self, rule: dict, _family: str):
+    # Source network of a ban rule (saddr match and drop), None for every other rule
+    if rule.get("comment") in self.ISOLATION_COMMENTS:
+      return None
+    exprs = [expr for expr in rule.get("expr", []) if isinstance(expr, dict)]
+    if not any('drop' in expr or 'reject' in expr for expr in exprs):
+      return None
+    for expr in exprs:
+      match = expr.get("match")
+      if not isinstance(match, dict) or match.get("op") != "==":
+        continue
+      left = match.get("left")
+      payload = left.get("payload") if isinstance(left, dict) else None
+      if not isinstance(payload, dict) or payload.get("protocol") != _family or payload.get("field") != "saddr":
+        continue
+      right = match.get("right")
+      try:
+        if isinstance(right, dict) and isinstance(right.get("prefix"), dict):
+          return ipaddress.ip_network(f'{right["prefix"]["addr"]}/{right["prefix"]["len"]}', strict=False)
+        if isinstance(right, str):
+          return ipaddress.ip_network(right, strict=False)
+      except (KeyError, ValueError):
+        return None
+    return None
 
-  def get_unban_ip_dict(self, ipaddr:str, _family: str):
-    json_command = self.get_base_dict()
-    # Command: 'nft list chain {s_family} filter  MAILCOW'
-    _chain_opts = {'family': _family, 'table': 'filter', 'name': self.chain_name}
-    _list = {'list': {'chain': _chain_opts} }
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    rule_handle = None
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
+  def split_families(self, sources):
+    families = {}
+    for source in sources:
+      network = ipaddress.ip_network(source, strict=False)
+      families.setdefault('ip' if network.version == 4 else 'ip6', []).append(network)
+    return families
 
-        rule = _object["rule"]["expr"][0]["match"]
-        if not "payload" in rule["left"]:
-          continue
-        left_opt = rule["left"]["payload"]
-        if not left_opt["protocol"] == _family:
-          continue
-        if not left_opt["field"] =="saddr":
-          continue
+  def ban(self, sources):
+    # One drop rule per network, existing rules are kept. Returns the networks that could not be banned
+    failed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if rules is None:
+        failed += networks
+        continue
+      existing = {self.ban_rule_network(rule, _family) for rule in rules}
+      missing = [network for network in dict.fromkeys(networks) if network not in existing]
+      if not missing: continue
 
-        # ip currently banned
-        rule_right = rule["right"]
-        if isinstance(rule_right, dict):
-          current_rule_ip = rule_right["prefix"]["addr"] + '/' + str(rule_right["prefix"]["len"])
-        else:
-          current_rule_ip = rule_right
-        current_rule_net = ipaddress.ip_network(current_rule_ip)
+      command = self.get_base_dict()
+      for network in missing:
+        command['nftables'].append(self.get_ban_rule(network, _family))
+      if not self.nft_exec_dict(command):
+        failed += missing
+    return [str(network) for network in failed]
 
-        # ip to ban
-        candidate_net = ipaddress.ip_network(ipaddr, strict=False)
+  def unban(self, sources):
+    # Removes every rule of the networks, also duplicates of older versions. Returns the unbanned networks
+    removed = []
+    for _family, networks in self.split_families(sources).items():
+      rules = self.list_rules(_family, 'filter', self.chain_name)
+      if not rules: continue
 
-        if current_rule_net == candidate_net:
-          rule_handle = _object["rule"]["handle"]
-          break
-
-      if rule_handle is not None:
-        mailcow_rule = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule_handle}
-        delete_rule = {'delete': {'rule': mailcow_rule} }
-        json_command["nftables"].append(delete_rule)
-      else:
-        return False
-
-    return json_command
-
-  def check_mailcow_chains(self, family: str, chain: str):
-    position = 0
-    rule_found = False
-    chain_name = self.nft_chain_names[family]['filter'][chain]
-
-    if not chain_name: return None
-
-    _chain_opts = {'family': family, 'table': 'filter', 'name': chain_name}
-    _list = {'list': {'chain': _chain_opts}}
-    command = self.get_base_dict()
-    command['nftables'].append(_list)
-    kernel_ruleset = self.nft_exec_dict(command)
-    if kernel_ruleset:
-      for _object in kernel_ruleset["nftables"]:
-        if not _object.get("rule"):
-          continue
-        rule = _object["rule"]
-        if rule.get("comment") and rule["comment"] == "mailcow":
-          rule_found = True
-          break
-
-        position+=1
-
-    return position if rule_found else False
+      targets = set(networks)
+      command = self.get_base_dict()
+      found = set()
+      for rule in rules:
+        network = self.ban_rule_network(rule, _family)
+        if network in targets:
+          _rule_opts = {'family': _family, 'table': 'filter', 'chain': self.chain_name, 'handle': rule['handle']}
+          command['nftables'].append({'delete': {'rule': _rule_opts}})
+          found.add(network)
+      if found and self.nft_exec_dict(command):
+        removed += found
+    return [str(network) for network in removed]
 
   def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
-    family = "ip"
-    table = "filter"
-    comment_filter_drop = "mailcow isolation"
-    comment_filter_allow = "mailcow isolation allow"
-    json_command = self.get_base_dict()
+    # One rule per port: iptables-nft cannot show an anonymous set, a host that saves and restores the ruleset
+    # with iptables-save (e.g. netfilter-persistent) would get a drop for all ports
+    for comment in self.ISOLATION_COMMENTS:
+      for handle in self.get_rules_handle("ip", "filter", self.chain_name, comment):
+        self.delete_filter_rule("ip", self.chain_name, handle)
 
-    # Delete old mailcow isolation rules
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_drop)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-    handles = self.get_rules_handle(family, table, self.chain_name, comment_filter_allow)
-    for handle in handles:
-      self.delete_filter_rule(family, self.chain_name, handle)
-
-    # insert mailcow isolation rule
-    _match_dict_drop = [
-      {
-        "match": {
-          "op": "!=",
-          "left": {
-            "meta": {
-              "key": "iifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "meta": {
-              "key": "oifname"
-            }
-          },
-          "right": _interface
-        }
-      },
-      {
-        "match": {
-          "op": "==",
-          "left": {
-            "payload": {
-              "protocol": "tcp",
-              "field": "dport"
-            }
-          },
-          "right": {
-            "set": _dports
-          }
-        }
-      },
-      {
-        "counter": {
-          "packets": 0,
-          "bytes": 0
-        }
-      },
-      {
-        "drop": None
-      }
-    ]
-    rule_drop = { "insert": { "rule": {
-      "family": family,
-      "table": table,
-      "chain": self.chain_name,
-      "comment": comment_filter_drop,
-      "expr": _match_dict_drop
-    }}}
-    json_command["nftables"].append(rule_drop)
-
-    # insert mailcow isolation allow rule
-    if _allow != "":
-      _match_dict_allow = [
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "ip",
-                "field": "saddr"
-              }
-            },
-            "right": _allow
-          }
-        },
-        {
-          "match": {
-            "op": "!=",
-            "left": {
-              "meta": {
-                "key": "iifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "meta": {
-                "key": "oifname"
-              }
-            },
-            "right": _interface
-          }
-        },
-        {
-          "match": {
-            "op": "==",
-            "left": {
-              "payload": {
-                "protocol": "tcp",
-                "field": "dport"
-              }
-            },
-            "right": {
-              "set": _dports
-            }
-          }
-        },
-        {
-          "counter": {
-            "packets": 0,
-            "bytes": 0
-          }
-        },
-        {
-          "accept": None
-        }
-      ]
-      rule_allow = { "insert": { "rule": {
-        "family": family,
-        "table": table,
-        "chain": self.chain_name,
-        "comment": comment_filter_allow,
-        "expr": _match_dict_allow
-      }}}
-      json_command["nftables"].append(rule_allow)
-
-    success = self.nft_exec_dict(json_command)
-    if success == False:
+    if not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[0], {"drop": None})):
       self.logger.logCrit(f"Error adding {self.chain_name} isolation")
       return False
 
-    return True
+    # separate, an invalid address must not prevent the drop rules. Inserted last, so they are above them
+    if _allow and not self.nft_exec_dict(self.get_isolation_dict(_interface, _dports, self.ISOLATION_COMMENTS[1], {"accept": None}, _allow)):
+      self.logger.logCrit(f"Error adding {self.chain_name} isolation exception for {_allow}")
+      return False
+
+    return True
+
+  def get_isolation_dict(self, _interface:str, _dports:list, _comment:str, _verdict:dict, _saddr:str = ""):
+    json_command = self.get_base_dict()
+    for port in _dports:
+      expr = []
+      if _saddr:
+        expr.append({"match": {"op": "==", "left": {"payload": {"protocol": "ip", "field": "saddr"}}, "right": _saddr}})
+      expr += [
+        {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"meta": {"key": "oifname"}}, "right": _interface}},
+        {"match": {"op": "==", "left": {"payload": {"protocol": "tcp", "field": "dport"}}, "right": int(port)}},
+        {"counter": {"packets": 0, "bytes": 0}},
+        _verdict
+      ]
+      json_command["nftables"].append({"insert": {"rule": {
+        "family": "ip",
+        "table": "filter",
+        "chain": self.chain_name,
+        "comment": _comment,
+        "expr": expr
+      }}})
+    return json_command

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -0,0 +1,200 @@
+import ipaddress
+import threading
+import dns.exception
+import dns.resolver
+from modules.BanManager import BanManager
+from modules.Options import Options
+from modules.Regexes import Regexes
+
+
+class Netfilter:
+  """Runs the workers of netfilter. Every worker is a loop that logs errors and goes on, so neither a broken log
+  line nor a Redis restart can stop the expiry of bans."""
+
+  ISOLATION_INTERFACE = 'br-mailcow'
+  ISOLATION_PORTS = [3306, 6379, 8983, 12345]
+
+  def __init__(self, r, tables, logger, env):
+    self.r = r
+    self.tables = tables
+    self.logger = logger
+    self.env = env
+    # reentrant, bans.reapply() runs within the chain check
+    self.lock = threading.RLock()
+    self.stop = threading.Event()
+    self.options = Options(logger)
+    self.regexes = Regexes(logger)
+    self.bans = BanManager(r, tables, self.options, logger, self.lock)
+    self.snat4_target = self.snat_target('SNAT_TO_SOURCE', 4)
+    self.snat6_target = self.snat_target('SNAT6_TO_SOURCE', 6)
+    self.resolved = {}
+    self.workers = {
+      'watch': (self.watch, 3),
+      'purge': (self.purge, 10),
+      'chains': (self.check_chains, 10),
+      'lists': (self.update_lists, 60),
+    }
+    if self.snat4_target or self.snat6_target:
+      self.workers['snat'] = (self.snat, 10)
+    self.threads = {}
+
+  def setup(self):
+    if self.r.exists('F2B_LOG'):
+      self.r.rename('F2B_LOG', 'NETFILTER_LOG')
+    # filled again by the lists worker
+    self.r.delete('F2B_PERM_BANS')
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    with self.lock:
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+      self.logger.logInfo('Initializing mailcow netfilter chain')
+      self.tables.initChainIPv4()
+      self.tables.initChainIPv6()
+      self.isolate()
+    self.bans.restore()
+
+  def run(self):
+    for name in self.workers:
+      self.start(name)
+    while not self.stop.wait(5):
+      for name, thread in list(self.threads.items()):
+        if not thread.is_alive():
+          self.logger.logCrit('Worker %s stopped unexpectedly, restarting it' % name)
+          self.start(name)
+
+  def shutdown(self):
+    """Removes the firewall rules, the active bans stay in Redis and are restored on the next start"""
+    self.stop.set()
+    for thread in self.threads.values():
+      thread.join(timeout=5)
+    with self.lock:
+      self.logger.logInfo('Clearing all bans')
+      self.tables.clearIPv4Table()
+      self.tables.clearIPv6Table()
+    try:
+      self.r.delete('F2B_PERM_BANS')
+    except Exception as ex:
+      self.logger.logWarn('Error clearing redis key F2B_PERM_BANS: %s' % ex)
+
+  def start(self, name):
+    task, interval = self.workers[name]
+    thread = threading.Thread(target=self.loop, args=(name, task, interval), name=name, daemon=True)
+    self.threads[name] = thread
+    thread.start()
+
+  def loop(self, name, task, interval):
+    while not self.stop.is_set():
+      try:
+        task()
+      except Exception as ex:
+        self.logger.logWarn('Error in worker %s: %s' % (name, ex))
+      self.stop.wait(interval)
+
+  def watch(self):
+    pubsub = self.r.pubsub(ignore_subscribe_messages=True)
+    try:
+      pubsub.subscribe('F2B_CHANNEL')
+      self.logger.logInfo('Watching Redis channel F2B_CHANNEL')
+      while not self.stop.is_set():
+        message = pubsub.get_message(timeout=1.0)
+        if message and message['type'] == 'message' and message['data']:
+          self.handle_line(message['data'])
+    finally:
+      try:
+        pubsub.close()
+      except Exception:
+        pass
+
+  def handle_line(self, line):
+    # a broken line must not end the watcher
+    try:
+      excused = Regexes.temporary_failure(line)
+      if excused is not None:
+        ip = Regexes.parse_ip(excused)
+        if self.external(ip):
+          self.bans.excuse(ip)
+        return
+      result = self.regexes.match(line)
+      if result is None:
+        return
+      rule_id, address = result
+      ip = Regexes.parse_ip(address)
+      if ip is None:
+        self.logger.logWarn('Rule id %s matched an invalid address %s (%s)' % (rule_id, address, line))
+        return
+      if not self.external(ip):
+        return
+      self.logger.logWarn('%s matched rule id %s (%s)' % (ip, rule_id, line))
+      self.bans.attempt(ip)
+    except Exception as ex:
+      self.logger.logWarn('Error processing log line %r: %s' % (line, ex))
+
+  @staticmethod
+  def external(ip):
+    # private and loopback addresses are never banned
+    return ip is not None and not (ip.is_private or ip.is_loopback)
+
+  def purge(self):
+    self.options.refresh(self.r)
+    self.regexes.refresh(self.r)
+    self.bans.purge()
+
+  def check_chains(self):
+    with self.lock:
+      if self.tables.ensure_chains():
+        self.isolate()
+        self.bans.reapply()
+
+  def snat(self):
+    with self.lock:
+      if self.snat4_target:
+        self.tables.snat4(self.snat4_target, self.env.get('IPV4_NETWORK', '172.22.1') + '.0/24')
+      if self.snat6_target:
+        self.tables.snat6(self.snat6_target, self.env.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
+
+  def update_lists(self):
+    self.bans.set_allowlist(self.resolve(self.r.hkeys('F2B_WHITELIST')))
+    self.bans.set_denylist(self.resolve(self.r.hkeys('F2B_BLACKLIST')))
+
+  def isolate(self):
+    if self.env.get('DISABLE_NETFILTER_ISOLATION_RULE', '').lower() in ('y', 'yes'):
+      self.logger.logInfo('Skipping MAILCOW isolation')
+      return
+    self.logger.logInfo('Setting MAILCOW isolation')
+    self.tables.create_mailcow_isolation_rule(self.ISOLATION_INTERFACE, self.ISOLATION_PORTS, self.env.get('MAILCOW_REPLICA_IP', ''))
+
+  def resolve(self, entries):
+    """Networks of the entries; hostnames are resolved, on DNS errors to their last known addresses"""
+    resolver = dns.resolver.Resolver()
+    networks = set()
+    for entry in entries:
+      if BanManager.network(entry) is not None:
+        networks.add(entry)
+        continue
+      addresses = set()
+      for rdtype in ['A', 'AAAA']:
+        try:
+          answer = resolver.resolve(qname=entry, rdtype=rdtype, lifetime=3)
+        except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+          continue
+        except dns.exception.DNSException as ex:
+          self.logger.logInfo('Could not resolve %s: %s' % (entry, ex))
+          addresses = self.resolved.get(entry, set())
+          break
+        addresses.update(rdata.to_text() for rdata in answer)
+      self.resolved[entry] = addresses
+      networks |= addresses
+    return networks
+
+  def snat_target(self, name, version):
+    value = self.env.get(name, '')
+    if not value or value == 'n':
+      return None
+    try:
+      if ipaddress.ip_address(value).version == version:
+        return value
+    except ValueError:
+      pass
+    self.logger.logWarn('%s is not a valid IPv%d address' % (value, version))
+    return None

data/Dockerfiles/netfilter/modules/Options.py

@@ -0,0 +1,120 @@
+import json
+import uuid
+
+
+class Options:
+  """Fail2ban options of the mailcow UI (Redis F2B_OPTIONS). The values are replaced as a whole and never changed
+  in place, so a thread that took them (options.values) keeps a consistent set."""
+
+  DEFAULTS = {
+    'ban_time': 1800,
+    'max_ban_time': 10000,
+    'ban_time_increment': True,
+    'max_attempts': 10,
+    'retry_window': 600,
+    'netban_ipv4': 32,
+    'netban_ipv6': 128,
+    'manage_external': 0,
+  }
+  # Single keys of old installations, read while F2B_OPTIONS does not exist
+  LEGACY_KEYS = {
+    'ban_time': 'F2B_BAN_TIME',
+    'max_ban_time': 'F2B_MAX_BAN_TIME',
+    'ban_time_increment': 'F2B_BAN_TIME_INCREMENT',
+    'max_attempts': 'F2B_MAX_ATTEMPTS',
+    'retry_window': 'F2B_RETRY_WINDOW',
+    'netban_ipv4': 'F2B_NETBAN_IPV4',
+    'netban_ipv6': 'F2B_NETBAN_IPV6',
+  }
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.values = self.normalize({})
+
+  def refresh(self, r):
+    """Reads the options of the UI; options it lacks are added to Redis once, existing ones are never overwritten"""
+    raw = r.get('F2B_OPTIONS')
+    if raw:
+      try:
+        stored = json.loads(raw)
+      except ValueError as ex:
+        self.logger.logWarn('F2B_OPTIONS is no valid JSON, keeping the current options: %s' % ex)
+        return
+      if not isinstance(stored, dict):
+        self.logger.logWarn('F2B_OPTIONS is no JSON object, keeping the current options')
+        return
+    else:
+      stored = {key: r.get(name) for key, name in self.LEGACY_KEYS.items()}
+      stored = {key: value for key, value in stored.items() if value is not None}
+
+    values = self.normalize(stored)
+    self.values = values
+    # First start or old installation: store the complete set, else only what the UI has not saved yet
+    missing = {key: value for key, value in values.items() if not raw or stored.get(key) is None}
+    if missing:
+      self.add_missing(r, missing)
+
+  def ban_time(self, ban_counter):
+    """Seconds of the next ban of a network that was banned ban_counter times before"""
+    values = self.values
+    ban_time = values['ban_time']
+    if values['ban_time_increment']:
+      # the exponent is capped, max_ban_time limits the result anyway
+      ban_time = ban_time * 2 ** min(ban_counter, 32)
+    return max(values['ban_time'], min(ban_time, values['max_ban_time']))
+
+  @classmethod
+  def normalize(cls, stored):
+    values = dict(stored)
+    for key, default in cls.DEFAULTS.items():
+      if isinstance(default, bool):
+        values[key] = cls.to_bool(stored.get(key), default)
+      else:
+        values[key] = cls.to_int(stored.get(key), default)
+    values['ban_time'] = max(1, values['ban_time'])
+    values['max_ban_time'] = max(1, values['max_ban_time'])
+    values['max_attempts'] = max(1, values['max_attempts'])
+    values['retry_window'] = max(1, values['retry_window'])
+    # A short prefix would ban whole regions or everything (/0)
+    values['netban_ipv4'] = min(32, max(8, values['netban_ipv4']))
+    values['netban_ipv6'] = min(128, max(8, values['netban_ipv6']))
+    values['manage_external'] = 1 if values['manage_external'] == 1 else 0
+    if not values.get('banlist_id'):
+      values['banlist_id'] = str(uuid.uuid4())
+    return values
+
+  @staticmethod
+  def to_int(value, default):
+    try:
+      return int(value)
+    except (TypeError, ValueError):
+      return default
+
+  @staticmethod
+  def to_bool(value, default):
+    if isinstance(value, bool):
+      return value
+    if isinstance(value, (int, float)):
+      return value != 0
+    if isinstance(value, str):
+      return value.strip().lower() in ('1', 'true', 'yes', 'on')
+    return default
+
+  def add_missing(self, r, missing):
+    # WATCH keeps a save of the UI between reading and writing
+    def fill(pipe):
+      raw = pipe.get('F2B_OPTIONS')
+      current = json.loads(raw) if raw else {}
+      if not isinstance(current, dict):
+        return
+      added = {key: value for key, value in missing.items() if current.get(key) is None}
+      if not added:
+        return
+      current.update(added)
+      pipe.multi()
+      pipe.set('F2B_OPTIONS', json.dumps(current, ensure_ascii=False))
+
+    try:
+      r.transaction(fill, 'F2B_OPTIONS')
+    except Exception as ex:
+      self.logger.logWarn('Could not add default options to F2B_OPTIONS: %s' % ex)

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -0,0 +1,88 @@
+import ipaddress
+import json
+import re
+
+
+class Regexes:
+  """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
+
+  DEFAULTS = {
+    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
+    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
+    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
+    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
+    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
+    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
+    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
+    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+  }
+  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
+  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
+  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+
+  def __init__(self, logger):
+    self.logger = logger
+    self.rules = []
+    self.source = None
+
+  def refresh(self, r):
+    """Compiles the rules of the UI again when they changed; invalid rules are skipped"""
+    raw = r.get('F2B_REGEX')
+    if not raw:
+      raw = json.dumps(self.DEFAULTS, ensure_ascii=False)
+      r.set('F2B_REGEX', raw, nx=True)
+    if raw == self.source:
+      return
+    try:
+      stored = json.loads(raw)
+    except ValueError as ex:
+      self.logger.logWarn('F2B_REGEX is no valid JSON, keeping the current rules: %s' % ex)
+      return
+    if not isinstance(stored, dict):
+      self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
+      return
+
+    rules = []
+    for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
+      try:
+        rules.append((str(rule_id), re.compile(regex)))
+      except (re.error, TypeError) as ex:
+        self.logger.logWarn('Skipping regex of rule id %s: %s' % (rule_id, ex))
+    self.rules = rules
+    self.source = raw
+
+  def match(self, line):
+    """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    for rule_id, regex in self.rules:
+      result = regex.search(line)
+      if result and result.lastindex and result.group(1):
+        return rule_id, result.group(1)
+    return None
+
+  @classmethod
+  def temporary_failure(cls, line):
+    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
+    result = cls.TEMPORARY_FAILURE.search(line)
+    return result.group(1) if result else None
+
+  @staticmethod
+  def parse_ip(text):
+    """IP address of a captured address, IPv4-mapped IPv6 as IPv4; a trailing colon (e.g. "1.2.3.4:") is
+    tolerated. None if the text is no address."""
+    for candidate in (text, text.rstrip(':')):
+      try:
+        ip = ipaddress.ip_address(candidate)
+      except ValueError:
+        continue
+      if ip.version == 6 and ip.ipv4_mapped:
+        return ip.ipv4_mapped
+      return ip
+    return None
+
+  @staticmethod
+  def sort_key(rule_id):
+    try:
+      return (0, int(rule_id))
+    except (TypeError, ValueError):
+      return (1, str(rule_id))

data/Dockerfiles/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:alpine
+FROM nginx:1.30.6-alpine
 LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 ENV PIP_BREAK_SYSTEM_PACKAGES=1

data/Dockerfiles/nginx/bootstrap.py

@@ -10,7 +10,7 @@ def includes_conf(env, template_vars):
   server_name_config = f"server_name {template_vars['MAILCOW_HOSTNAME']} autodiscover.* autoconfig.* {' '.join(template_vars['ADDITIONAL_SERVER_NAMES'])};"
   listen_plain_config = f"listen {template_vars['HTTP_PORT']};"
   listen_ssl_config = f"listen {template_vars['HTTPS_PORT']};"
-  if not template_vars['DISABLE_IPv6']:
+  if template_vars['ENABLE_IPV6']:
     listen_plain_config += f"\nlisten [::]:{template_vars['HTTP_PORT']};"
     listen_ssl_config += f"\nlisten [::]:{template_vars['HTTPS_PORT']} ssl;"
   listen_ssl_config += "\nhttp2 on;"
@@ -58,7 +58,7 @@ def prepare_template_vars():
     'SOGOHOST': os.getenv("SOGOHOST", ipv4_network + ".248"),
     'RSPAMDHOST': os.getenv("RSPAMDHOST", "rspamd-mailcow"),
     'PHPFPMHOST': os.getenv("PHPFPMHOST", "php-fpm-mailcow"),
-    'DISABLE_IPv6': os.getenv("DISABLE_IPv6", "n").lower() in ("y", "yes"),
+    'ENABLE_IPV6': os.getenv("ENABLE_IPV6", "true").lower() != "false",
     'HTTP_REDIRECT': os.getenv("HTTP_REDIRECT", "n").lower() in ("y", "yes"),
   }
 

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,19 +1,19 @@
-FROM php:8.2-fpm-alpine3.21
+FROM php:8.2-fpm-alpine3.24
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
 # renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG APCU_PECL_VERSION=5.1.24
+ARG APCU_PECL_VERSION=5.1.28
 # renovate: datasource=github-tags depName=Imagick/imagick versioning=semver-coerced extractVersion=(?<version>.*)$
-ARG IMAGICK_PECL_VERSION=3.8.0
+ARG IMAGICK_PECL_VERSION=3.8.1
 # renovate: datasource=github-tags depName=php/pecl-mail-mailparse versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG MAILPARSE_PECL_VERSION=3.1.8
+ARG MAILPARSE_PECL_VERSION=3.2.0
 # renovate: datasource=github-tags depName=php-memcached-dev/php-memcached versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG MEMCACHED_PECL_VERSION=3.2.0
+ARG MEMCACHED_PECL_VERSION=3.4.0
 # renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?<version>.*)$
-ARG REDIS_PECL_VERSION=6.1.0
+ARG REDIS_PECL_VERSION=6.3.0
 # renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?<version>.*)$
-ARG COMPOSER_VERSION=2.8.6
+ARG COMPOSER_VERSION=2.10.2
 
 RUN apk add -U --no-cache autoconf \
   aspell-dev \
@@ -47,6 +47,7 @@ RUN apk add -U --no-cache autoconf \
   libressl-dev \
   librsvg \
   libtool \
+  libwebp \
   libwebp-dev \
   libxml2-dev \
   libxpm \
@@ -63,13 +64,16 @@ RUN apk add -U --no-cache autoconf \
   samba-client \
   zlib-dev \
   tzdata \
-  && pecl install APCu-${APCU_PECL_VERSION} \
-  && pecl install imagick-${IMAGICK_PECL_VERSION} \
-  && pecl install mailparse-${MAILPARSE_PECL_VERSION} \
-  && pecl install memcached-${MEMCACHED_PECL_VERSION} \
-  && pecl install redis-${REDIS_PECL_VERSION} \
-  && docker-php-ext-enable apcu imagick memcached mailparse redis \
-  && pecl clear-cache \
+  && docker-php-source extract \
+  # PECL extensions from their GitHub tags (pecl.php.net is not reliable enough for builds)
+  && mkdir -p /usr/src/php/ext/apcu /usr/src/php/ext/imagick /usr/src/php/ext/mailparse /usr/src/php/ext/memcached /usr/src/php/ext/redis \
+  && curl -fsSL https://github.com/krakjoe/apcu/archive/refs/tags/v${APCU_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/apcu \
+  && curl -fsSL https://github.com/Imagick/imagick/archive/refs/tags/${IMAGICK_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/imagick \
+  && sed -i "s/@PACKAGE_VERSION@/${IMAGICK_PECL_VERSION}/" /usr/src/php/ext/imagick/php_imagick.h \
+  && curl -fsSL https://github.com/php/pecl-mail-mailparse/archive/refs/tags/v${MAILPARSE_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/mailparse \
+  && curl -fsSL https://github.com/php-memcached-dev/php-memcached/archive/refs/tags/v${MEMCACHED_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/memcached \
+  && curl -fsSL https://github.com/phpredis/phpredis/archive/refs/tags/${REDIS_PECL_VERSION}.tar.gz | tar -xz --strip-components=1 -C /usr/src/php/ext/redis \
+  && docker-php-ext-install -j 4 apcu imagick mailparse memcached redis \
   && docker-php-ext-configure intl \
   && docker-php-ext-configure exif \
   && docker-php-ext-configure gd --with-freetype=/usr/include/ \  
@@ -83,6 +87,7 @@ RUN apk add -U --no-cache autoconf \
   && curl --silent --show-error https://getcomposer.org/installer | php -- --version=${COMPOSER_VERSION} \
   && mv composer.phar /usr/local/bin/composer \
   && chmod +x /usr/local/bin/composer \
+  && docker-php-source delete \
   && apk del --purge autoconf \
     aspell-dev \
     cyrus-sasl-dev \

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-function array_by_comma { local IFS=","; echo "$*"; }
-
 # Wait for containers
 while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for SQL..."
@@ -132,33 +130,6 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   done
   fi
 
-  # Set API options if env vars are not empty
-  if [[ ${API_ALLOW_FROM} != "invalid" ]] && [[ ! -z ${API_ALLOW_FROM} ]]; then
-    IFS=',' read -r -a API_ALLOW_FROM_ARR <<< "${API_ALLOW_FROM}"
-    declare -a VALIDATED_API_ALLOW_FROM_ARR
-    REGEX_IP6='^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}(/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$'
-    REGEX_IP4='^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(/([0-9]|[1-2][0-9]|3[0-2]))?$'
-    for IP in "${API_ALLOW_FROM_ARR[@]}"; do
-      if [[ ${IP} =~ ${REGEX_IP6} ]] || [[ ${IP} =~ ${REGEX_IP4} ]]; then
-        VALIDATED_API_ALLOW_FROM_ARR+=("${IP}")
-      fi
-    done
-    VALIDATED_IPS=$(array_by_comma ${VALIDATED_API_ALLOW_FROM_ARR[*]})
-    if [[ ! -z ${VALIDATED_IPS} ]]; then
-      if [[ ${API_KEY} != "invalid" ]] && [[ ! -z ${API_KEY} ]]; then
-        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-DELETE FROM api WHERE access = 'rw';
-INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY}", "1", "${VALIDATED_IPS}", "rw");
-EOF
-      fi
-      if [[ ${API_KEY_READ_ONLY} != "invalid" ]] && [[ ! -z ${API_KEY_READ_ONLY} ]]; then
-        mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
-DELETE FROM api WHERE access = 'ro';
-INSERT INTO api (api_key, active, allow_from, access) VALUES ("${API_KEY_READ_ONLY}", "1", "${VALIDATED_IPS}", "ro");
-EOF
-      fi
-    fi
-  fi
 
   # Create events (master only, STATUS for event on slave will be SLAVESIDE_DISABLED)
   mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} << EOF
@@ -167,7 +138,7 @@ DELIMITER //
 CREATE EVENT clean_spamalias
 ON SCHEDULE EVERY 1 DAY DO
 BEGIN
-  DELETE FROM spamalias WHERE validity < UNIX_TIMESTAMP();
+  DELETE FROM spamalias WHERE validity < UNIX_TIMESTAMP() AND permanent = 0;
 END;
 //
 DELIMITER ;

data/Dockerfiles/postfix-tlspol/Dockerfile

@@ -0,0 +1,50 @@
+FROM golang:1.26-trixie AS builder
+WORKDIR /src
+
+ENV CGO_ENABLED=0 \
+    GO111MODULE=on \
+		NOOPT=1 \
+    VERSION=1.14.0
+
+RUN git clone --branch v${VERSION} https://github.com/Zuplu/postfix-tlspol && \
+    cd /src/postfix-tlspol && \
+    scripts/build.sh build-only
+
+
+FROM debian:trixie-slim
+LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL=C
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	ca-certificates \
+	dirmngr \
+  	dnsutils \
+	iputils-ping \
+	sudo \
+	supervisor \
+	redis-tools \
+	syslog-ng \
+	syslog-ng-core \
+	syslog-ng-mod-redis \
+  	tzdata \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& touch /etc/default/locale
+
+COPY supervisord.conf /etc/supervisor/supervisord.conf
+COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
+COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
+COPY postfix-tlspol.sh /opt/postfix-tlspol.sh
+COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+COPY --from=builder /src/postfix-tlspol/build/postfix-tlspol /usr/local/bin/postfix-tlspol
+
+RUN chmod +x /opt/postfix-tlspol.sh \
+  /usr/local/sbin/stop-supervisor.sh \
+  /docker-entrypoint.sh
+RUN rm -rf /tmp/* /var/tmp/*
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

data/Dockerfiles/postfix-tlspol/docker-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
+  cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
+fi
+
+exec "$@"

data/Dockerfiles/postfix-tlspol/postfix-tlspol.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+LOGLVL=info
+
+if [ ${DEV_MODE} != "n" ]; then
+  echo -e "\e[31mEnabling debug mode\e[0m"
+  set -x
+  LOGLVL=debug
+fi
+
+[[ ! -d /etc/postfix-tlspol ]] && mkdir -p /etc/postfix-tlspol
+[[ ! -d /var/lib/postfix-tlspol ]] && mkdir -p /var/lib/postfix-tlspol
+
+until dig +short mailcow.email > /dev/null; do
+  echo "Waiting for DNS..."
+  sleep 1
+done
+
+# Do not attempt to write to slave
+if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
+  export REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
+else
+  export REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
+fi
+
+until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
+  echo "Waiting for Redis..."
+  sleep 2
+done
+
+echo "Waiting for Postfix..."
+until ping postfix -c1 > /dev/null; do
+  sleep 1
+done
+echo "Postfix OK"
+
+cat <<EOF > /etc/postfix-tlspol/config.yaml
+server:
+  address: 0.0.0.0:8642
+
+  log-level: ${LOGLVL}
+
+  prefetch: true
+
+  cache-file: /var/lib/postfix-tlspol/cache.db
+
+dns:
+  # must support DNSSEC
+  address: 127.0.0.11:53
+EOF
+
+/usr/local/bin/postfix-tlspol -config /etc/postfix-tlspol/config.yaml

data/Dockerfiles/postfix-tlspol/stop-supervisor.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+printf "READY\n";
+
+while read line; do
+  echo "Processing Event: $line" >&2;
+  kill -3 $(cat "/var/run/supervisord.pid")
+done < /dev/stdin

data/Dockerfiles/postfix-tlspol/supervisord.conf

@@ -0,0 +1,25 @@
+[supervisord]
+pidfile=/var/run/supervisord.pid
+nodaemon=true
+user=root
+
+[program:syslog-ng]
+command=/usr/sbin/syslog-ng --foreground  --no-caps
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autostart=true
+
+[program:postfix-tlspol]
+startsecs=10
+autorestart=true
+command=/opt/postfix-tlspol.sh
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+[eventlistener:processes]
+command=/usr/local/sbin/stop-supervisor.sh
+events=PROCESS_STATE_STOPPED, PROCESS_STATE_EXITED, PROCESS_STATE_FATAL

data/Dockerfiles/postfix-tlspol/syslog-ng-redis_slave.conf

@@ -0,0 +1,45 @@
+@version: 4.8
+@include "scl.conf"
+options {
+  chain_hostnames(off);
+  flush_lines(0);
+  use_dns(no);
+  dns_cache(no);
+  use_fqdn(no);
+  owner("root"); group("adm"); perm(0640);
+  stats(freq(0));
+  bad_hostname("^gconfd$");
+};
+source s_src {
+  unix-stream("/dev/log");
+  internal();
+};
+destination d_stdout { pipe("/dev/stdout"); };
+destination d_redis_ui_log {
+  redis(
+    host("`REDIS_SLAVEOF_IP`")
+    persist-name("redis1")
+    port(`REDIS_SLAVEOF_PORT`)
+    auth("`REDISPASS`")
+    command("LPUSH" "POSTFIX_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
+  );
+};
+filter f_mail { facility(mail); };
+# start
+# overriding warnings are still displayed when the entrypoint runs its initial check
+# warnings logged by postfix-mailcow to syslog are hidden to reduce repeating msgs
+# Some other warnings are ignored
+filter f_ignore {
+  not match("overriding earlier entry" value("MESSAGE"));
+  not match("TLS SNI from checks.mailcow.email" value("MESSAGE"));
+  not match("no SASL support" value("MESSAGE"));
+  not facility (local0, local1, local2, local3, local4, local5, local6, local7);
+};
+# end
+log {
+  source(s_src);
+  filter(f_ignore);
+  destination(d_stdout);
+  filter(f_mail);
+  destination(d_redis_ui_log);
+};

data/Dockerfiles/postfix-tlspol/syslog-ng.conf

@@ -0,0 +1,45 @@
+@version: 4.8
+@include "scl.conf"
+options {
+  chain_hostnames(off);
+  flush_lines(0);
+  use_dns(no);
+  dns_cache(no);
+  use_fqdn(no);
+  owner("root"); group("adm"); perm(0640);
+  stats(freq(0));
+  bad_hostname("^gconfd$");
+};
+source s_src {
+  unix-stream("/dev/log");
+  internal();
+};
+destination d_stdout { pipe("/dev/stdout"); };
+destination d_redis_ui_log {
+  redis(
+    host("redis-mailcow")
+    persist-name("redis1")
+    port(6379)
+    auth("`REDISPASS`")
+    command("LPUSH" "POSTFIX_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
+  );
+};
+filter f_mail { facility(mail); };
+# start
+# overriding warnings are still displayed when the entrypoint runs its initial check
+# warnings logged by postfix-mailcow to syslog are hidden to reduce repeating msgs
+# Some other warnings are ignored
+filter f_ignore {
+  not match("overriding earlier entry" value("MESSAGE"));
+  not match("TLS SNI from checks.mailcow.email" value("MESSAGE"));
+  not match("no SASL support" value("MESSAGE"));
+  not facility (local0, local1, local2, local3, local4, local5, local6, local7);
+};
+# end
+log {
+  source(s_src);
+  filter(f_ignore);
+  destination(d_stdout);
+  filter(f_mail);
+  destination(d_redis_ui_log);
+};

data/Dockerfiles/postfix/Dockerfile

@@ -1,9 +1,9 @@
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
-LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
+LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ENV LC_ALL C
+ENV LC_ALL=C
 
 RUN dpkg-divert --local --rename --add /sbin/initctl \
 	&& ln -sf /bin/true /sbin/initctl \

data/Dockerfiles/postfix/postfix.sh

@@ -329,14 +329,17 @@ query = SELECT goto FROM alias
           SELECT id FROM alias
             WHERE address='%s'
             AND (active='1' OR active='2')
+            AND sender_allowed='1'
         ), (
           SELECT id FROM alias
             WHERE address='@%d'
             AND (active='1' OR active='2')
+            AND sender_allowed='1'
         )
       )
     )
     AND active='1'
+    AND sender_allowed='1'
     AND (domain IN
       (SELECT domain FROM domain
         WHERE domain='%d'
@@ -390,7 +393,7 @@ hosts = unix:/var/run/mysqld/mysqld.sock
 dbname = ${DBNAME}
 query = SELECT goto FROM spamalias
   WHERE address='%s'
-    AND validity >= UNIX_TIMESTAMP()
+    AND (validity >= UNIX_TIMESTAMP() OR permanent != 0)
 EOF
 
 if [ ! -f /opt/postfix/conf/dns_blocklists.cf ]; then

data/Dockerfiles/postfix/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/postfix/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.38
+@version: 4.8
 @include "scl.conf"
 options {
   chain_hostnames(off);
@@ -7,7 +7,7 @@ options {
   dns_cache(no);
   use_fqdn(no);
   owner("root"); group("adm"); perm(0640);
-  stats_freq(0);
+  stats(freq(0));
   bad_hostname("^gconfd$");
 };
 source s_src {

data/Dockerfiles/rspamd/Dockerfile

@@ -1,9 +1,9 @@
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG RSPAMD_VER=rspamd_3.11.1-1~ab0b44951
-ARG CODENAME=bookworm
+ARG RSPAMD_VER=rspamd_4.2.1-1~b7a16ae
+ARG CODENAME=trixie
 ENV LC_ALL=C
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -14,8 +14,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   dnsutils \
   netcat-traditional \
   wget \
-  redis-tools \ 
-  procps \ 
+  redis-tools \
+  procps \
   nano \
   lua-cjson \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \

data/Dockerfiles/rspamd/docker-entrypoint.sh

@@ -86,7 +86,8 @@ if [[ "${SKIP_OLEFY}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
     rm /etc/rspamd/local.d/external_services.conf
   fi
 else
-  cat <<EOF > /etc/rspamd/local.d/external_services.conf
+  if [[ ! -f /etc/rspamd/local.d/external_services.conf ]]; then
+    cat <<EOF > /etc/rspamd/local.d/external_services.conf
 oletools {
   # default olefy settings
   servers = "olefy:10055";
@@ -100,6 +101,7 @@ oletools {
   retransmits = 1;
 }
 EOF
+  fi
 fi
 
 # Provide additional lua modules

data/Dockerfiles/sogo/Dockerfile

@@ -1,47 +1,167 @@
-FROM debian:bookworm-slim
+# SOGo built from source to enable security patch application
+# Repository: https://github.com/Alinto/sogo
+# Version: SOGo-5.12.12
+#
+# Applied security patches:
+# -
+#
+# To add new patches, modify SOGO_SECURITY_PATCHES ARG below with space-separated commit hashes
+
+FROM debian:bookworm
 
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG DEBIAN_VERSION=bookworm
-ARG SOGO_DEBIAN_REPOSITORY=https://packagingv2.sogo.nu/sogo-nightly-debian/
+ARG SOGO_VERSION=SOGo-5.12.12
+ARG SOPE_VERSION=SOPE-5.12.12
+# Security patches to apply (space-separated commit hashes)
+ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
-ARG GOSU_VERSION=1.17
+ARG GOSU_VERSION=1.19
 ENV LC_ALL=C
 
-# Prerequisites
-RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
-  && apt-get update && apt-get install -y --no-install-recommends \
-  apt-transport-https \
-  ca-certificates \
-  gettext \
-  gnupg \
-  mariadb-client \
-  rsync \
-  supervisor \
-  syslog-ng \
-  syslog-ng-core \
-  syslog-ng-mod-redis \
-  dirmngr \
-  netcat-traditional \
-  psmisc \
-  wget \
-  patch \
+# Install dependencies, build SOPE and SOGo, then clean up (all in one layer to minimize image size)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    # Build dependencies
+    git \
+    build-essential \
+    gobjc \
+    pkg-config \
+    gnustep-make \
+    gnustep-base-runtime \
+    libgnustep-base-dev \
+    libxml2-dev \
+    libldap2-dev \
+    libssl-dev \
+    zlib1g-dev \
+    libpq-dev \
+    libmariadb-dev-compat \
+    libmemcached-dev \
+    libsodium-dev \
+    libcurl4-openssl-dev \
+    libzip-dev \
+    libytnef0-dev \
+    libwbxml2-dev \
+    curl \
+    ca-certificates \
+    # Runtime dependencies
+    apt-transport-https \
+    gettext \
+    gnupg \
+    mariadb-client \
+    rsync \
+    supervisor \
+    syslog-ng \
+    syslog-ng-core \
+    syslog-ng-mod-redis \
+    dirmngr \
+    netcat-traditional \
+    psmisc \
+    wget \
+    patch \
+    libobjc4 \
+    libxml2 \
+    libldap-2.5-0 \
+    libssl3 \
+    zlib1g \
+    libmariadb3 \
+    libmemcached11 \
+    libsodium23 \
+    libcurl4 \
+    libzip4 \
+    libytnef0 \
+    libwbxml2-1 \
+  # Download gosu
   && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch" \
   && chmod +x /usr/local/bin/gosu \
   && gosu nobody true \
-  && mkdir /usr/share/doc/sogo \
-  && touch /usr/share/doc/sogo/empty.sh \
-  && wget -O- https://keys.openpgp.org/vks/v1/by-fingerprint/74FFC6D72B925A34B5D356BDF8A27B36A6E2EAE9 | gpg --dearmor | apt-key add - \
-  && echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} ${DEBIAN_VERSION} main" > /etc/apt/sources.list.d/sogo.list \
-  && apt-get update && apt-get install -y --no-install-recommends \
-    sogo \
-    sogo-activesync \
-  && apt-get autoclean \
+  # Build SOPE
+  && git clone --depth 1 --branch ${SOPE_VERSION} https://github.com/Alinto/sope.git /tmp/sope \
+  && cd /tmp/sope \
+  && rm -rf .git \
+  && . /usr/share/GNUstep/Makefiles/GNUstep.sh \
+  && ./configure --prefix=/usr --disable-debug --disable-strip \
+  && make -j$(nproc) \
+  && make install \
+  && cd / \
+  && rm -rf /tmp/sope \
+  # Build SOGo with security patches
+  && git clone --depth 1 --branch ${SOGO_VERSION} https://github.com/Alinto/sogo.git /tmp/sogo \
+  && cd /tmp/sogo \
+  && git config user.email "builder@mailcow.local" \
+  && git config user.name "SOGo Builder" \
+  && for patch in ${SOGO_SECURITY_PATCHES}; do \
+       echo "Applying security patch: ${patch}"; \
+       git fetch origin ${patch} && git cherry-pick ${patch}; \
+     done \
+  && rm -rf .git \
+  && . /usr/share/GNUstep/Makefiles/GNUstep.sh \
+  && ./configure --disable-debug --disable-strip \
+  && make -j$(nproc) \
+  && make install \
+  && cd /tmp/sogo/ActiveSync \
+  && . /usr/share/GNUstep/Makefiles/GNUstep.sh \
+  && make -j$(nproc) install \
+  && cd / \
+  && rm -rf /tmp/sogo \
+  # Strip binaries
+  && strip --strip-unneeded /usr/local/sbin/sogod 2>/dev/null || true \
+  && strip --strip-unneeded /usr/local/sbin/sogo-tool 2>/dev/null || true \
+  && strip --strip-unneeded /usr/local/sbin/sogo-ealarms-notify 2>/dev/null || true \
+  && strip --strip-unneeded /usr/local/sbin/sogo-slapd-sockd 2>/dev/null || true \
+  # Remove build dependencies and clean up
+  && apt-get purge -y --auto-remove \
+    git \
+    build-essential \
+    gobjc \
+    gnustep-make \
+    libgnustep-base-dev \
+    libxml2-dev \
+    libldap2-dev \
+    libssl-dev \
+    zlib1g-dev \
+    libpq-dev \
+    libmariadb-dev-compat \
+    libmemcached-dev \
+    libsodium-dev \
+    libcurl4-openssl-dev \
+    libzip-dev \
+    libytnef0-dev \
+    curl \
+  && apt-get autoremove -y \
+  && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
+  && rm -rf /usr/share/doc/* \
+  && rm -rf /usr/share/man/* \
+  && rm -rf /var/cache/debconf/* \
+  && rm -rf /tmp/* \
+  && rm -rf /root/.cache \
+  && find /usr/local/lib -name '*.a' -delete \
+  && find /usr/lib -name '*.a' -delete \
+  && mkdir -p /usr/share/doc/sogo \
+  && touch /usr/share/doc/sogo/empty.sh \
   && touch /etc/default/locale
 
+# Configure library paths
+RUN echo "/usr/lib64" > /etc/ld.so.conf.d/sogo.conf \
+  && echo "/usr/local/lib/sogo" >> /etc/ld.so.conf.d/sogo.conf \
+  && echo "/usr/local/lib/GNUstep/Frameworks/SOGo.framework/Versions/5/sogo" >> /etc/ld.so.conf.d/sogo.conf \
+  && ldconfig
+
+# Create sogo user and group
+RUN groupadd -r -g 999 sogo \
+  && useradd -r -u 999 -g sogo -d /var/lib/sogo -s /bin/bash -c "SOGo Daemon" sogo \
+  && mkdir -p /var/lib/sogo /var/run/sogo /var/log/sogo /var/spool/sogo \
+  && chown -R sogo:sogo /var/lib/sogo /var/run/sogo /var/log/sogo /var/spool/sogo
+
+# Create symlinks for SOGo binaries
+RUN ln -s /usr/local/sbin/sogod /usr/sbin/sogod \
+  && ln -s /usr/local/sbin/sogo-tool /usr/sbin/sogo-tool \
+  && ln -s /usr/local/sbin/sogo-ealarms-notify /usr/sbin/sogo-ealarms-notify \
+  && ln -s /usr/local/sbin/sogo-slapd-sockd /usr/sbin/sogo-slapd-sockd
+
+# Copy configuration files and scripts
 COPY ./bootstrap-sogo.sh /bootstrap-sogo.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
@@ -56,4 +176,4 @@ RUN chmod +x /bootstrap-sogo.sh \
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

data/Dockerfiles/sogo/acl.diff

@@ -1,5 +1,5 @@
---- /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:57.987504204 +0200
-+++ /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:35.918291298 +0200
+--- /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:57.987504204 +0200
++++ /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox    2018-08-17 18:29:35.918291298 +0200
 @@ -46,7 +46,7 @@
            </md-item-template>
          </md-autocomplete>

data/Dockerfiles/sogo/bootstrap-sogo.sh

@@ -13,17 +13,19 @@ do
   sleep 3
 done
 
-# Wait for updated schema
+# Wait for updated schema (DB_VERSION from docker-compose.yml)
 DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
-while [[ "${DBV_NOW}" != "${DBV_NEW}" ]]; do
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
   echo "Waiting for schema update..."
   DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-  DBV_NEW=$(grep -oE '\$db_version = .*;' init_db.inc.php | sed 's/$db_version = //g;s/;//g' | cut -d \" -f2)
   sleep 5
 done
 echo "DB schema is ${DBV_NOW}"
 
+if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "DROP TRIGGER IF EXISTS sogo_update_password"
+fi
+
 # cat /dev/urandom seems to hang here occasionally and is not recommended anyway, better use openssl
 RAND_PASS=$(openssl rand -base64 16 | tr -dc _A-Z-a-z-0-9)
 
@@ -126,18 +128,22 @@ chmod 600 /var/lib/sogo/GNUstep/Defaults/sogod.plist
 # Patch ACLs
 #if [[ ${ACL_ANYONE} == 'allow' ]]; then
 #  #enable any or authenticated targets for ACL
-#  if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-#    patch -R /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+#  if patch -R -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+#    patch -R /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
 #  fi
 #else
 #  #disable any or authenticated targets for ACL
-#  if patch -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
-#    patch /usr/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
+#  if patch -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff > /dev/null; then
+#    patch /usr/local/lib/GNUstep/SOGo/Templates/UIxAclEditor.wox < /acl.diff;
 #  fi
 #fi
 
-if patch -R -sfN --dry-run /usr/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff > /dev/null; then
-  patch -R /usr/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff;
+# Apply custom UI patch (reverse patch to ADD buttons)
+if patch -R -sfN --dry-run /usr/local/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff > /dev/null; then
+  echo "Applying navMailcowBtns patch (reverse to add buttons)..."
+  patch -R /usr/local/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox < /navMailcowBtns.diff;
+else
+  echo "navMailcowBtns patch already applied or cannot be applied"
 fi
 
 # Rename custom logo, if any
@@ -145,7 +151,7 @@ fi
 
 # Rsync web content
 echo "Syncing web content with named volume"
-rsync -a /usr/lib/GNUstep/SOGo/. /sogo_web/
+rsync -a /usr/local/lib/GNUstep/SOGo/. /sogo_web/
 
 # Chown backup path
 chown -R sogo:sogo /sogo_backup

data/Dockerfiles/unbound/Dockerfile

@@ -1,12 +1,16 @@
-FROM alpine:3.21
+FROM alpine:3.23
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
+# install unbound from alpine:edge to get security patches
+# minimum version is pinned
+RUN apk add --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main "unbound>=1.26.1-r0"
+
+# install other packages from regular alpine stable repo
 RUN apk add --update --no-cache \
 	curl \
 	bind-tools \
 	coreutils \
-	unbound \
 	bash \
 	openssl \
 	drill \

data/Dockerfiles/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.21
+FROM alpine:3.23
 
 LABEL maintainer = "The Infrastructure Company GmbH <info@servercow.de>"
 
@@ -16,7 +16,6 @@ RUN apk add --update \
   fcgi \
   openssl \
   nagios-plugins-mysql \
-  nagios-plugins-dns \
   nagios-plugins-disk \
   bind-tools \
   redis \
@@ -32,9 +31,12 @@ RUN apk add --update \
   tzdata \
   whois \
   && curl https://raw.githubusercontent.com/mludvig/smtp-cli/v3.10/smtp-cli -o /smtp-cli \
-  && chmod +x smtp-cli
+  && chmod +x smtp-cli \
+  && mkdir /usr/lib/mailcow
 
 COPY watchdog.sh /watchdog.sh
 COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
+COPY check_dns.sh /usr/lib/mailcow/check_dns.sh
+COPY client.cnf /etc/my.cnf.d/client.cnf
 
 CMD ["/watchdog.sh"]

data/Dockerfiles/watchdog/check_dns.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+while getopts "H:s:" opt; do
+  case "$opt" in
+    H) HOST="$OPTARG" ;;
+    s) SERVER="$OPTARG" ;;
+    *) echo "Usage: $0 -H host -s server"; exit 3 ;;
+  esac
+done
+
+if [ -z "$SERVER" ]; then
+  echo "No DNS Server provided"
+  exit 3
+fi
+
+if [ -z "$HOST" ]; then
+  echo "No host to test provided"
+  exit 3
+fi
+
+# run dig and measure the time it takes to run
+START_TIME=$(perl -MTime::HiRes -e 'print Time::HiRes::time')
+dig_output=$(dig +short +timeout=2 +tries=1 "$HOST" @"$SERVER" 2>/dev/null)
+dig_rc=$?
+END_TIME=$(perl -MTime::HiRes -e 'print Time::HiRes::time')
+dig_output_ips=$(echo "$dig_output" | grep -E '^[0-9.]+$' | sort | paste -sd ',' -)
+ELAPSED_TIME=$(perl -e "printf('%.3f', $END_TIME - $START_TIME)")
+
+# validate and perform nagios like output and exit codes
+if [ $dig_rc -ne 0 ] || [ -z "$dig_output" ]; then
+  echo "Domain $HOST was not found by the server"
+  exit 2
+elif [ $dig_rc -eq 0 ]; then
+  echo "DNS OK: $ELAPSED_TIME seconds response time. $HOST returns $dig_output_ips"
+  exit 0
+else
+  echo "Unknown error"
+  exit 3
+fi

data/Dockerfiles/watchdog/client.cnf

@@ -0,0 +1,3 @@
+[client]
+ssl = false
+ssl-verify-server-cert = false

data/Dockerfiles/watchdog/watchdog.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+if [ "${DEV_MODE}" != "n" ]; then
+  echo -e "\e[31mEnabled Debug Mode\e[0m"
+  set -x
+fi
+
 trap "exit" INT TERM
 trap "kill 0" EXIT
 
@@ -33,7 +38,7 @@ if [[ ! -p /tmp/com_pipe ]]; then
 fi
 
 # Wait for containers
-while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
+while ! mariadb-admin status --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} --silent; do
   echo "Waiting for SQL..."
   sleep 2
 done
@@ -297,7 +302,7 @@ unbound_checks() {
     touch /tmp/unbound-mailcow; echo "$(tail -50 /tmp/unbound-mailcow)" > /tmp/unbound-mailcow
     host_ip=$(get_container_ip unbound-mailcow)
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_dns -s ${host_ip} -H stackoverflow.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/mailcow/check_dns.sh -s ${host_ip} -H stackoverflow.com 2>> /tmp/unbound-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     DNSSEC=$(dig com +dnssec | egrep 'flags:.+ad')
     if [[ -z ${DNSSEC} ]]; then
       echo "DNSSEC failure" 2>> /tmp/unbound-mailcow 1>&2
@@ -354,8 +359,8 @@ mysql_checks() {
   while [ ${err_count} -lt ${THRESHOLD} ]; do
     touch /tmp/mysql-mailcow; echo "$(tail -50 /tmp/mysql-mailcow)" > /tmp/mysql-mailcow
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_mysql -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
-    /usr/lib/nagios/plugins/check_mysql_query -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql -f /etc/my.cnf.d/client.cnf -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql_query -f /etc/my.cnf.d/client.cnf -s /var/run/mysqld/mysqld.sock -u ${DBUSER} -p ${DBPASS} -d ${DBNAME} -q "SELECT COUNT(*) FROM information_schema.tables" 2>> /tmp/mysql-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "MySQL/MariaDB" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -379,7 +384,7 @@ mysql_repl_checks() {
   while [ ${err_count} -lt ${THRESHOLD} ]; do
     touch /tmp/mysql_repl_checks; echo "$(tail -50 /tmp/mysql_repl_checks)" > /tmp/mysql_repl_checks
     err_c_cur=${err_count}
-    /usr/lib/nagios/plugins/check_mysql_slavestatus.sh -S /var/run/mysqld/mysqld.sock -u root -p ${DBROOT} 2>> /tmp/mysql_repl_checks 1>&2; err_count=$(( ${err_count} + $? ))
+    /usr/lib/nagios/plugins/check_mysql_slavestatus.sh -o /etc/my.cnf.d/client.cnf -S /var/run/mysqld/mysqld.sock -u root -p ${DBROOT} 2>> /tmp/mysql_repl_checks 1>&2; err_count=$(( ${err_count} + $? ))
     [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
     [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
     progress "MySQL/MariaDB replication" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
@@ -445,6 +450,31 @@ postfix_checks() {
   return 1
 }
 
+postfix-tlspol_checks() {
+  err_count=0
+  diff_c=0
+  THRESHOLD=${POSTFIX_TLSPOL_THRESHOLD}
+  # Reduce error count by 2 after restarting an unhealthy container
+  trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
+  while [ ${err_count} -lt ${THRESHOLD} ]; do
+    touch /tmp/postfix-tlspol-mailcow; echo "$(tail -50 /tmp/postfix-tlspol-mailcow)" > /tmp/postfix-tlspol-mailcow
+    host_ip=$(get_container_ip postfix-tlspol-mailcow)
+    err_c_cur=${err_count}
+    /usr/lib/nagios/plugins/check_tcp -4 -H ${host_ip} -p 8642 2>> /tmp/postfix-tlspol-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
+    [ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
+    [ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
+    progress "Postfix TLS Policy companion" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
+    if [[ $? == 10 ]]; then
+      diff_c=0
+      sleep 1
+    else
+      diff_c=0
+      sleep $(( ( RANDOM % 60 ) + 20 ))
+    fi
+  done
+  return 1
+}
+
 clamd_checks() {
   err_count=0
   diff_c=0
@@ -922,6 +952,18 @@ PID=$!
 echo "Spawned mailq_checks with PID ${PID}"
 BACKGROUND_TASKS+=(${PID})
 
+(
+while true; do
+  if ! postfix-tlspol_checks; then
+    log_msg "Postfix TLS Policy hit error limit"
+    echo postfix-tlspol-mailcow > /tmp/com_pipe
+  fi
+done
+) &
+PID=$!
+echo "Spawned postfix-tlspol_checks with PID ${PID}"
+BACKGROUND_TASKS+=(${PID})
+
 (
 while true; do
   if ! dovecot_checks; then

data/conf/dovecot/auth/mailcowauth.php

@@ -80,23 +80,39 @@ if ($isSOGoRequest) {
 }
 if ($result === false){
   // If it's a SOGo Request, don't check for protocol access
-  $service = ($isSOGoRequest) ? false : array($post['service'] => true);
-  $result = apppass_login($post['username'], $post['password'], $service, array(
+  if ($isSOGoRequest) {
+    $service = 'SOGO';
+    $post['service'] = 'NONE';
+  } else {
+    $service = $post['service'];
+  }
+
+  $result = apppass_login($post['username'], $post['password'], array(
+    'service' => $post['service'],
     'is_internal' => true,
     'remote_addr' => $post['real_rip']
   ));
   if ($result) {
-    error_log('MAILCOWAUTH: App auth for user ' . $post['username']);
-    set_sasl_log($post['username'], $post['real_rip'], $post['service']);
+    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $post['real_rip']);
+    set_sasl_log($post['username'], $post['real_rip'], $service);
   }
 }
 if ($result === false){
   // Init Identity Provider
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
-  $result = user_login($post['username'], $post['password'], array('is_internal' => true));
+  try {
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+  } catch (\Mailcow\Auth\IdpUnavailable $e) {
+    // temporary failure for Dovecot, no failed login
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
+    http_response_code(503); // Service Unavailable
+    echo json_encode($return);
+    session_destroy();
+    exit;
+  }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username']);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
     set_sasl_log($post['username'], $post['real_rip'], $post['service']);
   }
 }
@@ -105,7 +121,7 @@ if ($result) {
   http_response_code(200); // OK
   $return['success'] = true;
 } else {
-  error_log("MAILCOWAUTH: Login failed for user " . $post['username']);
+  error_log("MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
   http_response_code(401); // Unauthorized
 }
 

data/conf/dovecot/auth/passwd-verify.lua

@@ -32,8 +32,23 @@ function auth_password_verify(request, password)
   -- Returning PASSDB_RESULT_PASSWORD_MISMATCH will reset the user's auth cache entry.
   -- Returning PASSDB_RESULT_INTERNAL_FAILURE keeps the existing cache entry,
   -- even if the TTL has expired. Useful to avoid cache eviction during backend issues.
+
+  -- On a network-level failure (nginx unreachable, DNS failure, timeout) https.request
+  -- returns nil plus an error string, so c is not a numeric HTTP status code. Treat this
+  -- as a backend outage and keep the cache entry, rather than wiping it as a mismatch.
+  if type(c) ~= "number" then
+    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unreachable"
+  end
+
+  -- 5xx: the backend or the identity provider cannot answer, a temporary failure and no failed login
+  if c >= 500 then
+    dovecot.i_info("HTTP request to auth backend failed with " .. tostring(c) .. " for user " .. request.user)
+    return dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE, "Upstream unavailable"
+  end
+
   if c ~= 200 and c ~= 401 then
-    dovecot.i_info("HTTP request failed with " .. c .. " for user " .. request.user)
+    dovecot.i_info("HTTP request failed with " .. tostring(c) .. " for user " .. request.user)
     return dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH, "Upstream error"
   end
 

data/conf/dovecot/dovecot.conf

@@ -67,9 +67,11 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
+# cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
+# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
 passdb {
   driver = lua
-  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
 }
 # Set doveadm_password=your-secret-password in data/conf/dovecot/extra.conf (create if missing)
 service doveadm {

data/conf/dovecot/global_sieve_before

@@ -1,13 +1,3 @@
 # global_sieve_before script
 # global_sieve_before -> user sieve_before (mailcow UI) -> user sieve_after (mailcow UI) -> global_sieve_after
 
-require ["mailbox", "fileinto"];
-
-if header :contains ["Chat-Version"] [""] {
-  if mailboxexists "DeltaChat" {
-    fileinto "DeltaChat";
-  } else {
-    fileinto :create "DeltaChat";
-  }
-  stop;
-}

data/conf/nginx/templates/nginx.conf.j2

@@ -13,6 +13,7 @@ events {
 http {
     include /etc/nginx/mime.types;
     default_type  application/octet-stream;
+    server_tokens off;
 
     log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
@@ -41,20 +42,35 @@ http {
         https https;
     }
 
+    map $server_port $sogo_auth_internal {
+        65510   "1";
+        default "";
+    }
+
     {% if HTTP_REDIRECT %}
     # HTTP to HTTPS redirect
     server {
         root /web;
         listen {{ HTTP_PORT }} default_server;
+        {% if ENABLE_IPV6 %}
         listen [::]:{{ HTTP_PORT }} default_server;
+        {%endif%}
 
-        server_name {{ MAILCOW_HOSTNAME }} autodiscover.* autoconfig.* {{ ADDITIONAL_SERVER_NAMES | join(' ') }};
+        server_name {{ MAILCOW_HOSTNAME }} autodiscover.* autoconfig.* mta-sts.* {{ ADDITIONAL_SERVER_NAMES | join(' ') }};
 
         if ( $request_uri ~* "%0A|%0D" ) { return 403; }
         location ^~ /.well-known/acme-challenge/ {
             allow all;
             default_type "text/plain";
         }
+        location ^~ /.well-known/mta-sts.txt {
+            allow all;
+            fastcgi_split_path_info ^(.+\.php)(/.+)$;
+            fastcgi_pass {{ PHPFPMHOST }}:9002;
+            include /etc/nginx/fastcgi_params;
+            fastcgi_param SCRIPT_FILENAME $document_root/mta-sts.php;
+            fastcgi_param PATH_INFO $fastcgi_path_info;
+        }
         location / {
             return 301 https://$host$uri$is_args$args;
         }
@@ -70,7 +86,7 @@ http {
         {%endif%}
         listen {{ HTTPS_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%} ssl;
 
-        {% if not DISABLE_IPv6 %}
+        {% if ENABLE_IPV6 %}
         {% if not HTTP_REDIRECT %}
         listen [::]:{{ HTTP_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%};
         {%endif%}
@@ -82,7 +98,7 @@ http {
         ssl_certificate /etc/ssl/mail/cert.pem;
         ssl_certificate_key /etc/ssl/mail/key.pem;
 
-        server_name {{ MAILCOW_HOSTNAME }} autodiscover.* autoconfig.*;
+        server_name {{ MAILCOW_HOSTNAME }} autodiscover.* autoconfig.* mta-sts.*;
 
         include /etc/nginx/includes/sites-default.conf;
     }
@@ -97,7 +113,7 @@ http {
         {%endif%}
         listen {{ HTTPS_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%} ssl;
 
-        {% if not DISABLE_IPv6 %}
+        {% if ENABLE_IPV6 %}
         {% if not HTTP_REDIRECT %}
         listen [::]:{{ HTTP_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%};
         {%endif%}
@@ -118,7 +134,7 @@ http {
     # rspamd dynmaps:
     server {
         listen 8081;
-        {% if not DISABLE_IPv6 %}
+        {% if ENABLE_IPV6 %}
         listen [::]:8081;
         {%endif%}
         index index.php index.html;
@@ -191,7 +207,7 @@ http {
         {%endif%}
         listen {{ HTTPS_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%} ssl;
 
-        {% if not DISABLE_IPv6 %}
+        {% if ENABLE_IPV6 %}
         {% if not HTTP_REDIRECT %}
         listen [::]:{{ HTTP_PORT }}{% if NGINX_USE_PROXY_PROTOCOL %} proxy_protocol{%endif%};
         {%endif%}

data/conf/nginx/templates/sites-default.conf.j2

@@ -14,7 +14,6 @@ ssl_session_tickets off;
 
 add_header Strict-Transport-Security "max-age=15768000;";
 add_header X-Content-Type-Options nosniff;
-add_header X-XSS-Protection "1; mode=block";
 add_header X-Robots-Tag none;
 add_header X-Download-Options noopen;
 add_header X-Frame-Options "SAMEORIGIN" always;
@@ -76,6 +75,14 @@ location ^~ /.well-known/acme-challenge/ {
     allow all;
     default_type "text/plain";
 }
+location ^~ /.well-known/mta-sts.txt {
+    allow all;
+    fastcgi_split_path_info ^(.+\.php)(/.+)$;
+    fastcgi_pass {{ PHPFPMHOST }}:9002;
+    include /etc/nginx/fastcgi_params;
+    fastcgi_param SCRIPT_FILENAME $document_root/mta-sts.php;
+    fastcgi_param PATH_INFO $fastcgi_path_info;
+}
 
 rewrite ^/.well-known/caldav$ /SOGo/dav/ permanent;
 rewrite ^/.well-known/carddav$ /SOGo/dav/ permanent;
@@ -109,6 +116,9 @@ location ~ \.php$ {
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
+    fastcgi_param HTTP_X_REAL_IP $remote_addr;
+    # trusted internal-auth marker; empty for external clients (see nginx.conf map)
+    fastcgi_param SOGO_AUTH_INTERNAL $sogo_auth_internal;
     fastcgi_read_timeout 3600;
     fastcgi_send_timeout 3600;
 }
@@ -254,19 +264,19 @@ location ~* /sogo$ {
 }
 
 location /SOGo.woa/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location /.woa/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location /SOGo/WebServerResources/ {
-    alias /usr/lib/GNUstep/SOGo/WebServerResources/;
+    alias /usr/local/lib/GNUstep/SOGo/WebServerResources/;
 }
 
 location (^/SOGo/so/ControlPanel/Products/[^/]*UI/Resources/.*\.(jpg|png|gif|css|js)$) {
-    alias /usr/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
+    alias /usr/local/lib/GNUstep/SOGo/$1.SOGo/Resources/$2;
 }
 {% endif %}
 

data/conf/phpfpm/crons/group-grants.php

@@ -0,0 +1,65 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Group Shares") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/group-grants.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+$sharing = \Mailcow\MailboxGroup\MailboxGroupModule::create($pdo, \Mailcow\Core\Context::system())->sharing;
+
+// Every minute: subscriptions of changed groups; failed ones are retried and logged by the full run
+if ((int)date('i') % 15 !== 0) {
+  $sharing->syncSubscriptions();
+  exit(0);
+}
+
+// Every 15 minutes: IMAP ACLs on new top-level folders and failed grants, SOGo grants of owners that opened SOGo meanwhile
+foreach ($sharing->reconcile() as $warning) {
+  logMsg("warning", $warning[0] . ": " . html_entity_decode(implode(", ", array_slice($warning, 1))));
+}

data/conf/phpfpm/crons/identity-sync.php

@@ -0,0 +1,95 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "Identity Sync") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+
+// The legacy mailbox functions run as admin
+$_SESSION['mailcow_cc_username'] = "admin";
+$_SESSION['mailcow_cc_role'] = "admin";
+$_SESSION['acl']['tls_policy'] = "1";
+$_SESSION['acl']['quarantine_notification'] = "1";
+$_SESSION['acl']['quarantine_category'] = "1";
+$_SESSION['acl']['ratelimit'] = "1";
+$_SESSION['acl']['sogo_access'] = "1";
+$_SESSION['acl']['protocol_access'] = "1";
+$_SESSION['acl']['mailbox_relayhost'] = "1";
+$_SESSION['acl']['unlimited_quota'] = "1";
+
+// One run at a time, released when the process ends
+$lock = fopen('/tmp/identity-sync.lock', 'c');
+if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
+  exit(0);
+}
+
+// Users, offboarding and groups of Keycloak or LDAP; requested runs first, else every sync_interval minutes.
+// The IdP is only contacted when a run is due.
+$iam_settings = identity_provider('get');
+$identity = \Mailcow\Identity\IdentityModule::create($pdo, \Mailcow\Core\Context::system(), $iam_settings, function () {
+  return identity_provider('init');
+});
+$run = $identity->sync->runDue();
+if ($run !== null) {
+  $summary = $run['summary'];
+  if ($run['status'] == 'error') {
+    logMsg("err", ucfirst($run['mode']) . " failed: " . $summary['error']);
+  } else {
+    $counts = array();
+    foreach ($summary['counts'] as $key => $count) {
+      if ($count > 0) $counts[] = $key . "=" . $count;
+    }
+    logMsg(($run['status'] == 'blocked') ? "warning" : "info", ucfirst($run['mode']) . " " . $run['status'] . ": " . (empty($counts) ? "no changes" : implode(", ", $counts)));
+    foreach ($summary['details']['conflicts'] as $conflict) {
+      logMsg("warning", $conflict);
+    }
+    foreach ($summary['errors'] ?? array() as $error) {
+      logMsg("err", $error);
+    }
+  }
+}
+session_destroy();

data/conf/phpfpm/crons/imapsync-oauth-refresh.php

@@ -0,0 +1,73 @@
+<?php
+
+require_once(__DIR__ . '/../web/inc/vars.inc.php');
+if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
+  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
+}
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+  PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+  PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+  PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+} catch (PDOException $e) {
+  fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+$redis = new Redis();
+try {
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  } else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+} catch (Exception $e) {
+  fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n");
+  exit(1);
+}
+
+function logMsg($priority, $message, $task = "imapsync OAuth refresh") {
+  global $redis;
+  $redis->lPush('CRON_LOG', json_encode(array(
+    "time" => time(),
+    "priority" => $priority,
+    "task" => $task,
+    "message" => $message,
+  )));
+}
+
+require_once __DIR__ . '/../web/inc/functions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+require_once __DIR__ . '/../web/inc/sessions.inc.php';
+require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+
+// File-lock to prevent concurrent runs
+$lock_file = '/tmp/imapsync-oauth-refresh.lock';
+if (file_exists($lock_file)) {
+  $pid = (int)trim(@file_get_contents($lock_file));
+  if ($pid > 0 && posix_kill($pid, 0)) {
+    logMsg("info", "Previous refresh still running (pid=$pid), exiting");
+    exit(0);
+  }
+  @unlink($lock_file);
+}
+file_put_contents($lock_file, getmypid());
+
+try {
+  // client_credentials source tokens, unused per-user tokens (1 day grace), authorization_code per-user tokens;
+  // failing tokens are retried 15 minutes after their last attempt (C2), errors are sanitized
+  $imapsync = \Mailcow\Imapsync\ImapsyncModule::create($pdo, \Mailcow\Core\Context::system(), [
+    'hostname' => (string)getenv('MAILCOW_HOSTNAME'),
+  ]);
+  foreach ($imapsync->oauth->refreshDue() as $e) {
+    logMsg($e['priority'], $e['message']);
+  }
+} finally {
+  @unlink($lock_file);
+}

data/conf/phpfpm/crons/keycloak-sync.php

@@ -1,231 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "Keycloak Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "keycloak" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Keycloak Provider
-$iam_provider = identity_provider('init');
-
-// Loop until all users have been retrieved
-while (true) {
-  // Get admin access token
-  $admin_token = identity_provider("get-keycloak-admin-token");
-
-  // Make the API request to retrieve the users
-  $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users?first=$start&max=$max";
-  $ch = curl_init();
-  curl_setopt($ch, CURLOPT_URL, $url);
-  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-  curl_setopt($ch, CURLOPT_HTTPHEADER, [
-    "Content-Type: application/json",
-    "Authorization: Bearer " . $admin_token
-  ]);
-  $response = curl_exec($ch);
-  $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
-  curl_close($ch);
-
-  if ($code != 200){
-    logMsg("err", "Received HTTP {$code}");
-    session_destroy();
-    exit;
-  }
-  try {
-    $response = json_decode($response, true);
-  } catch (Exception $e) {
-    logMsg("err", $e->getMessage());
-    break;
-  }
-  if (!is_array($response)){
-    logMsg("err", "Received malformed response from keycloak api");
-    break;
-  }
-  if (count($response) == 0) {
-    break;
-  }
-
-  // Process the batch of users
-  foreach ($response as $user) {
-    if (empty($user['email'])){
-      logMsg("warning", "No email address in keycloak found for user " . $user['name']);
-      continue;
-    }
-
-    // try get mailbox user
-    $stmt = $pdo->prepare("SELECT
-      mailbox.*,
-      domain.active AS d_active
-      FROM `mailbox`
-      INNER JOIN domain on mailbox.domain = domain.domain
-      WHERE `kind` NOT REGEXP 'location|thing|group'
-        AND `username` = :user");
-    $stmt->execute(array(':user' => $user['email']));
-    $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-    // check if matching attribute mapping exists
-    $user_template = $user['attributes']['mailcow_template'][0];
-    $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-    $_SESSION['access_all_exception'] = '1';
-    if (!$row && intval($iam_settings['import_users']) == 1){
-      if ($mapper_key === false){
-        if (!empty($iam_settings['default_template'])) {
-          $mbox_template = $iam_settings['default_template'];
-          logMsg("warning", "Using default template for user " . $user['email']);
-        } else {
-          logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-          continue;
-        }
-      } else {
-        $mbox_template = $iam_settings['templates'][$mapper_key];
-      }
-      // mailbox user does not exist, create...
-      logMsg("info", "Creating user " . $user['email']);
-      $create_res = mailbox('add', 'mailbox_from_template', array(
-        'domain' => explode('@', $user['email'])[1],
-        'local_part' => explode('@', $user['email'])[0],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'authsource' => 'keycloak',
-        'template' => $mbox_template
-      ));
-      if (!$create_res){
-        logMsg("err", "Could not create user " . $user['email']);
-        continue;
-      }
-    } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "keycloak") {
-      if ($mapper_key === false){
-        logMsg("warning", "No matching attribute mapping found for user " . $user['email']);
-        continue;
-      }
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-      // mailbox user does exist, sync attribtues...
-      logMsg("info", "Syncing attributes for user " . $user['email']);
-      mailbox('edit', 'mailbox_from_template', array(
-        'username' => $user['email'],
-        'name' => $user['firstName'] . " " . $user['lastName'],
-        'template' => $mbox_template
-      ));
-    } else {
-      // skip mailbox user
-      logMsg("info", "Skipping user " . $user['email']);
-    }
-    $_SESSION['access_all_exception'] = '0';
-
-    sleep(0.025);
-  }
-
-  // Update the pagination variables for the next batch
-  $start += $max;
-  sleep(1);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/conf/phpfpm/crons/ldap-sync.php

@@ -1,198 +0,0 @@
-<?php
-
-require_once(__DIR__ . '/../web/inc/vars.inc.php');
-if (file_exists(__DIR__ . '/../web/inc/vars.local.inc.php')) {
-  include_once(__DIR__ . '/../web/inc/vars.local.inc.php');
-}
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-
-// Init database
-//$dsn = $database_type . ':host=' . $database_host . ';dbname=' . $database_name;
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  logMsg("err", $e->getMessage());
-  session_destroy();
-  exit;
-}
-
-// Init Redis
-$redis = new Redis();
-try {
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  echo "Exiting: " . $e->getMessage();
-  session_destroy();
-  exit;
-}
-
-function logMsg($priority, $message, $task = "LDAP Sync") {
-  global $redis;
-
-  $finalMsg = array(
-    "time" => time(),
-    "priority" => $priority,
-    "task" => $task,
-    "message" => $message
-  );
-  $redis->lPush('CRON_LOG', json_encode($finalMsg));
-}
-
-// Load core functions first
-require_once __DIR__ . '/../web/inc/functions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-require_once __DIR__ . '/../web/inc/sessions.inc.php';
-require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-
-$_SESSION['mailcow_cc_username'] = "admin";
-$_SESSION['mailcow_cc_role'] = "admin";
-$_SESSION['acl']['tls_policy'] = "1";
-$_SESSION['acl']['quarantine_notification'] = "1";
-$_SESSION['acl']['quarantine_category'] = "1";
-$_SESSION['acl']['ratelimit'] = "1";
-$_SESSION['acl']['sogo_access'] = "1";
-$_SESSION['acl']['protocol_access'] = "1";
-$_SESSION['acl']['mailbox_relayhost'] = "1";
-$_SESSION['acl']['unlimited_quota'] = "1";
-
-$iam_settings = identity_provider('get');
-if ($iam_settings['authsource'] != "ldap" || (intval($iam_settings['periodic_sync']) != 1 && intval($iam_settings['import_users']) != 1)) {
-  session_destroy();
-  exit;
-}
-
-// Set pagination variables
-$start = 0;
-$max = 100;
-
-// lock sync if already running
-$lock_file = '/tmp/iam-sync.lock';
-if (file_exists($lock_file)) {
-  $lock_file_parts = explode("\n", file_get_contents($lock_file));
-  $pid = $lock_file_parts[0];
-  if (count($lock_file_parts) > 1){
-    $last_execution = $lock_file_parts[1];
-    $elapsed_time = (time() - $last_execution) / 60;
-    if ($elapsed_time < intval($iam_settings['sync_interval'])) {
-      logMsg("warning", "Sync not ready (".number_format((float)$elapsed_time, 2, '.', '')."min / ".$iam_settings['sync_interval']."min)");
-      session_destroy();
-      exit;
-    }
-  }
-
-  if (posix_kill($pid, 0)) {
-    logMsg("warning", "Sync is already running");
-    session_destroy();
-    exit;
-  } else {
-    unlink($lock_file);
-  }
-}
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid());
-fclose($lock_file_handle);
-
-// Init Provider
-$iam_provider = identity_provider('init');
-
-// Get ldap users
-$ldap_query = $iam_provider->query();
-if (!empty($iam_settings['filter'])) {
-  $ldap_query = $ldap_query->rawFilter($iam_settings['filter']);
-}
-$response = $ldap_query->where($iam_settings['username_field'], "*")
-  ->where($iam_settings['attribute_field'], "*")
-  ->select([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname'])
-  ->paginate($max);
-
-// Process the users
-foreach ($response as $user) {
-  // try get mailbox user
-  $stmt = $pdo->prepare("SELECT
-    mailbox.*,
-    domain.active AS d_active
-    FROM `mailbox`
-    INNER JOIN domain on mailbox.domain = domain.domain
-    WHERE `kind` NOT REGEXP 'location|thing|group'
-      AND `username` = :user");
-  $stmt->execute(array(':user' => $user[$iam_settings['username_field']][0]));
-  $row = $stmt->fetch(PDO::FETCH_ASSOC);
-
-  // check if matching attribute mapping exists
-  $user_template = $user[$iam_settings['attribute_field']][0];
-  $mapper_key = array_search($user_template, $iam_settings['mappers']);
-
-  if (empty($user[$iam_settings['username_field']][0])){
-    logMsg("warning", "Skipping user " . $user['displayname'][0] . " due to empty LDAP ". $iam_settings['username_field'] . " property.");
-    continue;
-  }
-
-  $_SESSION['access_all_exception'] = '1';
-  if (!$row && intval($iam_settings['import_users']) == 1){
-    if ($mapper_key === false){
-      if (!empty($iam_settings['default_template'])) {
-        $mbox_template = $iam_settings['default_template'];
-      } else {
-        logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-        continue;
-      }
-    } else {
-      $mbox_template = $iam_settings['templates'][$mapper_key];
-    }
-    // mailbox user does not exist, create...
-    logMsg("info", "Creating user " .  $user[$iam_settings['username_field']][0]);
-    $create_res = mailbox('add', 'mailbox_from_template', array(
-      'domain' => explode('@',  $user[$iam_settings['username_field']][0])[1],
-      'local_part' => explode('@',  $user[$iam_settings['username_field']][0])[0],
-      'name' => $user['displayname'][0],
-      'authsource' => 'ldap',
-      'template' => $mbox_template
-    ));
-    if (!$create_res){
-      logMsg("err", "Could not create user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-  } else if ($row && intval($iam_settings['periodic_sync']) == 1 && $row['authsource'] == "ldap") {
-    if ($mapper_key === false){
-      logMsg("warning", "No matching attribute mapping found for user " . $user[$iam_settings['username_field']][0]);
-      continue;
-    }
-    $mbox_template = $iam_settings['templates'][$mapper_key];
-    // mailbox user does exist, sync attribtues...
-    logMsg("info", "Syncing attributes for user " . $user[$iam_settings['username_field']][0]);
-    mailbox('edit', 'mailbox_from_template', array(
-      'username' =>  $user[$iam_settings['username_field']][0],
-      'name' => $user['displayname'][0],
-      'template' => $mbox_template
-    ));
-  } else {
-    // skip mailbox user
-    logMsg("info", "Skipping user " .  $user[$iam_settings['username_field']][0]);
-  }
-  $_SESSION['access_all_exception'] = '0';
-
-  sleep(0.025);
-}
-
-logMsg("info", "DONE!");
-// add last execution time to lock file
-$lock_file_handle = fopen($lock_file, 'w');
-fwrite($lock_file_handle, getmypid() . "\n" . time());
-fclose($lock_file_handle);
-session_destroy();

data/conf/phpfpm/php-conf.d/opcache-recommended.ini

@@ -1,7 +1,16 @@
+; NOTE: Restart phpfpm on ANY manual changes to PHP files!
+
+; opcache
 opcache.enable=1
 opcache.enable_cli=1
 opcache.interned_strings_buffer=16
 opcache.max_accelerated_files=10000
 opcache.memory_consumption=128
 opcache.save_comments=1
-opcache.revalidate_freq=1
+opcache.validate_timestamps=0
+
+; JIT
+; Disabled for now due to some PHP segmentation faults observed
+; in certain environments. Possibly some PHP or PHP extension bug.
+opcache.jit=disable
+opcache.jit_buffer_size=0

data/conf/phpfpm/php-conf.d/other.ini

@@ -1,3 +1,5 @@
 max_execution_time = 3600
 max_input_time = 3600
 memory_limit = 512M
+; not used by mailcow, disabled as hardening (PHP security release 2026-10-21)
+session.upload_progress.enabled = Off

data/conf/postfix/main.cf

@@ -37,7 +37,7 @@ postscreen_access_list = permit_mynetworks,
   cidr:/opt/postfix/conf/postscreen_access.cidr,
   tcp:127.0.0.1:10027
 postscreen_bare_newline_enable = no
-postscreen_blacklist_action = drop
+postscreen_denylist_action = drop
 postscreen_cache_cleanup_interval = 24h
 postscreen_cache_map = proxy:btree:$data_directory/postscreen_cache
 postscreen_dnsbl_action = enforce
@@ -107,8 +107,6 @@ smtpd_sender_restrictions = reject_authenticated_sender_login_mismatch,
   reject_unknown_sender_domain
 smtpd_soft_error_limit = 3
 smtpd_tls_auth_only = yes
-smtpd_tls_dh1024_param_file = /etc/ssl/mail/dhparams.pem
-smtpd_tls_eecdh_grade = auto
 smtpd_tls_exclude_ciphers = ECDHE-RSA-RC4-SHA, RC4, aNULL, DES-CBC3-SHA, ECDHE-RSA-DES-CBC3-SHA, EDH-RSA-DES-CBC3-SHA
 smtpd_tls_loglevel = 1
 
@@ -152,7 +150,7 @@ smtp_sasl_auth_enable = yes
 smtp_sasl_password_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_sender_dependent.cf
 smtp_sasl_security_options =
 smtp_sasl_mechanism_filter = plain, login
-smtp_tls_policy_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf
+smtp_tls_policy_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_tls_policy_override_maps.cf socketmap:inet:postfix-tlspol:8642:QUERYwithTLSRPT
 smtp_header_checks = pcre:/opt/postfix/conf/anonymize_headers.pcre
 mail_name = Postcow
 # local_transport map catches local destinations and prevents routing local dests when the next map would route "*"

data/conf/postfix/master.cf

@@ -29,7 +29,6 @@ smtps    inet  n       -       n       -       -       smtpd
 # TLS protocol can be modified by setting submission_smtpd_tls_mandatory_protocols in extra.cf
 submission inet n       -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
-  -o smtpd_enforce_tls=yes
   -o smtpd_tls_security_level=encrypt
   -o smtpd_tls_mandatory_protocols=$submission_smtpd_tls_mandatory_protocols
   -o tls_preempt_cipherlist=yes
@@ -38,7 +37,6 @@ submission inet n       -       n       -       -       smtpd
 10587      inet n       -       n       -       -       smtpd
   -o smtpd_upstream_proxy_protocol=haproxy
   -o smtpd_client_restrictions=permit_mynetworks,permit_sasl_authenticated,reject
-  -o smtpd_enforce_tls=yes
   -o smtpd_tls_security_level=encrypt
   -o smtpd_tls_mandatory_protocols=$submission_smtpd_tls_mandatory_protocols
   -o tls_preempt_cipherlist=yes

data/conf/rspamd/dynmaps/aliasexp.php

@@ -133,7 +133,7 @@ try {
             error_log("ALIAS EXPANDER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
             $goto_branch_array = explode(',', $goto_branch);
           } else {
-            $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` AND '1'");
+            $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'");
             $stmt->execute(array(':domain' => $parsed_goto['domain']));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['target_domain'];
             if ($goto_branch) {

data/conf/rspamd/dynmaps/settings.php

@@ -56,7 +56,7 @@ function normalize_email($email) {
     $email = explode('@', $email);
     $email[0] = str_replace('.', '', $email[0]);
     $email = implode('@', $email);
-  } 
+  }
   $gm_alt = "@googlemail.com";
   if (substr_compare($email, $gm_alt, -strlen($gm_alt)) == 0) {
     $email = explode('@', $email);
@@ -114,7 +114,7 @@ function ucl_rcpts($object, $type) {
       $rcpt[] = str_replace('/', '\/', $row['address']);
     }
     // Aliases by alias domains
-    $stmt = $pdo->prepare("SELECT CONCAT(`local_part`, '@', `alias_domain`.`alias_domain`) AS `alias` FROM `mailbox` 
+    $stmt = $pdo->prepare("SELECT CONCAT(`local_part`, '@', `alias_domain`.`alias_domain`) AS `alias` FROM `mailbox`
       LEFT OUTER JOIN `alias_domain` ON `mailbox`.`domain` = `alias_domain`.`target_domain`
       WHERE `mailbox`.`username` = :object");
     $stmt->execute(array(
@@ -184,7 +184,7 @@ while ($row = array_shift($rows)) {
     rcpt = <?=json_encode($rcpt, JSON_UNESCAPED_SLASHES);?>;
 <?php
   }
-  $stmt = $pdo->prepare("SELECT `option`, `value` FROM `filterconf` 
+  $stmt = $pdo->prepare("SELECT `option`, `value` FROM `filterconf`
     WHERE (`option` = 'highspamlevel' OR `option` = 'lowspamlevel')
       AND `object`= :object");
   $stmt->execute(array(':object' => $row['object']));
@@ -468,4 +468,36 @@ while ($row = array_shift($rows)) {
 <?php
 }
 ?>
+
+<?php
+// Start internal aliases
+
+$stmt = $pdo->query("SELECT `id`, `address`, `domain` FROM `alias` WHERE `active` = '1' AND `internal` = '1'");
+$aliases = $stmt->fetchAll(PDO::FETCH_ASSOC);
+while ($alias = array_shift($aliases)) {
+  // build allowed_domains regex and add target domain and alias domains
+  $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `active` = '1' AND `target_domain` = :target_domain");
+  $stmt->execute(array(':target_domain' => $alias['domain']));
+  $allowed_domains = $stmt->fetchAll(PDO::FETCH_ASSOC);
+  $allowed_domains = array_map(function($item) {
+    return str_replace('.', '\.', $item['alias_domain']);
+  }, $allowed_domains);
+  $allowed_domains[] = str_replace('.', '\.', $alias['domain']);
+  $allowed_domains = implode('|', $allowed_domains);
+?>
+  internal_alias_<?=$alias['id'];?> {
+    priority = 10;
+    rcpt = "<?=$alias['address'];?>";
+    from = "/^((?!.*@(<?=$allowed_domains;?>)).)*$/";
+    apply "default" {
+      MAILCOW_INTERNAL_ALIAS = 9999.0;
+    }
+    symbols [
+      "MAILCOW_INTERNAL_ALIAS"
+    ]
+  }
+<?php
+}
+?>
+
 }

data/conf/rspamd/local.d/metadata_exporter.conf

@@ -3,8 +3,7 @@ rules {
     backend = "http";
     url = "http://nginx:9081/pipe.php";
     selector = "reject_no_global_bl";
-    formatter = "default";
-    meta_headers = true;
+    formatter = "multipart";
   }
 	RLINFO {
 		backend = "http";
@@ -16,8 +15,7 @@ rules {
     backend = "http";
     url = "http://nginx:9081/pushover.php";
     selector = "mailcow_rcpt";
-    formatter = "json";
-    meta_headers = true;
+    formatter = "multipart";
   }
 }
 

data/conf/rspamd/lua/rspamd.local.lua

@@ -102,7 +102,7 @@ rspamd_config:register_symbol({
       local rcpt_split = rspamd_str_split(rcpt['addr'], '@')
       if #rcpt_split == 2 then
         if rcpt_split[1] == 'postmaster' then
-          task:set_pre_result('accept', 'whitelisting postmaster smtp rcpt')
+          task:set_pre_result('accept', 'whitelisting postmaster smtp rcpt', 'postmaster')
           return
         end
       end
@@ -146,8 +146,171 @@ rspamd_config:register_symbol({
       return false
     end
 
+    -- Helper function to parse IPv6 into 8 segments
+    local function ipv6_to_segments(ip_str)
+      -- Remove zone identifier if present (e.g., %eth0)
+      ip_str = ip_str:gsub("%%.*$", "")
+      
+      local segments = {}
+      
+      -- Handle :: compression
+      if ip_str:find('::') then
+        local before, after = ip_str:match('^(.*)::(.*)$')
+        before = before or ''
+        after = after or ''
+        
+        local before_parts = {}
+        local after_parts = {}
+        
+        if before ~= '' then
+          for seg in before:gmatch('[^:]+') do
+            table.insert(before_parts, tonumber(seg, 16) or 0)
+          end
+        end
+        
+        if after ~= '' then
+          for seg in after:gmatch('[^:]+') do
+            table.insert(after_parts, tonumber(seg, 16) or 0)
+          end
+        end
+        
+        -- Add before segments
+        for _, seg in ipairs(before_parts) do
+          table.insert(segments, seg)
+        end
+        
+        -- Add compressed zeros
+        local zeros_needed = 8 - #before_parts - #after_parts
+        for i = 1, zeros_needed do
+          table.insert(segments, 0)
+        end
+        
+        -- Add after segments
+        for _, seg in ipairs(after_parts) do
+          table.insert(segments, seg)
+        end
+      else
+        -- No compression
+        for seg in ip_str:gmatch('[^:]+') do
+          table.insert(segments, tonumber(seg, 16) or 0)
+        end
+      end
+      
+      -- Ensure we have exactly 8 segments
+      while #segments < 8 do
+        table.insert(segments, 0)
+      end
+      
+      return segments
+    end
+
+    -- Generate all common IPv6 notations
+    local function get_ipv6_variants(ip_str)
+      local variants = {}
+      local seen = {}
+      
+      local function add_variant(v)
+        if v and not seen[v] then
+          table.insert(variants, v)
+          seen[v] = true
+        end
+      end
+      
+      -- For IPv4, just return the original
+      if not ip_str:find(':') then
+        add_variant(ip_str)
+        return variants
+      end
+      
+      local segments = ipv6_to_segments(ip_str)
+      
+      -- 1. Fully expanded form (all zeros shown as 0000)
+      local expanded_parts = {}
+      for _, seg in ipairs(segments) do
+        table.insert(expanded_parts, string.format('%04x', seg))
+      end
+      add_variant(table.concat(expanded_parts, ':'))
+      
+      -- 2. Standard form (no leading zeros, but all segments present)
+      local standard_parts = {}
+      for _, seg in ipairs(segments) do
+        table.insert(standard_parts, string.format('%x', seg))
+      end
+      add_variant(table.concat(standard_parts, ':'))
+      
+      -- 3. Find all possible :: compressions
+      -- RFC 5952: compress the longest run of consecutive zeros
+      -- But we need to check all possibilities since Redis might have any form
+      
+      -- Find all zero runs
+      local zero_runs = {}
+      local in_run = false
+      local run_start = 0
+      local run_length = 0
+      
+      for i = 1, 8 do
+        if segments[i] == 0 then
+          if not in_run then
+            in_run = true
+            run_start = i
+            run_length = 1
+          else
+            run_length = run_length + 1
+          end
+        else
+          if in_run then
+            if run_length >= 1 then  -- Allow single zero compression too
+              table.insert(zero_runs, {start = run_start, length = run_length})
+            end
+            in_run = false
+          end
+        end
+      end
+      
+      -- Don't forget the last run
+      if in_run and run_length >= 1 then
+        table.insert(zero_runs, {start = run_start, length = run_length})
+      end
+      
+      -- Generate variant for each zero run compression
+      for _, run in ipairs(zero_runs) do
+        local parts = {}
+        
+        -- Before compression
+        for i = 1, run.start - 1 do
+          table.insert(parts, string.format('%x', segments[i]))
+        end
+        
+        -- The compression
+        if run.start == 1 then
+          table.insert(parts, '')
+          table.insert(parts, '')
+        elseif run.start + run.length - 1 == 8 then
+          table.insert(parts, '')
+          table.insert(parts, '')
+        else
+          table.insert(parts, '')
+        end
+        
+        -- After compression
+        for i = run.start + run.length, 8 do
+          table.insert(parts, string.format('%x', segments[i]))
+        end
+        
+        local compressed = table.concat(parts, ':'):gsub('::+', '::')
+        add_variant(compressed)
+      end
+      
+      return variants
+    end
+
     local from_ip_string = tostring(ip)
-    ip_check_table = {from_ip_string}
+    local ip_check_table = {}
+    
+    -- Add all variants of the exact IP
+    for _, variant in ipairs(get_ipv6_variants(from_ip_string)) do
+      table.insert(ip_check_table, variant)
+    end
 
     local maxbits = 128
     local minbits = 32
@@ -155,10 +318,18 @@ rspamd_config:register_symbol({
         maxbits = 32
         minbits = 8
     end
+    
+    -- Add all CIDR notations with variants
     for i=maxbits,minbits,-1 do
-      local nip = ip:apply_mask(i):to_string() .. "/" .. i
-      table.insert(ip_check_table, nip)
+      local masked_ip = ip:apply_mask(i)
+      local cidr_base = masked_ip:to_string()
+      
+      for _, variant in ipairs(get_ipv6_variants(cidr_base)) do
+        local cidr = variant .. "/" .. i
+        table.insert(ip_check_table, cidr)
+      end
     end
+    
     local function keep_spam_cb(err, data)
       if err then
         rspamd_logger.infox(rspamd_config, "keep_spam query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
@@ -166,12 +337,15 @@ rspamd_config:register_symbol({
       else
         for k,v in pairs(data) do
           if (v and v ~= userdata and v == '1') then
-            rspamd_logger.infox(rspamd_config, "found ip in keep_spam map, setting pre-result")
-            task:set_pre_result('accept', 'ip matched with forward hosts')
+            rspamd_logger.infox(rspamd_config, "found ip %s (checked as: %s) in keep_spam map, setting pre-result accept", from_ip_string, ip_check_table[k])
+            task:set_pre_result('accept', 'ip matched with forward hosts', 'keep_spam')
+            task:set_flag('no_stat')
+            return
           end
         end
       end
     end
+    
     table.insert(ip_check_table, 1, 'KEEP_SPAM')
     local redis_ret_user = rspamd_redis_make_request(task,
       redis_params, -- connect params
@@ -210,6 +384,7 @@ rspamd_config:register_symbol({
 rspamd_config:register_symbol({
   name = 'TAG_MOO',
   type = 'postfilter',
+  flags = 'ignore_passthrough',
   callback = function(task)
     local util = require("rspamd_util")
     local rspamd_logger = require "rspamd_logger"
@@ -217,9 +392,7 @@ rspamd_config:register_symbol({
     local rspamd_http = require "rspamd_http"
     local rcpts = task:get_recipients('smtp')
     local lua_util = require "lua_util"
-
     local tagged_rcpt = task:get_symbol("TAGGED_RCPT")
-    local mailcow_domain = task:get_symbol("RCPT_MAILCOW_DOMAIN")
 
     local function remove_moo_tag()
       local moo_tag_header = task:get_header('X-Moo-Tag', false)
@@ -231,101 +404,147 @@ rspamd_config:register_symbol({
       return true
     end
 
-    if tagged_rcpt and tagged_rcpt[1].options and mailcow_domain then
-      local tag = tagged_rcpt[1].options[1]
-      rspamd_logger.infox("found tag: %s", tag)
-      local action = task:get_metric_action('default')
-      rspamd_logger.infox("metric action now: %s", action)
+    -- Check if we have exactly one recipient
+    if not (rcpts and #rcpts == 1) then
+      rspamd_logger.infox("TAG_MOO: not exactly one rcpt (%s), removing moo tag", rcpts and #rcpts or 0)
+      remove_moo_tag()
+      return
+    end
 
-      if action ~= 'no action' and action ~= 'greylist' then
-        rspamd_logger.infox("skipping tag handler for action: %s", action)
-        remove_moo_tag()
-        return true
+    local rcpt_addr = rcpts[1]['addr']
+    local rcpt_user = rcpts[1]['user']
+    local rcpt_domain = rcpts[1]['domain']
+
+    -- Check if recipient has a tag (contains '+')
+    local tag = nil
+    if tagged_rcpt ~= nil then
+      tag = tagged_rcpt
+      rspamd_logger.infox("TAG_MOO: found tag in recipient: %s (base: %s, tag: %s)", rcpt_addr, base_user, tag)
+    end
+
+    if not tag then
+      rspamd_logger.infox("TAG_MOO: no tag found in recipient %s, removing moo tag", rcpt_addr)
+      remove_moo_tag()
+      return
+    end
+
+    -- Optional: Check if domain is a mailcow domain
+    -- When KEEP_SPAM is active, RCPT_MAILCOW_DOMAIN might not be set
+    -- If the mail is being delivered, we can assume it's valid
+    local mailcow_domain = task:get_symbol("RCPT_MAILCOW_DOMAIN")
+    if not mailcow_domain then
+      rspamd_logger.infox("TAG_MOO: RCPT_MAILCOW_DOMAIN not set (possibly due to pre-result), proceeding anyway for domain %s", rcpt_domain)
+    end
+
+    local action = task:get_metric_action('default')
+    rspamd_logger.infox("TAG_MOO: metric action: %s", action)
+
+    -- Check if we have a pre-result (e.g., from KEEP_SPAM or POSTMASTER_HANDLER)
+    local allow_processing = false
+    
+    if task.has_pre_result then
+      local has_pre, pre_action = task:has_pre_result()
+      if has_pre then
+        rspamd_logger.infox("TAG_MOO: pre-result detected: %s", tostring(pre_action))
+        if pre_action == 'accept' then
+          allow_processing = true
+          rspamd_logger.infox("TAG_MOO: pre-result is accept, will process")
+        end
       end
+    end
 
-      local function http_callback(err_message, code, body, headers)
-        if body ~= nil and body ~= "" then
-          rspamd_logger.infox(rspamd_config, "expanding rcpt to \"%s\"", body)
+    -- Allow processing for mild actions or when we have pre-result accept
+    if not allow_processing and action ~= 'no action' and action ~= 'greylist' then
+      rspamd_logger.infox("TAG_MOO: skipping tag handler for action: %s", action)
+      remove_moo_tag()
+      return true
+    end
 
-          local function tag_callback_subject(err, data)
-            if err or type(data) ~= 'string' then
-              rspamd_logger.infox(rspamd_config, "subject tag handler rcpt %s returned invalid or empty data (\"%s\") or error (\"%s\") - trying subfolder tag handler...", body, data, err)
+    rspamd_logger.infox("TAG_MOO: processing allowed")
 
-              local function tag_callback_subfolder(err, data)
-                if err or type(data) ~= 'string' then
-                  rspamd_logger.infox(rspamd_config, "subfolder tag handler for rcpt %s returned invalid or empty data (\"%s\") or error (\"%s\")", body, data, err)
-                  remove_moo_tag()
-                else
-                  rspamd_logger.infox("Add X-Moo-Tag header")
-                  task:set_milter_reply({
-                    add_headers = {['X-Moo-Tag'] = 'YES'}
-                  })
-                end
-              end
+    local function http_callback(err_message, code, body, headers)
+      if body ~= nil and body ~= "" then
+        rspamd_logger.infox(rspamd_config, "TAG_MOO: expanding rcpt to \"%s\"", body)
 
-              local redis_ret_subfolder = rspamd_redis_make_request(task,
-                redis_params, -- connect params
-                body, -- hash key
-                false, -- is write
-                tag_callback_subfolder, --callback
-                'HGET', -- command
-                {'RCPT_WANTS_SUBFOLDER_TAG', body} -- arguments
-              )
-              if not redis_ret_subfolder then
-                rspamd_logger.infox(rspamd_config, "cannot make request to load tag handler for rcpt")
+        local function tag_callback_subject(err, data)
+          if err or type(data) ~= 'string' or data == '' then
+            rspamd_logger.infox(rspamd_config, "TAG_MOO: subject tag handler rcpt %s returned invalid or empty data (\"%s\") or error (\"%s\") - trying subfolder tag handler...", body, data, err)
+
+            local function tag_callback_subfolder(err, data)
+              if err or type(data) ~= 'string' or data == '' then
+                rspamd_logger.infox(rspamd_config, "TAG_MOO: subfolder tag handler for rcpt %s returned invalid or empty data (\"%s\") or error (\"%s\")", body, data, err)
                 remove_moo_tag()
+              else
+                rspamd_logger.infox("TAG_MOO: User wants subfolder tag, adding X-Moo-Tag header")
+                task:set_milter_reply({
+                  add_headers = {['X-Moo-Tag'] = 'YES'}
+                })
               end
-
-            else
-              rspamd_logger.infox("user wants subject modified for tagged mail")
-              local sbj = task:get_header('Subject')
-              new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag .. '] ' .. sbj)) .. '?='
-              task:set_milter_reply({
-                remove_headers = {
-                  ['Subject'] = 1,
-                  ['X-Moo-Tag'] = 0
-                },
-                add_headers = {['Subject'] = new_sbj}
-              })
             end
-          end
 
-          local redis_ret_subject = rspamd_redis_make_request(task,
-            redis_params, -- connect params
-            body, -- hash key
-            false, -- is write
-            tag_callback_subject, --callback
-            'HGET', -- command
-            {'RCPT_WANTS_SUBJECT_TAG', body} -- arguments
-          )
-          if not redis_ret_subject then
-            rspamd_logger.infox(rspamd_config, "cannot make request to load tag handler for rcpt")
-            remove_moo_tag()
-          end
-
-        end
-      end
-
-      if rcpts and #rcpts == 1 then
-        for _,rcpt in ipairs(rcpts) do
-          local rcpt_split = rspamd_str_split(rcpt['addr'], '@')
-          if #rcpt_split == 2 then
-            if rcpt_split[1] == 'postmaster' then
-              rspamd_logger.infox(rspamd_config, "not expanding postmaster alias")
+            local redis_ret_subfolder = rspamd_redis_make_request(task,
+              redis_params, -- connect params
+              body, -- hash key
+              false, -- is write
+              tag_callback_subfolder, --callback
+              'HGET', -- command
+              {'RCPT_WANTS_SUBFOLDER_TAG', body} -- arguments
+            )
+            if not redis_ret_subfolder then
+              rspamd_logger.infox(rspamd_config, "TAG_MOO: cannot make request to load tag handler for rcpt")
               remove_moo_tag()
-            else
-              rspamd_http.request({
-                task=task,
-                url='http://nginx:8081/aliasexp.php',
-                body='',
-                callback=http_callback,
-                headers={Rcpt=rcpt['addr']},
-              })
             end
+
+          else
+            rspamd_logger.infox("TAG_MOO: user wants subject modified for tagged mail")
+            local sbj = task:get_header('Subject') or ''
+            local tag_value = tag[1] and tag[1].options and tag[1].options[1] or ''
+            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag_value .. '] ' .. sbj)) .. '?='
+            task:set_milter_reply({
+              remove_headers = {
+                ['Subject'] = 1,
+                ['X-Moo-Tag'] = 0
+              },
+              add_headers = {['Subject'] = new_sbj}
+            })
           end
         end
+
+        local redis_ret_subject = rspamd_redis_make_request(task,
+          redis_params, -- connect params
+          body, -- hash key
+          false, -- is write
+          tag_callback_subject, --callback
+          'HGET', -- command
+          {'RCPT_WANTS_SUBJECT_TAG', body} -- arguments
+        )
+        if not redis_ret_subject then
+          rspamd_logger.infox(rspamd_config, "TAG_MOO: cannot make request to load tag handler for rcpt")
+          remove_moo_tag()
+        end
+      else
+        rspamd_logger.infox("TAG_MOO: alias expansion returned empty body")
+        remove_moo_tag()
+      end
+    end
+
+    local rcpt_split = rspamd_str_split(rcpt_addr, '@')
+    if #rcpt_split == 2 then
+      if rcpt_split[1]:match('^postmaster') then
+        rspamd_logger.infox(rspamd_config, "TAG_MOO: not expanding postmaster alias")
+        remove_moo_tag()
+      else
+        rspamd_logger.infox("TAG_MOO: requesting alias expansion for %s", rcpt_addr)
+        rspamd_http.request({
+          task=task,
+          url='http://nginx:8081/aliasexp.php',
+          body='',
+          callback=http_callback,
+          headers={Rcpt=rcpt_addr},
+        })
       end
     else
+      rspamd_logger.infox("TAG_MOO: invalid rcpt format")
       remove_moo_tag()
     end
   end,
@@ -335,6 +554,7 @@ rspamd_config:register_symbol({
 rspamd_config:register_symbol({
   name = 'BCC',
   type = 'postfilter',
+  flags = 'ignore_passthrough',
   callback = function(task)
     local util = require("rspamd_util")
     local rspamd_http = require "rspamd_http"
@@ -363,11 +583,13 @@ rspamd_config:register_symbol({
       local email_content = tostring(task:get_content())
       email_content = string.gsub(email_content, "\r\n%.", "\r\n..")
       -- send mail
+      local from_smtp = task:get_from('smtp')
+      local from_addr = (from_smtp and from_smtp[1] and from_smtp[1].addr) or 'mailer-daemon@localhost'
       lua_smtp.sendmail({
         task = task,
         host = os.getenv("IPV4_NETWORK") .. '.253',
         port = 591,
-        from = task:get_from(stp)[1].addr,
+        from = from_addr,
         recipients = bcc_dest,
         helo = 'bcc',
         timeout = 20,
@@ -397,27 +619,41 @@ rspamd_config:register_symbol({
     end
 
     local action = task:get_metric_action('default')
-    rspamd_logger.infox("metric action now: %s", action)
+    rspamd_logger.infox("BCC: metric action: %s", action)
+
+    -- Check for pre-result accept (e.g., from KEEP_SPAM)
+    local allow_bcc = false
+    if task.has_pre_result then
+      local has_pre, pre_action = task:has_pre_result()
+      if has_pre and pre_action == 'accept' then
+        allow_bcc = true
+        rspamd_logger.infox("BCC: pre-result accept detected, will send BCC")
+      end
+    end
+
+    -- Allow BCC for mild actions or when we have pre-result accept
+    if not allow_bcc and action ~= 'no action' and action ~= 'add header' and action ~= 'rewrite subject' then
+      rspamd_logger.infox("BCC: skipping for action: %s", action)
+      return
+    end
 
     local function rcpt_callback(err_message, code, body, headers)
       if err_message == nil and code == 201 and body ~= nil then
-        if action == 'no action' or action == 'add header' or action == 'rewrite subject' then
-          send_mail(task, body)
-        end
+        rspamd_logger.infox("BCC: sending BCC to %s for rcpt match", body)
+        send_mail(task, body)
       end
     end
 
     local function from_callback(err_message, code, body, headers)
       if err_message == nil and code == 201 and body ~= nil then
-        if action == 'no action' or action == 'add header' or action == 'rewrite subject' then
-          send_mail(task, body)
-        end
+        rspamd_logger.infox("BCC: sending BCC to %s for from match", body)
+        send_mail(task, body)
       end
     end
 
     if rcpt_table then
       for _,e in ipairs(rcpt_table) do
-        rspamd_logger.infox(rspamd_config, "checking bcc for rcpt address %s", e)
+        rspamd_logger.infox(rspamd_config, "BCC: checking bcc for rcpt address %s", e)
         rspamd_http.request({
           task=task,
           url='http://nginx:8081/bcc.php',
@@ -430,7 +666,7 @@ rspamd_config:register_symbol({
 
     if from_table then
       for _,e in ipairs(from_table) do
-        rspamd_logger.infox(rspamd_config, "checking bcc for from address %s", e)
+        rspamd_logger.infox(rspamd_config, "BCC: checking bcc for from address %s", e)
         rspamd_http.request({
           task=task,
           url='http://nginx:8081/bcc.php',
@@ -441,7 +677,7 @@ rspamd_config:register_symbol({
       end
     end
 
-    return true
+    -- Don't return true to avoid symbol being logged
   end,
   priority = 20
 })
@@ -454,12 +690,18 @@ rspamd_config:register_symbol({
     local redis_params = rspamd_parse_redis_server('dyn_rl')
     local rspamd_logger = require "rspamd_logger"
     local envfrom = task:get_from(1)
+    local envrcpt = task:get_recipients(1) or {}
     local uname = task:get_user()
     if not envfrom or not uname then
       return false
     end
+
     local uname = uname:lower()
 
+    if #envrcpt == 1 and envrcpt[1].addr:lower() == uname then
+      return false
+    end
+
     local env_from_domain = envfrom[1].domain:lower() -- get smtp from domain in lower case
 
     local function redis_cb_user(err, data)
@@ -544,13 +786,13 @@ rspamd_config:register_symbol({
     -- determine newline type
     local function newline(task)
       local t = task:get_newlines_type()
-    
+
       if t == 'cr' then
         return '\r'
       elseif t == 'lf' then
         return '\n'
       end
-    
+
       return '\r\n'
     end
     -- retrieve footer
@@ -558,7 +800,7 @@ rspamd_config:register_symbol({
       if err or type(data) ~= 'string' then
         rspamd_logger.infox(rspamd_config, "domain wide footer request for user %s returned invalid or empty data (\"%s\") or error (\"%s\")", uname, data, err)
       else
-        
+
         -- parse json string
         local footer = cjson.decode(data)
         if not footer then
@@ -607,37 +849,43 @@ rspamd_config:register_symbol({
             if footer.plain and footer.plain ~= "" then
               footer.plain = lua_util.jinja_template(footer.plain, replacements, true)
             end
-  
+
             -- add footer
             local out = {}
             local rewrite = lua_mime.add_text_footer(task, footer.html, footer.plain) or {}
-        
+
             local seen_cte
             local newline_s = newline(task)
-        
+            -- use the CTE add_text_footer actually applied
+            local new_cte = rewrite.new_cte or 'quoted-printable'
+
             local function rewrite_ct_cb(name, hdr)
               if rewrite.need_rewrite_ct then
                 if name:lower() == 'content-type' then
-                  local nct = string.format('%s: %s/%s; charset=utf-8',
-                      'Content-Type', rewrite.new_ct.type, rewrite.new_ct.subtype)
+                  -- include boundary if present
+                  local boundary_part = rewrite.new_ct.boundary and
+                    string.format('; boundary="%s"', rewrite.new_ct.boundary) or ''
+                  local nct = string.format('%s: %s/%s; charset=utf-8%s',
+                      'Content-Type', rewrite.new_ct.type, rewrite.new_ct.subtype, boundary_part)
                   out[#out + 1] = nct
-                  -- update Content-Type header
+                  -- update Content-Type header (include boundary if present)
                   task:set_milter_reply({
                     remove_headers = {['Content-Type'] = 0},
                   })
                   task:set_milter_reply({
-                    add_headers = {['Content-Type'] = string.format('%s/%s; charset=utf-8', rewrite.new_ct.type, rewrite.new_ct.subtype)}
+                    add_headers = {['Content-Type'] = string.format('%s/%s; charset=utf-8%s',
+                      rewrite.new_ct.type, rewrite.new_ct.subtype, boundary_part)}
                   })
                   return
                 elseif name:lower() == 'content-transfer-encoding' then
                   out[#out + 1] = string.format('%s: %s',
-                      'Content-Transfer-Encoding', 'quoted-printable')
+                      'Content-Transfer-Encoding', new_cte)
                   -- update Content-Transfer-Encoding header
                   task:set_milter_reply({
                     remove_headers = {['Content-Transfer-Encoding'] = 0},
                   })
                   task:set_milter_reply({
-                    add_headers = {['Content-Transfer-Encoding'] = 'quoted-printable'}
+                    add_headers = {['Content-Transfer-Encoding'] = new_cte}
                   })
                   seen_cte = true
                   return
@@ -645,16 +893,16 @@ rspamd_config:register_symbol({
               end
               out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
             end
-        
+
             task:headers_foreach(rewrite_ct_cb, {full = true})
-        
+
             if not seen_cte and rewrite.need_rewrite_ct then
-              out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', 'quoted-printable')
+              out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', new_cte)
             end
-        
+
             -- End of headers
             out[#out + 1] = newline_s
-        
+
             if rewrite.out then
               for _,o in ipairs(rewrite.out) do
                 out[#out + 1] = o
@@ -668,7 +916,7 @@ rspamd_config:register_symbol({
                 out_parts[#out_parts + 1] = o
                 out_parts[#out_parts + 1] = newline_s
               else
-                local removePrefix = "--\x0D\x0AContent-Type"
+                local removePrefix = "--\x0D\x0AContent-"
                 if string.lower(string.sub(tostring(o[1]), 1, string.len(removePrefix))) == string.lower(removePrefix) then
                   o[1] = string.sub(tostring(o[1]), string.len("--\x0D\x0A") + 1)
                 end

data/conf/rspamd/meta_exporter/pipe.php

@@ -32,47 +32,42 @@ function parse_email($email) {
   $a = strrpos($email, '@');
   return array('local' => substr($email, 0, $a), 'domain' => substr(substr($email, $a), 1));
 }
-if (!function_exists('getallheaders'))  {
-  function getallheaders() {
-    if (!is_array($_SERVER)) {
-      return array();
-    }
-    $headers = array();
-    foreach ($_SERVER as $name => $value) {
-      if (substr($name, 0, 5) == 'HTTP_') {
-        $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value;
-      }
-    }
-    return $headers;
-  }
+// rspamd metadata_exporter (multipart formatter):
+//   - $_POST['metadata']        JSON with the rspamd metadata
+//   - $_FILES['message']        raw RFC822 message
+if (empty($_POST['metadata']) || !isset($_FILES['message']) || $_FILES['message']['error'] !== UPLOAD_ERR_OK) {
+  error_log("QUARANTINE: missing multipart parts from rspamd" . PHP_EOL);
+  http_response_code(400);
+  exit;
 }
 
-$raw_data_content = file_get_contents('php://input');
+$meta = json_decode($_POST['metadata'], true);
+if (!is_array($meta)) {
+  error_log("QUARANTINE: cannot decode metadata JSON" . PHP_EOL);
+  http_response_code(400);
+  exit;
+}
+
+$raw_data_content = file_get_contents($_FILES['message']['tmp_name']);
 $raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
-$headers = getallheaders();
+$raw_size = (int)$_FILES['message']['size'];
 
-$qid      = $headers['X-Rspamd-Qid'];
-$fuzzy    = $headers['X-Rspamd-Fuzzy'];
-$subject  = iconv_mime_decode($headers['X-Rspamd-Subject']);
-$score    = $headers['X-Rspamd-Score'];
-$rcpts    = $headers['X-Rspamd-Rcpt'];
-$user     = $headers['X-Rspamd-User'];
-$ip       = $headers['X-Rspamd-Ip'];
-$action   = $headers['X-Rspamd-Action'];
-$sender   = $headers['X-Rspamd-From'];
-$symbols  = $headers['X-Rspamd-Symbols'];
-
-$raw_size = (int)$_SERVER['CONTENT_LENGTH'];
+$qid      = $meta['qid']     ?? 'unknown';
+$subject  = $meta['subject'] ?? '';
+$score    = $meta['score']   ?? 0;
+$rcpts    = $meta['rcpt']    ?? array();
+$user     = $meta['user']    ?? 'unknown';
+$ip       = $meta['ip']      ?? 'unknown';
+$action   = $meta['action']  ?? 'no action';
+$sender   = $meta['from']    ?? '';
+$symbols  = json_encode($meta['symbols'] ?? array());
+$fuzzy    = json_encode(is_array($meta['fuzzy'] ?? null) ? $meta['fuzzy'] : array());
 
 if (empty($sender)) {
   error_log("QUARANTINE: Unknown sender, assuming empty-env-from@localhost" . PHP_EOL);
   $sender = 'empty-env-from@localhost';
 }
 
-if ($fuzzy == 'unknown') {
-  $fuzzy = '[]';
-}
-
 try {
   $max_size = (int)$redis->Get('Q_MAX_SIZE');
   if (($max_size * 1048576) < $raw_size) {
@@ -94,7 +89,7 @@ catch (RedisException $e) {
 $rcpt_final_mailboxes = array();
 
 // Loop through all rcpts
-foreach (json_decode($rcpts, true) as $rcpt) {
+foreach ($rcpts as $rcpt) {
   // Remove tag
   $rcpt = preg_replace('/^(.*?)\+.*(@.*)$/', '$1$2', $rcpt);
 
@@ -158,12 +153,12 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 
       // Loop through all found gotos
       foreach ($gotos_array as $index => &$goto) {
-        error_log("RCPT RESOVLER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
+        error_log("RCPT RESOLVER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
         $stmt = $pdo->prepare("SELECT `username` FROM `mailbox` WHERE `username` = :goto AND (`active`= '1' OR `active`= '2');");
         $stmt->execute(array(':goto' => $goto));
         $username = $stmt->fetch(PDO::FETCH_ASSOC)['username'];
         if (!empty($username)) {
-          error_log("RCPT RESOVLER: http pipe: mailbox found: " . $username . PHP_EOL);
+          error_log("RCPT RESOLVER: http pipe: mailbox found: " . $username . PHP_EOL);
           // Current goto is a mailbox, save to rcpt_final_mailboxes if not a duplicate
           if (!in_array($username, $rcpt_final_mailboxes)) {
             $rcpt_final_mailboxes[] = $username;
@@ -172,21 +167,21 @@ foreach (json_decode($rcpts, true) as $rcpt) {
         else {
           $parsed_goto = parse_email($goto);
           if (!$redis->hGet('DOMAIN_MAP', $parsed_goto['domain'])) {
-            error_log("RCPT RESOVLER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
+            error_log("RCPT RESOLVER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
           }
           else {
             $stmt = $pdo->prepare("SELECT `goto` FROM `alias` WHERE `address` = :goto AND `active` = '1'");
             $stmt->execute(array(':goto' => $goto));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
             if ($goto_branch) {
-              error_log("RCPT RESOVLER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
+              error_log("RCPT RESOLVER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
               $goto_branch_array = explode(',', $goto_branch);
             } else {
-              $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` AND '1'");
+              $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'");
               $stmt->execute(array(':domain' => $parsed_goto['domain']));
               $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['target_domain'];
               if ($goto_branch) {
-                error_log("RCPT RESOVLER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
+                error_log("RCPT RESOLVER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
                 $goto_branch_array = array($parsed_goto['local'] . '@' . $goto_branch);
               }
             }
@@ -208,11 +203,11 @@ foreach (json_decode($rcpts, true) as $rcpt) {
       // Force exit if loop cannot be solved
       // Postfix does not allow for alias loops, so this should never happen.
       $loop_c++;
-      error_log("RCPT RESOVLER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
+      error_log("RCPT RESOLVER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
     }
   }
   catch (PDOException $e) {
-    error_log("RCPT RESOVLER: " . $e->getMessage() . PHP_EOL);
+    error_log("RCPT RESOLVER: " . $e->getMessage() . PHP_EOL);
     http_response_code(502);
     exit;
   }
@@ -236,6 +231,9 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
       ':action' => $action,
       ':fuzzy_hashes' => $fuzzy
     ));
+    $lastId = $pdo->lastInsertId();
+    $stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
+    $stmt_update->execute(array(':id' => $lastId));
     $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
       SELECT `id`
       FROM (

data/conf/rspamd/meta_exporter/pushover.php

@@ -32,50 +32,46 @@ function parse_email($email) {
   $a = strrpos($email, '@');
   return array('local' => substr($email, 0, $a), 'domain' => substr(substr($email, $a), 1));
 }
-if (!function_exists('getallheaders'))  {
-  function getallheaders() {
-    if (!is_array($_SERVER)) {
-      return array();
-    }
-    $headers = array();
-    foreach ($_SERVER as $name => $value) {
-      if (substr($name, 0, 5) == 'HTTP_') {
-        $headers[str_replace(' ', '-', ucwords(strtolower(str_replace('_', ' ', substr($name, 5)))))] = $value;
-      }
-    }
-    return $headers;
-  }
+// rspamd metadata_exporter (multipart formatter): metadata JSON arrives as $_POST['metadata'].
+if (empty($_POST['metadata'])) {
+  error_log("NOTIFY: missing metadata part from rspamd" . PHP_EOL);
+  http_response_code(400);
+  exit;
 }
 
-$headers = getallheaders();
-$json_body = json_decode(file_get_contents('php://input'));
+$meta = json_decode($_POST['metadata'], true);
+if (!is_array($meta)) {
+  error_log("NOTIFY: cannot decode metadata JSON" . PHP_EOL);
+  http_response_code(400);
+  exit;
+}
 
-$qid      = $headers['X-Rspamd-Qid'];
-$rcpts    = $headers['X-Rspamd-Rcpt'];
-$sender   = $headers['X-Rspamd-From'];
-$ip       = $headers['X-Rspamd-Ip'];
-$subject  = iconv_mime_decode($headers['X-Rspamd-Subject']);
-$messageid= $json_body->message_id;
+$qid      = $meta['qid']    ?? 'unknown';
+$rcpts    = $meta['rcpt']   ?? array();
+$sender   = $meta['from']   ?? '';
+$ip       = $meta['ip']     ?? 'unknown';
+$subject  = $meta['subject'] ?? '';
+$messageid= $meta['message_id'] ?? '';
 $priority = 0;
 
-$symbols_array = json_decode($headers['X-Rspamd-Symbols'], true);
+$symbols_array = $meta['symbols'] ?? array();
 if (is_array($symbols_array)) {
   foreach ($symbols_array as $symbol) {
-    if ($symbol['name'] == 'HAS_X_PRIO_ONE') {
+    if (($symbol['name'] ?? null) == 'HAS_X_PRIO_ONE') {
       $priority = 1;
       break;
     }
   }
 }
 
-$sender_address = $json_body->header_from[0];
+$sender_address = $meta['header_from'][0] ?? '';
 $sender_name = '-';
 if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $sender_address, $matches)) {
 	$sender_address = $matches['address'];
   $sender_name =  trim($matches['name'], '"\' ');
 }
 
-$to_address = $json_body->header_to[0];
+$to_address = $meta['header_to'][0] ?? '';
 $to_name = '-';
 if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $to_address, $matches)) {
 	$to_address = $matches['address'];
@@ -85,7 +81,7 @@ if (preg_match('/(?<name>.*?)<(?<address>.*?)>/i', $to_address, $matches)) {
 $rcpt_final_mailboxes = array();
 
 // Loop through all rcpts
-foreach (json_decode($rcpts, true) as $rcpt) {
+foreach ($rcpts as $rcpt) {
   // Remove tag
   $rcpt = preg_replace('/^(.*?)\+.*(@.*)$/', '$1$2', $rcpt);
 
@@ -143,12 +139,12 @@ foreach (json_decode($rcpts, true) as $rcpt) {
 
       // Loop through all found gotos
       foreach ($gotos_array as $index => &$goto) {
-        error_log("RCPT RESOVLER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
+        error_log("RCPT RESOLVER: http pipe: query " . $goto . " as username from mailbox" . PHP_EOL);
         $stmt = $pdo->prepare("SELECT `username` FROM `mailbox` WHERE `username` = :goto AND (`active`= '1' OR `active`= '2');");
         $stmt->execute(array(':goto' => $goto));
         $username = $stmt->fetch(PDO::FETCH_ASSOC)['username'];
         if (!empty($username)) {
-          error_log("RCPT RESOVLER: http pipe: mailbox found: " . $username . PHP_EOL);
+          error_log("RCPT RESOLVER: http pipe: mailbox found: " . $username . PHP_EOL);
           // Current goto is a mailbox, save to rcpt_final_mailboxes if not a duplicate
           if (!in_array($username, $rcpt_final_mailboxes)) {
             $rcpt_final_mailboxes[] = $username;
@@ -157,21 +153,21 @@ foreach (json_decode($rcpts, true) as $rcpt) {
         else {
           $parsed_goto = parse_email($goto);
           if (!$redis->hGet('DOMAIN_MAP', $parsed_goto['domain'])) {
-            error_log("RCPT RESOVLER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
+            error_log("RCPT RESOLVER:" . $goto . " is not a mailcow handled mailbox or alias address" . PHP_EOL);
           }
           else {
             $stmt = $pdo->prepare("SELECT `goto` FROM `alias` WHERE `address` = :goto AND `active` = '1'");
             $stmt->execute(array(':goto' => $goto));
             $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['goto'];
             if ($goto_branch) {
-              error_log("RCPT RESOVLER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
+              error_log("RCPT RESOLVER: http pipe: goto address " . $goto . " is an alias branch for " . $goto_branch . PHP_EOL);
               $goto_branch_array = explode(',', $goto_branch);
             } else {
-              $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` AND '1'");
+              $stmt = $pdo->prepare("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'");
               $stmt->execute(array(':domain' => $parsed_goto['domain']));
               $goto_branch = $stmt->fetch(PDO::FETCH_ASSOC)['target_domain'];
               if ($goto_branch) {
-                error_log("RCPT RESOVLER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
+                error_log("RCPT RESOLVER: http pipe: goto domain " . $parsed_goto['domain'] . " is a domain alias branch for " . $goto_branch . PHP_EOL);
                 $goto_branch_array = array($parsed_goto['local'] . '@' . $goto_branch);
               }
             }
@@ -193,11 +189,11 @@ foreach (json_decode($rcpts, true) as $rcpt) {
       // Force exit if loop cannot be solved
       // Postfix does not allow for alias loops, so this should never happen.
       $loop_c++;
-      error_log("RCPT RESOVLER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
+      error_log("RCPT RESOLVER: http pipe: goto array count on loop #". $loop_c . " is " . count($gotos_array) . PHP_EOL);
     }
   }
   catch (PDOException $e) {
-    error_log("RCPT RESOVLER: " . $e->getMessage() . PHP_EOL);
+    error_log("RCPT RESOLVER: " . $e->getMessage() . PHP_EOL);
     http_response_code(502);
     exit;
   }

data/conf/sogo/sogo.conf

@@ -86,6 +86,12 @@
     SOGoMaximumFailedLoginInterval = 900;
     SOGoFailedLoginBlockInterval = 900;
 
+    // Enable SOGo URL Description for GDPR compliance, this may cause some issues with calendars and contacts. Also uncomment the encryption key below to use it.
+    //SOGoURLEncryptionEnabled = NO;
+
+    // Set a 16 character encryption key for SOGo URL Description, change this to your own value
+    //SOGoURLPathEncryptionKey = "SOGoSuperSecret0";
+
     GCSChannelCollectionTimer = 60;
     GCSChannelExpireAge = 60;
 

data/web/admin/dashboard.php

@@ -2,18 +2,7 @@
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
 
-if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'domainadmin') {
-  header('Location: /domainadmin/mailbox');
-  exit();
-}
-elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'user') {
-  header('Location: /user');
-  exit();
-}
-elseif (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != "admin") {
-  header('Location: /admin');
-  exit();
-}
+protect_route(['admin']);
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/header.inc.php';
 $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];

data/web/admin/index.php

@@ -3,8 +3,11 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
 
 if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'admin') {
-  header('Location: /admin/dashboard');
-  exit();
+  // Only redirect to dashboard if NO pending actions
+  if (empty($_SESSION['pending_tfa_setup']) && empty($_SESSION['pending_pw_update'])) {
+    header('Location: /admin/dashboard');
+    exit();
+  }
 }
 elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'domainadmin') {
   header('Location: /domainadmin/mailbox');

data/web/admin/mailbox.php

@@ -2,24 +2,15 @@
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
 
-if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'domainadmin') {
-  header('Location: /domainadmin/mailbox');
-  exit();
-}
-elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'user') {
-  header('Location: /user');
-  exit();
-}
-elseif (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != "admin") {
-  header('Location: /admin');
-  exit();
-}
+protect_route(['admin']);
 
 require_once $_SERVER['DOCUMENT_ROOT'] .  '/inc/header.inc.php';
 $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 
 
+$js_minifier->add('/web/js/site/imapsync.js');
+$js_minifier->add('/web/js/site/groups.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -49,10 +40,12 @@ $template_data = [
   'global_filters' => mailbox('get', 'global_filter_details'),
   'domains' => $domains,
   'mailboxes' => $mailboxes,
+  'mbox_templates' => mailbox('get', 'mailbox_templates'),
   'lang_mailbox' => json_encode($lang['mailbox']),
   'lang_rl' => json_encode($lang['ratelimit']),
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
+  'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/queue.php

@@ -2,19 +2,7 @@
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
 
-if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'domainadmin') {
-  header('Location: /domainadmin/mailbox');
-  exit();
-}
-elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'user') {
-  header('Location: /user');
-  exit();
-}
-elseif (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != "admin") {
-  header('Location: /admin');
-  exit();
-}
-
+protect_route(['admin']);
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/header.inc.php';
 $js_minifier->add('/web/js/site/queue.js');

data/web/admin/system.php

@@ -2,18 +2,7 @@
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
 
-if (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'domainadmin') {
-  header('Location: /domainadmin/mailbox');
-  exit();
-}
-elseif (isset($_SESSION['mailcow_cc_role']) && $_SESSION['mailcow_cc_role'] == 'user') {
-  header('Location: /user');
-  exit();
-}
-elseif (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != "admin") {
-  header('Location: /admin');
-  exit();
-}
+protect_route(['admin']);
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/header.inc.php';
 $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
@@ -23,6 +12,7 @@ $fido2_data = fido2(array("action" => "get_friendly_names"));
 $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
+$js_minifier->add('/web/js/site/identity.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -104,10 +94,6 @@ $template_data = [
   'tfa_id' => @$_SESSION['tfa_id'],
   'fido2_cid' => @$_SESSION['fido2_cid'],
   'fido2_data' => $fido2_data,
-  'api' => [
-    'ro' => admin_api('ro', 'get'),
-    'rw' => admin_api('rw', 'get'),
-  ],
   'dkim_domains' => $dkim_domains,
   'dkim_domains_with_keys' => $dkim_domains_with_keys,
   'dkim_blind_domains' => $dkim_blind_domains,
@@ -126,7 +112,10 @@ $template_data = [
   'logo_dark_specs' => customize('get', 'main_logo_dark_specs'),
   'ip_check' => customize('get', 'ip_check'),
   'custom_login' => customize('get', 'custom_login'),
+  'ui_themes' => customize('get', 'ui_themes'),
+  'ui_theme' => customize('get', 'ui_theme'),
   'password_complexity' => password_complexity('get'),
+  'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',

data/web/autoconfig.php

@@ -29,8 +29,8 @@ header('Content-Type: application/xml');
 <clientConfig version="1.1">
     <emailProvider id="<?=$mailcow_hostname; ?>">
       <domain>%EMAILDOMAIN%</domain>
-      <displayName>A mailcow mail server</displayName>
-      <displayShortName>mail server</displayShortName>
+      <displayName><?=$autodiscover_config['displayName']; ?></displayName>
+      <displayShortName><?=$autodiscover_config['displayShortName']; ?></displayShortName>
 
       <incomingServer type="imap">
          <hostname><?=$autodiscover_config['imap']['server']; ?></hostname>

data/web/autodiscover.php

@@ -7,6 +7,8 @@ if(file_exists('inc/vars.local.inc.php')) {
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.auth.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/sessions.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.mailbox.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.ratelimit.inc.php';
 $default_autodiscover_config = $autodiscover_config;
 $autodiscover_config = array_merge($default_autodiscover_config, $autodiscover_config);
 
@@ -58,101 +60,31 @@ $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
 $iam_provider = identity_provider('init');
 $iam_settings = identity_provider('get');
 
-$login_user = strtolower(trim($_SERVER['PHP_AUTH_USER']));
-$login_pass = trim(htmlspecialchars_decode($_SERVER['PHP_AUTH_PW']));
+// Passwordless autodiscover - no authentication required
+// Email will be extracted from the request body
+$login_user = null;
+$login_role = null;
 
-if (empty($_SERVER['PHP_AUTH_USER']) || empty($_SERVER['PHP_AUTH_PW'])) {
-  $json = json_encode(
-    array(
-      "time" => time(),
-      "ua" => $_SERVER['HTTP_USER_AGENT'],
-      "user" => "none",
-      "ip" => $_SERVER['REMOTE_ADDR'],
-      "service" => "Error: must be authenticated"
-    )
-  );
-  $redis->lPush('AUTODISCOVER_LOG', $json);
-  header('WWW-Authenticate: Basic realm="' . $_SERVER['HTTP_HOST'] . '"');
-  header('HTTP/1.0 401 Unauthorized');
-  exit(0);
-}
-
-$login_role = check_login($login_user, $login_pass, array('eas' => TRUE));
-
-if ($login_role === "user") {
-  header("Content-Type: application/xml");
-  echo '<?xml version="1.0" encoding="utf-8" ?>' . PHP_EOL;
+header("Content-Type: application/xml");
+echo '<?xml version="1.0" encoding="utf-8" ?>' . PHP_EOL;
 ?>
 <Autodiscover xmlns="http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006">
 <?php
-  if(!$data) {
-    try {
-      $json = json_encode(
-        array(
-          "time" => time(),
-          "ua" => $_SERVER['HTTP_USER_AGENT'],
-          "user" => $_SERVER['PHP_AUTH_USER'],
-          "ip" => $_SERVER['REMOTE_ADDR'],
-          "service" => "Error: invalid or missing request data"
-        )
-      );
-      $redis->lPush('AUTODISCOVER_LOG', $json);
-      $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
-    }
-    catch (RedisException $e) {
-      $_SESSION['return'][] = array(
-        'type' => 'danger',
-        'msg' => 'Redis: '.$e
-      );
-      return false;
-    }
-    list($usec, $sec) = explode(' ', microtime());
-?>
-  <Response>
-    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="2477272013">
-      <ErrorCode>600</ErrorCode>
-      <Message>Invalid Request</Message>
-      <DebugData />
-    </Error>
-  </Response>
-</Autodiscover>
-<?php
-    exit(0);
-  }
-  try {
-    $discover = new SimpleXMLElement($data);
-    $email = $discover->Request->EMailAddress;
-  } catch (Exception $e) {
-    $email = $_SERVER['PHP_AUTH_USER'];
-  }
-
-  $username = trim($email);
-  try {
-    $stmt = $pdo->prepare("SELECT `name` FROM `mailbox` WHERE `username`= :username");
-    $stmt->execute(array(':username' => $username));
-    $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
-  }
-  catch(PDOException $e) {
-    die("Failed to determine name from SQL");
-  }
-  if (!empty($MailboxData['name'])) {
-    $displayname = $MailboxData['name'];
-  }
-  else {
-    $displayname = $email;
-  }
+if(!$data) {
   try {
     $json = json_encode(
       array(
         "time" => time(),
         "ua" => $_SERVER['HTTP_USER_AGENT'],
-        "user" => $_SERVER['PHP_AUTH_USER'],
+        "user" => "none",
         "ip" => $_SERVER['REMOTE_ADDR'],
-        "service" => $autodiscover_config['autodiscoverType']
+        "service" => "Error: invalid or missing request data"
       )
     );
     $redis->lPush('AUTODISCOVER_LOG', $json);
     $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+    $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid request by " . $_SERVER['REMOTE_ADDR']);
+    error_log("Autodiscover: Invalid request by " . $_SERVER['REMOTE_ADDR']);
   }
   catch (RedisException $e) {
     $_SESSION['return'][] = array(
@@ -161,7 +93,143 @@ if ($login_role === "user") {
     );
     return false;
   }
-  if ($autodiscover_config['autodiscoverType'] == 'imap') {
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>600</ErrorCode>
+      <Message>Invalid Request</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
+try {
+  $discover = new SimpleXMLElement($data);
+  $email = $discover->Request->EMailAddress;
+} catch (Exception $e) {
+  // If parsing fails, return error
+  try {
+    $json = json_encode(
+      array(
+        "time" => time(),
+        "ua" => $_SERVER['HTTP_USER_AGENT'],
+        "user" => "none",
+        "ip" => $_SERVER['REMOTE_ADDR'],
+        "service" => "Error: could not parse email from request"
+      )
+    );
+    $redis->lPush('AUTODISCOVER_LOG', $json);
+    $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+    $redis->publish("F2B_CHANNEL", "Autodiscover: Malformed XML by " . $_SERVER['REMOTE_ADDR']);
+    error_log("Autodiscover: Malformed XML by " . $_SERVER['REMOTE_ADDR']);
+  }
+  catch (RedisException $e) {
+    // Silently fail
+  }
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>600</ErrorCode>
+      <Message>Invalid Request</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
+
+$username = trim((string)$email);
+try {
+  $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox` 
+    INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
+    WHERE `mailbox`.`username` = :username 
+    AND `mailbox`.`active` = '1'
+    AND `domain`.`active` = '1'");
+  $stmt->execute(array(':username' => $username));
+  $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
+}
+catch(PDOException $e) {
+  // Database error - return error response with complete XML
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>500</ErrorCode>
+      <Message>Database Error</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
+
+// Mailbox not found or not active - return generic error to prevent user enumeration
+if (empty($MailboxData)) {
+  try {
+    $json = json_encode(
+      array(
+        "time" => time(),
+        "ua" => $_SERVER['HTTP_USER_AGENT'],
+        "user" => $email,
+        "ip" => $_SERVER['REMOTE_ADDR'],
+        "service" => "Error: mailbox not found or inactive"
+      )
+    );
+    $redis->lPush('AUTODISCOVER_LOG', $json);
+    $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+    $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+    error_log("Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+  }
+  catch (RedisException $e) {
+    // Silently fail
+  }
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>600</ErrorCode>
+      <Message>Invalid Request</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
+
+if (!empty($MailboxData['name'])) {
+  $displayname = $MailboxData['name'];
+}
+else {
+  $displayname = $email;
+}
+try {
+  $json = json_encode(
+    array(
+      "time" => time(),
+      "ua" => $_SERVER['HTTP_USER_AGENT'],
+      "user" => $email,
+      "ip" => $_SERVER['REMOTE_ADDR'],
+      "service" => $autodiscover_config['autodiscoverType']
+    )
+  );
+  $redis->lPush('AUTODISCOVER_LOG', $json);
+  $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+}
+catch (RedisException $e) {
+  $_SESSION['return'][] = array(
+    'type' => 'danger',
+    'msg' => 'Redis: '.$e
+  );
+  return false;
+}
+if ($autodiscover_config['autodiscoverType'] == 'imap') {
 ?>
   <Response xmlns="http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a">
     <User>
@@ -236,6 +304,3 @@ if ($login_role === "user") {
   }
 ?>
 </Autodiscover>
-<?php
-}
-?>

data/web/css/build/003-bootstrap-select.css

@@ -1,10 +1,10 @@
-/*!
- * Bootstrap-select v1.14.0-beta2 (https://developer.snapappointments.com/bootstrap-select)
- *
- * Copyright 2012-2021 SnapAppointments, LLC
- * Licensed under MIT (https://github.com/snapappointments/bootstrap-select/blob/master/LICENSE)
- */
-
+/*!
+ * Bootstrap-select v1.14.0-beta3 (https://developer.snapappointments.com/bootstrap-select)
+ *
+ * Copyright 2012-2022 SnapAppointments, LLC
+ * Licensed under MIT (https://github.com/snapappointments/bootstrap-select/blob/master/LICENSE)
+ */
+
 @-webkit-keyframes bs-notify-fadeOut {
   0% {
     opacity: 0.9;
@@ -35,8 +35,7 @@ select.selectpicker {
   display: none !important;
 }
 .bootstrap-select {
-  width: 220px \0;
-  /*IE9 and below*/
+  width: 220px;
   vertical-align: middle;
 }
 .bootstrap-select > .dropdown-toggle {