Merge branch 'feat/backup' into nightly

[Update] Restart update.sh automatically after it updated itself

_modules/scripts/backup.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# _modules/scripts/backup.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+# Backup with restic before the update when it is set up (target, confirmed password, initialized repository):
+# interactive: asked for; --backup: without asking; --force: none (with --backup: without asking, a failure or a backup
+# that cannot run stops the update; only "not set up" continues)
+backup_failed() {
+  echo -e "\e[31m${1}\e[0m"
+  if [ "$FORCE" ]; then
+    echo "The update was started with --force --backup, it stops here. Nothing was changed."
+    exit 1
+  fi
+  read -r -p "Continue the update without a backup? [y/N] " backup_response
+  if [[ ! "${backup_response}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    echo "OK, exiting."
+    exit 1
+  fi
+}
+
+# Whether to back up (asked with the other questions); the backup itself runs before the first change of the checkout
+backup_decide() {
+  local output rc
+  # exported: update.sh restarts itself after a branch switch, the decision and a backup already made stay
+  if [ "$MAILCOW_UPDATE_BACKUP_DECIDED" ]; then
+    return 0
+  fi
+  export MAILCOW_UPDATE_BACKUP_DECIDED=y
+  export RUN_BACKUP=
+  if [ "$FORCE" ] && [ ! "$BACKUP" ]; then
+    return 0
+  fi
+  if [[ -z "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
+    output="the backup container does not run"
+    rc=1
+  else
+    # exit code 0: set up, 2: not set up, others: error (e.g. credentials this container cannot read)
+    output=$($COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --configured 2>&1 < /dev/null)
+    rc=$?
+  fi
+  if [[ ${rc} -eq 2 ]]; then
+    [ "$BACKUP" ] && echo -e "\e[33mNo backup before the update: ${output}.\e[0m"
+    return 0
+  elif [[ ${rc} -ne 0 ]]; then
+    if [ "$BACKUP" ]; then
+      backup_failed "No backup before the update possible: ${output}"
+    elif [[ -n "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
+      echo -e "\e[33mNo backup before the update: ${output}\e[0m"
+    fi
+    return 0
+  fi
+  if [ ! "$BACKUP" ]; then
+    read -r -p "Create a backup before the update? [Y/n] " backup_response
+    if [[ "${backup_response}" =~ ^([nN][oO]|[nN])$ ]]; then
+      echo "OK, no backup."
+      return 0
+    fi
+  fi
+  RUN_BACKUP=y
+}
+
+# The backup chosen by backup_decide, once
+backup_run() {
+  if [ ! "$RUN_BACKUP" ]; then
+    return 0
+  fi
+  RUN_BACKUP=
+  if [[ "${RESTIC_BACKUP_MODE}" =~ ^(data|config)$ ]]; then
+    echo -e "\e[33mRESTIC_BACKUP_MODE=${RESTIC_BACKUP_MODE}: the backup holds no complete mailcow directory, a restore does not bring back this mailcow version.\e[0m"
+  fi
+  echo -e "\e[32mCreating a backup before the update...\e[0m"
+  if ! $COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --backup < /dev/null; then
+    backup_failed "The backup before the update failed, see the output above and the backup jobs in the mailcow UI."
+  fi
+}

_modules/scripts/core.sh

@@ -192,6 +192,7 @@ detect_major_update() {
       "2025-02"
       "2025-03"
       "2025-09"
+      "2026-11"
     )
 
     current_version=""

_modules/scripts/ipv6_controller.sh

@@ -3,6 +3,11 @@
 # THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
 # DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
 
+# Global IPv6 address that is no unique local address (fc00::/7, e.g. Docker bridges)
+_has_public_ipv6() {
+  ip -6 -o addr show scope global 2>/dev/null | awk '{print $4}' | grep -qviE '^f[cd]'
+}
+
 # 1) Check if the host supports IPv6
 get_ipv6_support() {
   # ---- helper: probe external IPv6 connectivity without DNS ----
@@ -37,6 +42,9 @@ get_ipv6_support() {
     if _probe_ipv6_connectivity; then
       DETECTED_IPV6=true
       echo -e "IPv6 detected on host – ${LIGHT_GREEN}leaving IPv6 support enabled${YELLOW}.${NC}"
+    elif _has_public_ipv6; then
+      DETECTED_IPV6=true
+      echo -e "${YELLOW}External IPv6 connectivity test failed, but the host has a public IPv6 address – ${LIGHT_GREEN}leaving IPv6 support enabled${YELLOW}.${NC}"
     else
       DETECTED_IPV6=false
       echo -e "${YELLOW}Default IPv6 route present but external IPv6 connectivity failed – ${LIGHT_RED}disabling IPv6 support${YELLOW}.${NC}"
@@ -192,9 +200,31 @@ EOF
   fi
 }
 
+_set_enable_ipv6() {
+  if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]]; then
+    if grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
+      sed -i "s/^ENABLE_IPV6=.*/ENABLE_IPV6=$1/" "$MAILCOW_CONF"
+    else
+      echo "ENABLE_IPV6=$1" >> "$MAILCOW_CONF"
+    fi
+  else
+    export IPV6_BOOL=$1
+  fi
+}
+
+_warn_ipv6_disabled() {
+  _has_public_ipv6 || return 0
+  ip -6 route show default 2>/dev/null | grep -qE '^default' || return 0
+  grep -Eq '"userland-proxy"[[:space:]]*:[[:space:]]*false' /etc/docker/daemon.json 2>/dev/null && return 0
+  echo -e "${LIGHT_RED}WARNING: ENABLE_IPV6=false, but this host has a public IPv6 address.${NC}"
+  echo -e "${YELLOW}Docker forwards IPv6 connections to mailcow with an internal source address, which Postfix trusts. This can make your server an open relay over IPv6.${NC}"
+  echo -e "${YELLOW}Set ENABLE_IPV6=true in mailcow.conf, or set \"userland-proxy\": false in /etc/docker/daemon.json and restart Docker.${NC}"
+  echo -e "${YELLOW}See https://docs.mailcow.email/post_installation/firststeps-disable_ipv6/${NC}"
+  sleep 5
+}
+
 # 3) Main wrapper for generate_config.sh and update.sh
 configure_ipv6() {
-  # detect manual override if mailcow.conf is present
   if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]] && grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
     MANUAL_SETTING=$(grep '^ENABLE_IPV6=' "$MAILCOW_CONF" | cut -d= -f2)
   elif [[ -z "$MAILCOW_CONF" ]] && [[ -n "${ENABLE_IPV6:-}" ]]; then
@@ -203,19 +233,23 @@ configure_ipv6() {
     MANUAL_SETTING=""
   fi
 
+  if [[ "${MANUAL_SETTING,,}" == "false" ]]; then
+    _set_enable_ipv6 false
+    echo "ENABLE_IPV6=false is set, skipping Docker IPv6 configuration."
+    _warn_ipv6_disabled
+    echo "IPv6 configuration complete: ENABLE_IPV6=false"
+    return
+  fi
+
   get_ipv6_support
 
-  # if user manually set it, check for mismatch
+  if [[ "$DETECTED_IPV6" != "true" && "${MANUAL_SETTING,,}" == "true" ]] && _has_public_ipv6; then
+    echo -e "${YELLOW}Keeping ENABLE_IPV6=true, the host has a public IPv6 address.${NC}"
+    DETECTED_IPV6=true
+  fi
+
   if [[ "$DETECTED_IPV6" != "true" ]]; then
-    if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]]; then
-      if grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
-        sed -i 's/^ENABLE_IPV6=.*/ENABLE_IPV6=false/' "$MAILCOW_CONF"
-      else
-        echo "ENABLE_IPV6=false" >> "$MAILCOW_CONF"
-      fi
-    else
-      export IPV6_BOOL=false
-    fi
+    _set_enable_ipv6 false
     echo "Skipping Docker IPv6 configuration because host does not support IPv6."
     echo "Make sure to check if your docker daemon.json does not include \"enable_ipv6\": true if you do not want IPv6."
     echo "IPv6 configuration complete: ENABLE_IPV6=false"
@@ -224,16 +258,7 @@ configure_ipv6() {
   fi
 
   docker_daemon_edit
-
-  if [[ -n "$MAILCOW_CONF" && -f "$MAILCOW_CONF" ]]; then
-    if grep -q '^ENABLE_IPV6=' "$MAILCOW_CONF"; then
-      sed -i 's/^ENABLE_IPV6=.*/ENABLE_IPV6=true/' "$MAILCOW_CONF"
-    else
-      echo "ENABLE_IPV6=true" >> "$MAILCOW_CONF"
-    fi
-  else
-    export IPV6_BOOL=true
-  fi
+  _set_enable_ipv6 true
 
   echo "IPv6 configuration complete: ENABLE_IPV6=true"
 }

_modules/scripts/new_options.sh

@@ -60,6 +60,9 @@ adapt_new_options() {
   "ACME_DNS_CHALLENGE"
   "ACME_DNS_PROVIDER"
   "ACME_ACCOUNT_EMAIL"
+  "NGINX_MAX_BODY_SIZE"
+  "RESTIC_LOCAL_PATH"
+  "RESTIC_BACKUP_MODE"
   )
 
   sed -i --follow-symlinks '$a\' mailcow.conf
@@ -309,6 +312,24 @@ adapt_new_options() {
             echo '# Account email for ACME DNS-01 challenge registration' >> mailcow.conf
             echo 'ACME_ACCOUNT_EMAIL=me@example.com' >> mailcow.conf
             ;;
+        NGINX_MAX_BODY_SIZE)
+            echo '# Maximum request size for SOGo webmail, CalDAV/CardDAV and ActiveSync, e.g. 128m or 1g - 0 means unlimited' >> mailcow.conf
+            echo '# Keep it above the message size limit of Postfix (100 MB by default). The mailcow UI and API stay limited to 64m.' >> mailcow.conf
+            echo 'NGINX_MAX_BODY_SIZE=128m' >> mailcow.conf
+            ;;
+        RESTIC_LOCAL_PATH)
+            echo '# Folder on the host for backups to the target "local" (backup settings in the mailcow UI), as an absolute path,' >> mailcow.conf
+            echo '# e.g. a NAS mounted from another disk. The repository is created in a subfolder of it.' >> mailcow.conf
+            echo '# Empty: the Docker volume restic-repo-vol-1 of mailcow, on the same disk as the mailcow data.' >> mailcow.conf
+            echo 'RESTIC_LOCAL_PATH=' >> mailcow.conf
+            ;;
+        RESTIC_BACKUP_MODE)
+            echo '# What the backups of the mailcow UI hold:' >> mailcow.conf
+            echo '# - all (default),' >> mailcow.conf
+            echo '# - data (vmail, vmail-index, database, redis, mail crypt keys and mailcow.conf)' >> mailcow.conf
+            echo '# - config (database, redis, mail crypt keys, mailcow.conf and data/conf).' >> mailcow.conf
+            echo 'RESTIC_BACKUP_MODE=all' >> mailcow.conf
+            ;;
         *)
             echo "${option}=" >> mailcow.conf
             ;;

data/Dockerfiles/dockerapi/main.py

@@ -159,6 +159,9 @@ async def post_containers(container_id : str, post_action : str, request: Reques
       api_call_method = getattr(dockerapi, api_call_method_name, lambda container_id: Response(content=json.dumps({'type': 'danger', 'msg':'container_post - unknown api call' }, indent=4), media_type="application/json"))
 
       dockerapi.logger.info("api call: %s, container_id: %s" % (api_call_method_name, container_id))
+      if getattr(api_call_method, 'long_running', False):
+        # runs for minutes (e.g. sogo-tool of a backup restore): in a worker thread, the other api calls go on
+        return await asyncio.to_thread(api_call_method, request_json, container_id=container_id)
       return api_call_method(request_json, container_id=container_id)
     except Exception as e:
       dockerapi.logger.error("error - container_post: %s" % str(e))

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -6,9 +6,15 @@ import time
 import json
 import asyncio
 import platform
+import docker
 from datetime import datetime
 from fastapi import FastAPI, Response, Request
 
+def long_running(method):
+  """Marks an api call that runs for minutes: main.py runs it in a worker thread, the other api calls go on"""
+  method.long_running = True
+  return method
+
 class DockerApi:
   def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
     self.redis_client = redis_client
@@ -318,7 +324,7 @@ class DockerApi:
 
     if 'username' in request_json and 'script_name' in request_json:
       for container in self.sync_docker_client.containers.list(filters=filters):
-        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]
+        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' -- '" + request_json['script_name'].replace("'", "'\\''") + "'"]
         sieve_return = container.exec_run(cmd)
         return self.exec_run_handler('utf8_text_only', sieve_return)
   # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
@@ -445,6 +451,35 @@ class DockerApi:
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
       init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
       return self.exec_run_handler('generic', init_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: restore
+  # Resets calendars, address books and preferences of a user to the sogo-tool backup file <dir>/<username> in a
+  # restore batch of the backup (/restore/<job>/b<batch>/sogo_backup, only the sogo user can read it); values are
+  # checked and passed as arguments, never through the shell. Runs in a worker thread (main.py) with its own client.
+  @long_running
+  def container_post__exec__sogo__restore(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    directory = str(request_json.get('dir', ''))
+    username = str(request_json.get('username', ''))
+    if (not re.fullmatch(r'/restore/[0-9]+/b[0-9]+/sogo_backup', directory)
+      or not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+', username)):
+      return Response(content=json.dumps({ 'type': 'danger', 'msg': 'dir and username required' }, indent=4), media_type="application/json")
+    # the folder: no symlink, uid 999, mode 0700; the file: plain, uid 999, mode 0600. sogo-tool restores folders and
+    # preferences in two runs, both stopped before the runner gives up waiting (30 minutes)
+    script = ('[ -d "$1" ] && [ ! -L "$1" ] && [ "$(realpath -e -- "$1")" = "$1" ] && [ "$(stat -c "%u %a" -- "$1")" = "999 700" ] '
+      '&& [ -f "$1/$2" ] && [ ! -L "$1/$2" ] && [ "$(stat -c "%u %a" -- "$1/$2")" = "999 600" ] '
+      '|| { echo "invalid directory or file"; exit 1; }; '
+      'timeout -k 30 850 sogo-tool restore -f ALL "$1" "$2" && exec timeout -k 30 850 sogo-tool restore -p "$1" "$2"')
+    client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version='auto', timeout=1800)
+    try:
+      for container in client.containers.list(filters=filters):
+        restore_return = container.exec_run(["/bin/bash", "-c", script, "restore", directory, username], user='sogo')
+        return self.exec_run_handler('generic', restore_return)
+    finally:
+      client.close()
   # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
   # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
   def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
@@ -491,7 +526,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' -- '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -510,7 +545,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' -- '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -534,7 +569,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -568,7 +603,7 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
   # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
@@ -585,9 +620,9 @@ class DockerApi:
       rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
       if not user or id in ('user=', 'group='):
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
-      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
-      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
-        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
+      action = 'doveadm acl set -u "$ACL_USER" -- "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" -- "$folder" "$ACL_ID"'
+      script = ('set -o pipefail; doveadm -f tab mailbox status -u "$ACL_USER" guid "*" | tail -n +2 | cut -f1 | while IFS= read -r folder; do '
+        'case "$folder" in Shared|Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
       acl_return = container.exec_run(["/bin/bash", "-c", script],
         environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
       return self.exec_run_handler('generic', acl_return)
@@ -601,7 +636,7 @@ class DockerApi:
 
     for container in self.sync_docker_client.containers.list(filters=filters):
       users = [str(user) for user in request_json.get('users', []) if str(user)]
-      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush", "--"] + users)
       return self.exec_run_handler('generic', flush_return)
 
   # Rights doveadm acl set accepts from the API

data/Dockerfiles/dovecot/Dockerfile

@@ -150,6 +150,8 @@ COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
 COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
+COPY retention_runner.py /usr/local/bin/retention_runner.py
+COPY restore_runner.py /usr/local/bin/restore_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -281,6 +281,8 @@ chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
   /usr/local/bin/imapsync_runner.py \
+  /usr/local/bin/retention_runner.py \
+  /usr/local/bin/restore_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -306,13 +308,20 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
 # Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
-DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+# No error while the versions table is missing (fresh install, set -e)
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
   echo "Waiting for schema update..."
   sleep 5
-  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
 done
-echo "DB schema is ${DBV_NOW}"
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
+else
+  echo "DB schema is ${DBV_NOW}"
+fi
 
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -1,10 +1,12 @@
 #!/usr/bin/python3
 
 import fcntl
+import ipaddress
 import json
 import os
 import re
 import signal
+import socket
 import subprocess
 import sys
 import tempfile
@@ -18,6 +20,15 @@ max_output = 16 * 1024 * 1024
 lock_file = run_dir + '/imapsync_busy.lock'
 # An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
 token_min_valid = 900
+# Custom params never passed, also when stored before (same list as CustomParams::BLOCKED in the web UI)
+blocked_params = {'abort', 'delete2foldersbutnot', 'delete2foldersonly', 'logdir', 'logfile', 'maxlinelengthcmd',
+  'oauthrefreshcmd1', 'oauthrefreshcmd2', 'passfile1', 'passfile2', 'pidfile', 'pipemess', 'regexflag', 'regexmess',
+  'regextrans2', 'showpasswords', 'skipmess', 'tests', 'testsdebug', 'testslive', 'testslive6', 'testsunit', 'tmpdir'}
+# imapsync only gets these variables, not the secrets of the container environment
+job_env_keys = ('PATH', 'LANG', 'LC_ALL', 'TZ')
+# A source must not point to mailcow itself (internal Docker network), host1 is checked before each run
+internal_nets = [ipaddress.ip_network(os.environ.get('IPV4_NETWORK', '172.22.1') + '.0/24', strict=False),
+  ipaddress.ip_network(os.environ.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'), strict=False)]
 # Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
 token_sync_interval = 60
 
@@ -25,7 +36,7 @@ token_sync_interval = 60
 next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
   i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
   i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
-  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
+  s.host1, s.port1, s.enc1, s.verify_cert, s.auth_type, s.oauth_flow
     FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
       WHERE i.active = 1
         AND s.active = 1
@@ -106,6 +117,28 @@ def defer_job(job_id, text, exit_status):
     (text, exit_status, job_id))
   deferred.add(job_id)
 
+# True when host resolves to loopback, link-local or the internal network; unknown hosts are left to imapsync
+def internal_host(host):
+  try:
+    addrs = {a[4][0] for a in socket.getaddrinfo(host, None)}
+  except (socket.gaierror, UnicodeError):
+    return False
+  for addr in addrs:
+    ip = ipaddress.ip_address(addr.split('%')[0])
+    if ip.version == 6 and ip.ipv4_mapped:
+      ip = ip.ipv4_mapped
+    if ip.is_loopback or ip.is_unspecified or ip.is_link_local or any(ip in net for net in internal_nets):
+      return True
+  return False
+
+# SNI takes host names only
+def ip_literal(host):
+  try:
+    ipaddress.ip_address(host)
+    return True
+  except ValueError:
+    return False
+
 def oauth_token(j):
   if j['oauth_flow'] == 'authorization_code':
     return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
@@ -150,6 +183,11 @@ def prepare_job(j, global_bw, files):
       f.write(content)
     return path
 
+  if internal_host(j['host1']):
+    finish_job(j['id'], 'Host1 %s is an address of mailcow itself, the sync job does not connect to it\n' % j['host1'], 0,
+      'EXIT_CONNECTION_FAILURE_HOST1')
+    return None
+
   auth_type = j['auth_type'] or 'PLAIN'
   oauth = None
   if auth_type == 'XOAUTH2':
@@ -174,7 +212,11 @@ def prepare_job(j, global_bw, files):
   except ValueError:
     pairs = []
   for p in pairs if isinstance(pairs, list) else []:
-    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
+    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']) \
+        or p['o'] in blocked_params:
+      continue
+    # The certificate check protects the OAuth token of the source, a job must not turn it off
+    if oauth and p['o'] == 'sslargs1':
       continue
     value = '' if p.get('v') is None else str(p['v'])
     custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
@@ -205,7 +247,13 @@ def prepare_job(j, global_bw, files):
       cmd.append(opt)
   enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
   if enc1:
-    cmd.append(enc1)
+    # Mail::IMAPClient starts TLS without a host name: check the certificate against host1, not its address
+    cmd += [enc1, '--sslargs1', 'SSL_verifycn_scheme=imap', '--sslargs1', 'SSL_verifycn_name=' + j['host1']]
+    if not ip_literal(j['host1']):
+      cmd += ['--sslargs1', 'SSL_hostname=' + j['host1']]
+    # imapsync does not check certificates by default; custom sslargs1 of password jobs can still change it
+    if j['verify_cert'] == 1:
+      cmd += ['--sslargs1', 'SSL_verify_mode=1']
   cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
   cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
   if j['dry'] == 1:
@@ -216,8 +264,10 @@ def prepare_job(j, global_bw, files):
 
 # Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
 def start_job(job):
+  env = {k: os.environ[k] for k in job_env_keys if k in os.environ}
+  env['HOME'] = run_dir
   with open(job['outfile'], 'wb') as out:
-    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
+    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, env=env)
 
 # Store the result of a finished worker and remove its temp files
 def reap_job(job):

data/Dockerfiles/dovecot/restore_runner.py

@@ -0,0 +1,712 @@
+#!/usr/bin/python3
+# Mail part of restores (ofelia, dovecot container, as vmail): the restic container restores a batch of mailboxes
+# into /restore/<job id>/b<n> and sets the job to step 'mails'. Per mailbox this imports the mails folder by folder
+# (into "Restored <snapshot time>" or only the missing ones into the original folders), adds IMAP ACLs and missing
+# Sieve scripts, merges the results into the job and sets step 'mails_done'; the restic runner then removes the staging
+# and goes on with the next batch. Nothing that exists is changed or deleted.
+
+import base64
+import fcntl
+import json
+import os
+import re
+import smtplib
+import stat
+import subprocess
+import sys
+import threading
+import time
+from email.message import EmailMessage
+from email.utils import formatdate, make_msgid
+import MySQLdb
+import MySQLdb.cursors
+
+lock_file = '/tmp/restore_runner.lock'
+restore_dir = '/restore'
+max_log = 1000000
+# import, moves and the removal of the temporary folder ignore the quota (a restore may need more than is free),
+# the quota is recalculated afterwards
+no_quota = ['-o', 'plugin/quota=']
+# uid ranges per import (one argument stays far below the 128 KiB limit), folders per fetch and status
+uid_ranges = 2000
+folder_batch = 200
+# the restic runner waits for the batch while this heartbeat is fresh
+heartbeat_interval = 30
+# seconds to wait for the next batch of a restore
+batch_wait = 120
+rights_by_letter = {'l': 'lookup', 'r': 'read', 'w': 'write', 's': 'write-seen', 't': 'write-deleted', 'i': 'insert',
+  'p': 'post', 'e': 'expunge', 'k': 'create', 'x': 'delete', 'a': 'admin'}
+components_here = {'mails', 'imap_acl', 'filters', 'sieve_activate'}
+
+conn = None
+
+
+def connect():
+  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
+    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
+
+# Reconnect when the server dropped the connection (imports can outlast wait_timeout)
+def query(sql, args=None):
+  global conn
+  try:
+    conn.ping()
+  except (AttributeError, MySQLdb.Error):
+    conn = connect()
+  cur = conn.cursor(MySQLdb.cursors.DictCursor)
+  cur.execute(sql, args)
+  return cur
+
+# state_restore_seen while a batch runs, in its own connection
+class Heartbeat(threading.Thread):
+  def __init__(self):
+    super().__init__(daemon=True)
+    self.done = threading.Event()
+
+  def run(self):
+    db = None
+    while not self.done.is_set():
+      try:
+        if db is None:
+          db = connect()
+        db.cursor().execute("""INSERT INTO backup_setting (name, value) VALUES ('state_restore_seen', %s)
+          ON DUPLICATE KEY UPDATE value = VALUES(value)""", (str(int(time.time())),))
+      except MySQLdb.Error:
+        db = None
+      self.done.wait(heartbeat_interval)
+
+def log(job_id, text):
+  print('restore job %d: %s' % (job_id, text), flush=True)
+  line = '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), text)
+  query("UPDATE backup_job SET log = RIGHT(CONCAT(IFNULL(log, ''), %s), %s) WHERE id = %s", (line, max_log, job_id))
+
+def save(job_id, result):
+  query("UPDATE backup_job SET result = %s WHERE id = %s", (json.dumps(result), job_id))
+
+def doveadm(*args, data=None, check=True, partial=False):
+  result = subprocess.run(['doveadm', *args], input=data, stdin=None if data is not None else subprocess.DEVNULL,
+    capture_output=True, timeout=86400)
+  # mailbox status fails for folders that cannot be opened (implicit parents); the others are listed
+  if check and result.returncode != 0 and not (partial and result.stdout.strip()):
+    # mail_log writes an info line per message, the errors are the other lines
+    lines = [l for l in result.stderr.decode('utf-8', 'replace').splitlines() if ': Info: ' not in l]
+    raise RuntimeError('\n'.join(lines)[-500:] or 'doveadm exit code %d' % result.returncode)
+  return result.stdout.decode('utf-8', 'replace')
+
+def doveadm_json(*args, partial=False):
+  out = doveadm('-f', 'json', *args, partial=partial)
+  return json.loads(out) if out.strip() else []
+
+# free disk space that must stay after the import: max(2 GiB, 5 % of the filesystem), a full disk stops mail
+# (disk_reserve, mib and count_errors as in restic_runner.py of the backup image; tests/python/backup compares them)
+def disk_reserve(total):
+  return max(2 << 30, total // 20)
+
+def import_space_ok(need, free, total):
+  return need * 11 // 10 + disk_reserve(total) <= free
+
+def mib(size):
+  return '%.1f MiB' % (size / 1048576)
+
+def component(errors=None):
+  return {'added': 0, 'skipped': 0, 'errors': errors or []}
+
+# Folder names on disk: modified UTF-7, listescape writes . / \ as \xx
+def mutf7_decode(text):
+  def utf16(m):
+    if not m.group(1):
+      return '&'
+    data = m.group(1).replace(',', '/')
+    return base64.b64decode(data + '=' * (-len(data) % 4)).decode('utf-16-be')
+  return re.sub(r'&([A-Za-z0-9+,]*)-', utf16, text)
+
+def unescape(part):
+  return mutf7_decode(re.sub(r'\\([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1), 16)), part))
+
+# Maildir++: the maildir is INBOX, ".A.B" is A/B
+def folder_name(entry):
+  return '/'.join(unescape(part) for part in entry[1:].split('.')) if entry else 'INBOX'
+
+def is_dir(path):
+  try:
+    return stat.S_ISDIR(os.lstat(path).st_mode)
+  except OSError:
+    return False
+
+def read_file(path):
+  fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+  with os.fdopen(fd, 'rb') as f:
+    return f.read()
+
+# Prefixes of the shared and public namespaces, their folders belong to others
+def shared_prefixes():
+  prefixes, namespace, depth = {'Shared/'}, None, 0
+  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
+  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
+    stripped = line.strip()
+    if stripped.endswith('{'):
+      depth += 1
+      if depth == 1 and stripped.startswith('namespace'):
+        namespace = {}
+    elif stripped == '}':
+      if depth == 1 and namespace is not None:
+        if namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
+          prefixes.add(namespace['prefix'].split('%')[0])
+        namespace = None
+      depth = max(depth - 1, 0)
+    elif depth == 1 and namespace is not None and '=' in stripped:
+      key, value = (part.strip() for part in stripped.split('=', 1))
+      namespace[key] = value
+  return tuple(prefix for prefix in prefixes if prefix)
+
+# name => guid of the user's own folders (implicit parents have none); the names are looked up literally
+def own_folders(user, prefixes):
+  names = [n for n in doveadm('mailbox', 'list', '-u', user).splitlines() if n and not n.startswith(prefixes)]
+  wanted, guids = set(names), {}
+  for i in range(0, len(names), folder_batch):
+    for row in doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', *names[i:i + folder_batch], partial=True):
+      if row.get('mailbox') in wanted and re.fullmatch(r'[0-9a-f]{32}', row.get('guid') or ''):
+        guids[row['mailbox']] = row['guid']
+  return guids
+
+def masked(user):
+  return '*' in user or '?' in user
+
+def free_name(base, existing):
+  name, count = base, 1
+  while name in existing or any(folder.startswith(name + '/') for folder in existing):
+    count += 1
+    name = '%s (%d)' % (base, count)
+  return name
+
+# Path of the maildir now (/var/vmail/<domain>/<local>/Maildir), None for unknown or inactive mailboxes
+def mail_location(user):
+  result = subprocess.run(['doveadm', 'user', '-f', 'mail', '--', user], capture_output=True, text=True)
+  m = re.match(r'maildir:([^:]+)', result.stdout.strip()) if result.returncode == 0 else None
+  return m.group(1).rstrip('/') if m else None
+
+# The restored maildir of a mailbox (may hold only ACL files), None if the snapshot has nothing of it
+def staged_maildir(staging, home, location):
+  for path in (staging + location, home + '/' + os.environ.get('MAILDIR_SUB', 'Maildir'), home):
+    if is_dir(path):
+      return path
+  return None
+
+def uid_chunks(uids):
+  ranges, start, prev = [], None, None
+  for uid in sorted(uids):
+    if start is None:
+      start = prev = uid
+    elif uid == prev + 1:
+      prev = uid
+    else:
+      ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
+      start = prev = uid
+  if start is not None:
+    ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
+  return [','.join(ranges[i:i + uid_ranges]) for i in range(0, len(ranges), uid_ranges)]
+
+# Same message: same Message-ID, without one the same Date and size
+def message_key(row):
+  message_id = re.sub(r'\s+', '', row.get('hdr.message-id') or '').strip('<>')
+  if message_id:
+    return 'id:' + message_id
+  return 'date:%s:%s' % (row.get('date.sent'), row.get('size.physical'))
+
+def present_keys(user, guids):
+  keys = set()
+  guids = list(guids)
+  for i in range(0, len(guids), folder_batch):
+    search = []
+    for guid in guids[i:i + folder_batch]:
+      search += (['OR'] if search else []) + ['mailbox-guid', guid]
+    keys.update(message_key(row) for row in doveadm_json('fetch', '-u', user, 'hdr.message-id date.sent size.physical',
+      *search))
+  return keys
+
+def location(path):
+  if ':' in path:
+    raise RuntimeError('unsupported path')
+  return 'maildir:%s:INDEX=MEMORY' % path
+
+# The folders of a restored maildir as standalone maildirs (their messages are the INBOX there): the subfolders are
+# moved out of the maildir, so every folder is imported without the name rules of the source (listescape)
+def split_folders(maildir, home):
+  folders, failed = [('INBOX', maildir)], []
+  out = home + '/_restore_folders'
+  for entry in sorted(os.listdir(maildir)):
+    path = os.path.join(maildir, entry)
+    if not entry.startswith('.') or entry in ('.', '..') or not is_dir(path) or not is_dir(path + '/cur'):
+      continue
+    try:
+      name = folder_name(entry)
+    except (ValueError, UnicodeDecodeError):
+      failed.append(entry)
+      continue
+    if not os.path.isdir(out):
+      os.mkdir(out, 0o700)
+    target = '%s/f%d' % (out, len(folders))
+    os.rename(path, target)
+    folders.append((name, target))
+  return folders, failed
+
+def source_messages(user, root):
+  return doveadm_json('-o', 'mail_location=' + location(root), 'fetch', '-u', user,
+    'uid hdr.message-id date.sent size.physical', 'mailbox', 'INBOX')
+
+def remove_tree(user, name):
+  doveadm(*no_quota, 'mailbox', 'delete', '-u', user, '-r', '-s', '--', name, check=False)
+
+def folder_guid(user, name):
+  rows = doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', name)
+  for row in rows:
+    if row.get('mailbox') == name:
+      return row['guid']
+  raise RuntimeError('folder not found after the import')
+
+def quota_room(user):
+  # free bytes of the storage quota, None without a limit
+  for row in doveadm_json('quota', 'get', '-u', user):
+    if row.get('type') == 'STORAGE' and str(row.get('limit', '-')).isdigit():
+      return (int(row['limit']) - int(row.get('value') or 0)) * 1024
+  return None
+
+# Subscriptions of the backup (subscriptions file: "V\t2", then names with \t between the levels)
+def backup_subscriptions(maildir):
+  try:
+    lines = read_file(maildir + '/subscriptions').decode('utf-8', 'replace').splitlines()
+  except OSError:
+    return []
+  if lines and lines[0].startswith('V\t'):
+    return ['/'.join(unescape(part) for part in line.split('\t')) for line in lines[1:] if line.strip()]
+  # version 1: storage names with the Maildir++ separator
+  return [folder_name('.' + line.strip()) for line in lines if line.strip()]
+
+def restore_subscriptions(user, wanted, prefixes):
+  if not wanted:
+    return 0
+  existing = set(own_folders(user, prefixes)) | {'INBOX'}
+  subscribed = set(doveadm('mailbox', 'list', '-u', user, '-s').splitlines())
+  missing = [name for name in dict.fromkeys(wanted) if name in existing and name not in subscribed]
+  if missing:
+    doveadm('mailbox', 'subscribe', '-u', user, '--', *missing)
+  return len(missing)
+
+# Mails folder by folder: original = only messages missing anywhere in the mailbox, into their original folders;
+# folder = all messages below "Restored <snapshot time>"
+def restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota):
+  result = component()
+  result.update(folders=0, folders_failed=0)
+  if not maildir or not is_dir(maildir + '/cur'):
+    # a key the UI translates
+    result['note'] = 'no_mails'
+    return result
+  tmp = 'Restoring %d' % job_id
+  # a leftover of an interrupted run goes first, it is never counted as present
+  remove_tree(user, tmp)
+  subscriptions = backup_subscriptions(maildir)
+  folders, broken = split_folders(maildir, home)
+  for entry in broken:
+    result['errors'].append('folder %s: name not readable' % entry)
+    result['folders_failed'] += 1
+  live = own_folders(user, prefixes)
+  present = present_keys(user, live.values()) if mode == 'original' else set()
+  plan, need = [], 0
+  for name, root in folders:
+    try:
+      rows = source_messages(user, root)
+    except RuntimeError as e:
+      result['errors'].append('%s: %s' % (name, e))
+      result['folders_failed'] += 1
+      continue
+    wanted = [row for row in rows if mode != 'original' or message_key(row) not in present]
+    result['skipped'] += len(rows) - len(wanted)
+    if wanted:
+      size = sum(int(row.get('size.physical') or 0) for row in wanted)
+      plan.append((name, root, [int(row['uid']) for row in wanted], size))
+      need += size
+  if not plan:
+    if mode == 'original':
+      result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
+    return result
+  vfs = os.statvfs('/var/vmail')
+  free, total = vfs.f_bavail * vfs.f_frsize, vfs.f_blocks * vfs.f_frsize
+  if not import_space_ok(need, free, total):
+    raise RuntimeError('not enough free disk space for the mails: %s needed plus %s reserve, %s free' % (
+      mib(need * 11 // 10), mib(disk_reserve(total)), mib(free)))
+  if check_quota:
+    room = quota_room(user)
+    if room is not None and need > room:
+      raise RuntimeError('the mails do not fit into the quota of the mailbox: %s needed, %s free' % (
+        mib(need), mib(max(room, 0))))
+  names = set(live) | {'INBOX'}
+  dest = None
+  if mode != 'original':
+    dest = free_name('Restored ' + snapshot_time, names)
+    doveadm('mailbox', 'create', '-u', user, '-s', '--', dest)
+    names.add(dest)
+    result['folder'] = dest
+  created = []
+  try:
+    for number, (name, root, uids, size) in enumerate(plan):
+      target = dest + '/' + name if dest else name
+      stage = '%s/f%d' % (tmp, number)
+      try:
+        for chunk in uid_chunks(uids):
+          doveadm(*no_quota, 'import', '-u', user, '-s', location(root), stage, 'mailbox', 'INBOX', 'uid', chunk)
+        guid = folder_guid(user, stage + '/INBOX')
+        if target not in names:
+          doveadm('mailbox', 'create', '-u', user, '-s', '--', target)
+          names.add(target)
+          created.append(target)
+        doveadm(*no_quota, 'move', '-u', user, '--', target, 'mailbox-guid', guid, 'all')
+        result['added'] += len(uids)
+        result['folders'] += 1
+      except RuntimeError as e:
+        result['errors'].append('%s: %s' % (name, e))
+        result['folders_failed'] += 1
+  finally:
+    remove_tree(user, tmp)
+  result['folders_created'] = len(created)
+  if mode == 'original':
+    result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
+  return result
+
+def acl_entries(path):
+  entries = []
+  try:
+    lines = read_file(path).decode('utf-8', 'replace').splitlines()
+  except OSError:
+    return entries
+  for line in lines:
+    parts = line.split()
+    # negative rights would take rights away
+    if len(parts) < 2 or parts[0].startswith('-'):
+      continue
+    rights = [rights_by_letter[c] for c in parts[1] if c in rights_by_letter] + [p for p in parts[2:] if p.startswith(':')]
+    if rights:
+      entries.append((parts[0], rights))
+  return entries
+
+# (folder, entries) of the dovecot-acl files of a restored maildir
+def maildir_acls(maildir):
+  acls = []
+  if not maildir or not is_dir(maildir):
+    return acls
+  for entry in [''] + sorted(e for e in os.listdir(maildir) if e.startswith('.') and e not in ('.', '..')):
+    path = os.path.join(maildir, entry, 'dovecot-acl') if entry else os.path.join(maildir, 'dovecot-acl')
+    if entry and not is_dir(os.path.join(maildir, entry)):
+      continue
+    try:
+      if stat.S_ISREG(os.lstat(path).st_mode):
+        acls.append((folder_name(entry), acl_entries(path)))
+    except (OSError, ValueError, UnicodeDecodeError):
+      continue
+  return acls
+
+# owner => (folder, entries) of every mailbox in the ACL staging (domain admins: only their domains)
+def staged_acls(acl_dir, allowed):
+  acls = {}
+  base = (acl_dir or '') + '/var/vmail'
+  sub = os.environ.get('MAILDIR_SUB', '')
+  if not acl_dir or not is_dir(base):
+    return acls
+  for domain in os.listdir(base):
+    if '.' not in domain or not is_dir(os.path.join(base, domain)) or (allowed is not None and domain.lower() not in allowed):
+      continue
+    for local in os.listdir(os.path.join(base, domain)):
+      home = os.path.join(base, domain, local)
+      maildir = os.path.join(home, sub) if sub and is_dir(os.path.join(home, sub)) else home
+      found = maildir_acls(maildir)
+      if found:
+        acls['%s@%s' % (local, domain)] = found
+  return acls
+
+class AclRestore:
+  def __init__(self, prefixes, allowed):
+    self.prefixes = prefixes
+    self.allowed = allowed
+    self.users, self.folders = {}, {}
+    # anyone/authenticated only when the configuration allows them
+    self.anyone = os.environ.get('ACL_ANYONE', 'disallow') == 'allow'
+
+  def in_scope(self, address):
+    return self.allowed is None or address.rpartition('@')[2].lower() in self.allowed
+
+  def user_exists(self, user):
+    if user not in self.users:
+      self.users[user] = query("SELECT 1 FROM mailbox WHERE username = %s", (user,)).fetchone() is not None
+    return self.users[user]
+
+  def identifier_ok(self, identifier):
+    if identifier.startswith('user='):
+      return self.user_exists(identifier[5:].lower())
+    # group=mcg_<id> belongs to the mailbox groups, the group grants cron keeps them
+    if identifier.startswith('group=mcg_'):
+      return False
+    if identifier in ('anyone', 'authenticated'):
+      return self.anyone
+    return True
+
+  def shown(self, identifier):
+    # domain admins do not see addresses of other domains
+    if identifier.startswith('user=') and not self.in_scope(identifier[5:]):
+      return 'user=(other domain)'
+    return identifier
+
+  def add(self, owner, folder, identifier, rights, result):
+    if not self.in_scope(owner) or masked(owner):
+      result['skipped'] += 1
+      return
+    try:
+      if owner not in self.folders:
+        self.folders[owner] = set(own_folders(owner, self.prefixes)) | {'INBOX'}
+      if folder not in self.folders[owner] or not self.identifier_ok(identifier):
+        result['skipped'] += 1
+        return
+      have = set()
+      for row in doveadm_json('acl', 'get', '-u', owner, '--', folder):
+        if row.get('id') == identifier:
+          have.update((row.get('rights') or '').split())
+      missing = [right for right in rights if right not in have]
+      if not missing:
+        result['skipped'] += 1
+        return
+      doveadm('acl', 'add', '-u', owner, '--', folder, identifier, *missing)
+      result['added'] += 1
+    except RuntimeError as e:
+      result['errors'].append('%s %s %s: %s' % (owner, folder, self.shown(identifier), e))
+
+# Rights the mailbox granted on its folders and rights other mailboxes granted to it; added, never removed
+def restore_acl(acl, user, own, granted):
+  result = component()
+  for folder, entries in own:
+    for identifier, rights in entries:
+      acl.add(user, folder, identifier, rights, result)
+  if acl.user_exists(user):
+    for owner, folders in granted.items():
+      if owner == user or not acl.in_scope(owner) or not acl.user_exists(owner):
+        continue
+      for folder, entries in folders:
+        for identifier, rights in entries:
+          if identifier.lower() == 'user=' + user:
+            acl.add(owner, folder, identifier, rights, result)
+  return result
+
+# Own Sieve scripts (incl. the SOGo vacation script): missing names are added; with activate the active script of the
+# backup is activated if none is active now
+def restore_sieve(user, home, active_link, activate):
+  result = component()
+  folder = home + '/sieve'
+  scripts = []
+  if is_dir(folder):
+    scripts = sorted(f[:-6] for f in os.listdir(folder) if f.endswith('.sieve') and stat.S_ISREG(os.lstat(os.path.join(folder, f)).st_mode))
+  current, active = set(), None
+  for line in doveadm('sieve', 'list', '-u', user).splitlines():
+    name, _, flag = line.rpartition(' ')
+    if name:
+      current.add(name)
+      if flag == 'ACTIVE':
+        active = name
+  for name in scripts:
+    if name in current:
+      result['skipped'] += 1
+      continue
+    try:
+      doveadm('sieve', 'put', '-u', user, '--', name, data=read_file(os.path.join(folder, name + '.sieve')))
+      current.add(name)
+      result['added'] += 1
+    except (OSError, RuntimeError) as e:
+      result['errors'].append('%s: %s' % (name, e))
+  if not activate:
+    return result
+  target = os.readlink(active_link) if os.path.islink(active_link) else ''
+  backup_active = os.path.basename(target)[:-6] if target.endswith('.sieve') else None
+  if backup_active and not active and backup_active in current:
+    try:
+      doveadm('sieve', 'activate', '-u', user, '--', backup_active)
+      result['activated'] = backup_active
+    except RuntimeError as e:
+      result['errors'].append('%s: %s' % (backup_active, e))
+  return result
+
+# filters: backup.php reported its part already, the counts are added up
+def merge(results, key, part):
+  old = results.get(key)
+  if not isinstance(old, dict):
+    results[key] = part
+    return
+  for count in ('added', 'skipped'):
+    old[count] = (old.get(count) or 0) + part.get(count, 0)
+  old['errors'] = (old.get('errors') or []) + part.get('errors', [])
+  for name, value in part.items():
+    if name not in ('added', 'skipped', 'errors'):
+      old[name] = value
+
+def count_errors(result):
+  count = sum(len(result.get(key, {}).get('errors') or []) for key in ('domain', 'aliases') if isinstance(result.get(key), dict))
+  for entry in (result.get('mailboxes') or {}).values():
+    count += int(bool(entry.get('error')))
+    count += sum(len(value.get('errors') or []) for value in entry.values() if isinstance(value, dict))
+  return count
+
+def restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, result):
+  local, _, domain = user.rpartition('@')
+  home = '%s/var/vmail/%s/%s' % (staging, domain, local)
+  if masked(user):
+    for key in components & {'mails', 'imap_acl', 'filters'}:
+      merge(result, key, component(['username with * or ?, doveadm reads it as a mask']))
+    return
+  location = mail_location(user)
+  if not location:
+    for key in components & {'mails', 'imap_acl', 'filters'}:
+      merge(result, key, component(['the mailbox does not exist now or is inactive']))
+    return
+  maildir = staged_maildir(staging, home, location)
+  # the ACL files are read before the folders of the maildir are moved for the import
+  own = maildir_acls(maildir) if 'imap_acl' in components else []
+  if 'mails' in components:
+    try:
+      result['mails'] = restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota)
+    except Exception as e:
+      result['mails'] = component([str(e)[:1000]])
+    try:
+      doveadm('quota', 'recalc', '-u', user)
+    except RuntimeError as e:
+      result['mails']['errors'].append('quota recalc: %s' % e)
+    # new folders
+    acl.folders.pop(user, None)
+  if 'imap_acl' in components:
+    try:
+      result['imap_acl'] = restore_acl(acl, user, own, granted)
+    except Exception as e:
+      result['imap_acl'] = component([str(e)[:1000]])
+  if 'filters' in components:
+    try:
+      part = restore_sieve(user, home, '%s/var/vmail/sieve/%s.sieve' % (staging, user), 'sieve_activate' in components)
+    except Exception as e:
+      part = component([str(e)[:1000]])
+    merge(result, 'filters', part)
+
+def fail(job_id, error):
+  query("""UPDATE backup_job SET state = IF(state = 'cancelled', state, 'failed'), error = %s, finished = NOW()
+    WHERE id = %s""", (error, job_id))
+  log(job_id, 'restore job failed: ' + error)
+  notify(job_id, error)
+
+def notify(job_id, error):
+  hostname = os.environ.get('MAILCOW_HOSTNAME', '')
+  row = query("SELECT value FROM backup_setting WHERE name = 'notify_email'").fetchone()
+  rcpts = [r.strip() for r in (row['value'] if row else '').split(',') if re.fullmatch(r'[^@\s<>,]+@[^@\s<>,]+', r.strip())]
+  if not rcpts:
+    return
+  msg = EmailMessage()
+  msg['From'] = 'backup@' + hostname
+  msg['To'] = ', '.join(rcpts)
+  msg['Subject'] = 'mailcow backup on %s: restore job %d failed' % (hostname, job_id)
+  msg['Date'] = formatdate(localtime=True)
+  msg['Message-ID'] = make_msgid(domain=hostname or None)
+  msg.set_content('The restore job %d failed:\n\n%s\n\nThe job log is in the backup section of the mailcow UI.\n'
+    % (job_id, error))
+  try:
+    with smtplib.SMTP('postfix', 588, hostname or None, timeout=60) as smtp:
+      smtp.send_message(msg)
+  except (OSError, smtplib.SMTPException) as e:
+    print('restore job %d: the notification could not be sent: %s' % (job_id, e), file=sys.stderr)
+
+def staging_dir(job_id, path):
+  # the batch folder of this job, never a symlink
+  if not isinstance(path, str) or not re.fullmatch(r'/restore/%d/[A-Za-z0-9_-]+' % job_id, path) or \
+      os.path.realpath(path) != path or not is_dir(path):
+    return None
+  return path
+
+def run_batch(job):
+  job_id = job['id']
+  params = json.loads(job['params'] or '{}')
+  result = json.loads(job['result'] or '{}')
+  batch = result.get('batch') or {}
+  staging = staging_dir(job_id, batch.get('dir'))
+  acl_dir = staging_dir(job_id, batch.get('acl_dir')) if batch.get('acl_dir') else None
+  mailboxes = [m for m in batch.get('mailboxes') or [] if isinstance(m, str)]
+  components = {c for c in params.get('components') or [] if c in components_here}
+  mode = 'original' if params.get('mail_target') == 'original' else 'folder'
+  allowed = {d.lower() for d in params['allowed_domains']} if isinstance(params.get('allowed_domains'), list) else None
+  check_quota = params.get('requested_role', 'admin') != 'admin'
+  snapshot_time = (result.get('snapshot') or {}).get('time') or time.strftime('%Y-%m-%d %H:%M')
+  results = result.setdefault('mailboxes', {})
+  prefixes = shared_prefixes()
+  acl = AclRestore(prefixes, allowed)
+  granted = staged_acls(acl_dir, allowed) if 'imap_acl' in components else {}
+  log(job_id, 'restoring %s for %d mailboxes%s' % (', '.join(sorted(components)), len(mailboxes),
+    ' (mails: %s)' % mode if 'mails' in components else ''))
+  for user in mailboxes:
+    entry = results.setdefault(user, {})
+    if entry.get('error'):
+      continue
+    started = time.monotonic()
+    if not staging:
+      for key in components & {'mails', 'imap_acl', 'filters'}:
+        merge(entry, key, component(['the restored files are missing']))
+    else:
+      restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, entry)
+    summary = ', '.join('%s +%d' % (key, entry[key].get('added', 0)) for key in sorted(components) if isinstance(entry.get(key), dict))
+    errors = sum(len(entry[key].get('errors') or []) for key in components if isinstance(entry.get(key), dict))
+    log(job_id, '%s: %s%s (%d s)' % (user, summary, ', %d errors' % errors if errors else '', time.monotonic() - started))
+    result['errors'] = count_errors(result)
+    save(job_id, result)
+  result['errors'] = count_errors(result)
+  save(job_id, result)
+  # the restic runner removes the staging and goes on
+  query("UPDATE backup_job SET step = 'mails_done' WHERE id = %s AND state = 'running' AND step = 'mails_running'", (job_id,))
+
+def main():
+  global conn
+  if os.geteuid() == 0:
+    # as vmail like the ofelia job: a run as root would leave a lock file the runs as vmail cannot open
+    os.execv('/usr/local/bin/gosu', ['gosu', 'vmail', sys.executable, os.path.abspath(__file__)] + sys.argv[1:])
+  os.umask(0o077)
+  # released by the kernel when the process ends, a crashed run leaves no stale lock
+  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+  try:
+    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  except OSError:
+    sys.exit("restore: can't lock " + lock_file)
+  conn = connect()
+  heartbeat = Heartbeat()
+  heartbeat.start()
+  try:
+    # under the lock: a batch in mails_running was left by a crashed or stopped run, its temporary folders go
+    for row in query("""SELECT id, result FROM backup_job WHERE type = 'restore' AND state = 'running'
+        AND step = 'mails_running'""").fetchall():
+      try:
+        for user in (json.loads(row['result'] or '{}').get('batch') or {}).get('mailboxes') or []:
+          remove_tree(user, 'Restoring %d' % row['id'])
+          doveadm('quota', 'recalc', '-u', user, check=False)
+      except (ValueError, AttributeError, RuntimeError):
+        pass
+      fail(row['id'], 'interrupted: the restore of the mails was stopped')
+    last, waited = None, 0
+    while True:
+      job = query("""SELECT id, params, result FROM backup_job WHERE type = 'restore' AND state = 'running'
+        AND step = 'mails' ORDER BY id LIMIT 1""").fetchone()
+      if not job:
+        # the next batch of a restore follows within seconds: wait for it instead of the next ofelia run
+        row = query("SELECT state FROM backup_job WHERE id = %s", (last,)).fetchone() if last else None
+        if row and row['state'] == 'running' and waited < batch_wait:
+          time.sleep(2)
+          waited += 2
+          continue
+        break
+      last, waited = job['id'], 0
+      if query("UPDATE backup_job SET step = 'mails_running' WHERE id = %s AND state = 'running' AND step = 'mails'",
+          (job['id'],)).rowcount != 1:
+        continue
+      try:
+        run_batch(job)
+      except Exception as e:
+        fail(job['id'], str(e)[:1000])
+  finally:
+    heartbeat.done.set()
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/dovecot/retention_runner.py

@@ -0,0 +1,256 @@
+#!/usr/bin/python3
+# Nightly run of the retention rules (ofelia, dovecot container): gets the plan page by page from the mailcow UI
+# (nginx:8081/retention.php), gives every folder of a mailbox to the first entry of its plan that matches it, counts
+# the mail saved in the folders before the period of the rule with doveadm, deletes it unless the rule only
+# simulates, and reports the results back.
+
+import fcntl
+import hashlib
+import hmac
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+plan_url = 'http://nginx:8081/retention.php'
+lock_file = '/tmp/retention_runner.lock'
+# folder keys of the rules and the special-use flag of their folders in the Dovecot configuration
+special_use = {'trash': '\\Trash', 'junk': '\\Junk', 'sent': '\\Sent', 'drafts': '\\Drafts', 'archive': '\\Archive'}
+own_folder = 'folder:'
+# mailboxes per page of the plan and results per report, a large instance has many mailboxes
+plan_page = 200
+report_batch = 200
+# a request is tried this often, waiting longer each time
+tries = 3
+
+token = hmac.new(os.environ.get('DBPASS', '').encode(), b'retention', hashlib.sha256).hexdigest()
+
+def request(url, body=None):
+  data = None if body is None else json.dumps(body).encode()
+  req = urllib.request.Request(url, data=data, headers={'Content-Type': 'application/json', 'X-Retention-Token': token},
+    method='GET' if body is None else 'POST')
+  with urllib.request.urlopen(req, timeout=300) as response:
+    return json.load(response)
+
+def retried(url, body=None):
+  for attempt in range(1, tries + 1):
+    try:
+      return request(url, body)
+    except Exception as e:
+      if attempt == tries:
+        raise
+      print('retention: request failed, trying again: %s' % e, file=sys.stderr)
+      time.sleep(30 * attempt)
+
+def plan():
+  # one page after the other, so memory stays small; rule changes apply to the pages fetched after them
+  after = ''
+  while after is not None:
+    page = retried(plan_url + '?' + urllib.parse.urlencode({'after': after, 'limit': plan_page}))
+    yield from page['mailboxes']
+    after = page['next']
+
+def post(body):
+  # False if the report could not be sent
+  try:
+    retried(plan_url, body)
+    return True
+  except Exception as e:
+    print('retention: report failed: %s' % e, file=sys.stderr)
+    return False
+
+class Reports:
+  # results go back in parts; once a part cannot be sent, the rest waits for the end of the run
+  def __init__(self):
+    self.pending, self.failed = [], False
+
+  def add(self, result):
+    self.pending.append(result)
+    if len(self.pending) >= report_batch and not self.failed:
+      self.failed = not self.send()
+
+  def send(self, summary=None, run=None):
+    # full parts, at the end all results with the summary and the run in the last part
+    final = summary is not None
+    while len(self.pending) >= report_batch or final:
+      body = {'results': self.pending[:report_batch]}
+      if final and len(self.pending) <= report_batch:
+        body.update(summary=summary, run=run)
+      if not post(body):
+        return False
+      del self.pending[:report_batch]
+      if 'summary' in body:
+        break
+    return True
+
+def doveadm(*args, partial=False):
+  result = subprocess.run(['doveadm', *args], capture_output=True, text=True, timeout=3600)
+  # mailbox status also fails for single folders that cannot be opened; the others are still listed
+  if result.returncode != 0 and not (partial and result.stdout):
+    raise RuntimeError((result.stderr or result.stdout).strip()[:500] or 'doveadm exit %d' % result.returncode)
+  return result.stdout
+
+def configuration():
+  # names of the special folders per key, and the prefixes of the shared and public namespaces (not the user's)
+  names = {key: set() for key in special_use}
+  prefixes = ['Shared/']
+  namespace, mailbox = None, None
+  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
+  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
+    stripped = line.strip()
+    if line.startswith('namespace ') and line.endswith('{'):
+      namespace = {}
+    elif line == '}':
+      if namespace and namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
+        prefixes.append(namespace['prefix'].split('%')[0])
+      namespace = None
+    elif namespace is None:
+      continue
+    elif re.match(r'mailbox .+ \{$', stripped):
+      mailbox = stripped[len('mailbox '):-2].strip().strip('"')
+    elif stripped == '}':
+      mailbox = None
+    elif '=' in stripped:
+      key, value = (part.strip() for part in stripped.split('=', 1))
+      if mailbox is not None and key == 'special_use':
+        for folder, flag in special_use.items():
+          if flag in value.split():
+            names[folder].add(mailbox)
+      elif mailbox is None and key in ('type', 'prefix'):
+        namespace[key] = value
+  if not names['trash']:
+    raise RuntimeError('no special folders in the output of doveconf -n')
+  return names, [prefix for prefix in prefixes if prefix]
+
+def own_folders(user, prefixes):
+  # folder name => guid of the user's own folders; listed first, so folders others share are not opened
+  names = set(name for name in doveadm('-f', 'tab', 'mailbox', 'list', '-u', user).splitlines()[1:]
+    if name and not any((name + '/').startswith(prefix) for prefix in prefixes))
+  folders = {}
+  if not names:
+    return folders
+  for line in doveadm('-f', 'tab', 'mailbox', 'status', '-u', user, '--', 'guid', *sorted(names), partial=True).splitlines()[1:]:
+    name, _, guid = line.rpartition('\t')
+    if name in names and re.fullmatch(r'[0-9a-f]{32}', guid):
+      folders[name] = guid
+  return folders
+
+def canonical(name):
+  # Dovecot writes INBOX in capitals, also as the parent of folders
+  return 'INBOX' + name[5:] if name[:5].upper() == 'INBOX' and name[5:6] in ('', '/') else name
+
+def matches(action, name, names):
+  # match: the folder itself, an own folder name with *, the subfolders of a special folder or the inbox, all folders
+  key, match = action['folder'], action['match']
+  if match == 'all':
+    return True
+  if key.startswith(own_folder):
+    # * is the only wildcard of own folder names
+    pattern = '.*'.join(re.escape(part) for part in canonical(key[len(own_folder):]).split('*'))
+    return re.fullmatch(pattern, name) is not None
+  folders = ['INBOX'] if key == 'inbox' else names.get(key, ())
+  if match == 'subfolders':
+    return any(name.startswith(folder + '/') for folder in folders)
+  return name in folders
+
+def decide(actions, folders, names):
+  # rule id => (its action, the folders it decides about): every folder goes to the first action that matches it
+  decided = {}
+  for name in sorted(folders):
+    for action in actions:
+      if matches(action, canonical(name), names):
+        decided.setdefault(action['rule_id'], (action, []))[1].append(name)
+        break
+  return decided
+
+def found(user, query):
+  return len(doveadm('search', '-u', user, '--', *query).splitlines())
+
+def clean(user, action, selected, folders, result):
+  # name and guid together: doveadm then only opens these folders
+  query = []
+  for name in selected:
+    query += (['OR'] if query else []) + ['(', 'mailbox', name, 'mailbox-guid', folders[name], ')']
+  query = ['('] + query + [')', 'savedbefore', '%dd' % int(action['days'])]
+  if action['keep_flagged']:
+    query += ['NOT', 'FLAGGED']
+  result['matched'] = found(user, query)
+  if not result['matched'] or action['mode'] != 'delete':
+    return
+  try:
+    doveadm('expunge', '-u', user, '--', *query)
+    result['deleted'] = result['matched']
+  except Exception as e:
+    # part of the mail may be gone: what is left is counted
+    result['error'] = str(e)[:1000]
+    try:
+      result['deleted'] = max(0, result['matched'] - found(user, query))
+    except Exception:
+      pass
+
+def main():
+  if os.geteuid() == 0:
+    # as vmail like the nightly job: a run as root would leave a lock file the runs as vmail cannot open
+    os.execv('/usr/local/bin/gosu', ['gosu', 'vmail', sys.executable, os.path.abspath(__file__)] + sys.argv[1:])
+  # released by the kernel when the process ends, a crashed run leaves no stale lock
+  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+  try:
+    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  except OSError:
+    sys.exit("retention: can't lock " + lock_file)
+  reports = Reports()
+  run = {'started': int(time.time()), 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0, 'error': ''}
+  summary = {}
+  try:
+    names, prefixes = configuration()
+    for mailbox in plan():
+      user = mailbox['username']
+      run['mailboxes'] += 1
+      try:
+        if '*' in user or '?' in user:
+          # doveadm -u would run for every user the name matches as a mask
+          raise RuntimeError('username with * or ?, doveadm reads it as a mask')
+        folders, error = own_folders(user, prefixes), ''
+        decided = decide(mailbox['actions'], folders, names)
+      except Exception as e:
+        # the folders are unknown: an error for every rule of the mailbox
+        folders, error, decided = {}, str(e)[:1000], {}
+        for action in mailbox['actions']:
+          decided.setdefault(action['rule_id'], (action, []))
+      for rule_id, (action, selected) in decided.items():
+        rule = summary.setdefault(rule_id, {'rule_id': rule_id, 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0,
+          'simulated': int(action['mode'] != 'delete')})
+        # the log keeps 10000 characters of the folder names, more would only make the report too large
+        result = {'username': user, 'rule_id': rule_id, 'folder': '\n'.join(selected)[:10000], 'days': action['days'], 'matched': 0,
+          'deleted': 0, 'simulated': rule['simulated'], 'error': error}
+        if not error:
+          try:
+            clean(user, action, selected, folders, result)
+          except Exception as e:
+            result['error'] = str(e)[:1000]
+        for counter in (rule, run):
+          counter['matched'] += result['matched']
+          counter['deleted'] += result['deleted']
+          counter['errors'] += int(result['error'] != '')
+        rule['mailboxes'] += 1
+        # only mailboxes with found mail or an error are logged, every rule gets a summary
+        if result['matched'] or result['error']:
+          reports.add(result)
+  except Exception as e:
+    run['error'] = str(e)[:1000]
+  reported = reports.send(list(summary.values()), run)
+  print('retention: %d mailboxes, %d mails deleted, %d found by simulating rules, %d errors' % (run['mailboxes'],
+    run['deleted'], sum(rule['matched'] for rule in summary.values() if rule['simulated']), run['errors']))
+  if run['error']:
+    print('retention: %s' % run['error'], file=sys.stderr)
+  if not reported:
+    print('retention: the results could not be reported', file=sys.stderr)
+  if run['error'] or not reported:
+    sys.exit(1)
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/dovecot/trim_logs.sh

@@ -4,7 +4,7 @@ catch_non_zero() {
   ${CMD} > /dev/null
   EC=$?
   if [ ${EC} -ne 0 ]; then
-    echo "Command ${CMD} failed to execute, exit code was ${EC}"
+    echo "Command ${CMD//${REDISPASS}/***} failed to execute, exit code was ${EC}"
   fi
 }
 source /source_env.sh

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -7,9 +7,6 @@ class BanManager:
   and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
   (chains, SNAT), because the backends must not run in parallel."""
 
-  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
-  EXCUSE_TIME = 60
-
   def __init__(self, r, tables, options, logger, lock, clock=time.time):
     self.r = r
     self.tables = tables
@@ -21,16 +18,6 @@ class BanManager:
     self.banned = set()
     self.denied = {}
     self.allowlist = []
-    self.excused = {}
-
-  def excuse(self, ip):
-    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
-    with self.lock:
-      now = self.clock()
-      if self.excused.get(ip, 0) <= now:
-        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
-          ip, self.EXCUSE_TIME))
-      self.excused[ip] = now + self.EXCUSE_TIME
 
   def attempt(self, ip):
     """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
@@ -43,9 +30,6 @@ class BanManager:
       if allowed is not None:
         self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
         return
-      if self.excused.get(ip, 0) > now:
-        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
-        return
       if net in self.banned:
         # log lines of connections from before the ban
         return
@@ -208,8 +192,6 @@ class BanManager:
       for net in [net for net, counter in self.counters.items()
                   if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
         del self.counters[net]
-      for ip in [ip for ip, until in self.excused.items() if until <= now]:
-        del self.excused[ip]
 
   def apply(self, nets):
     for net in self.tables.ban(nets):

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -109,12 +109,6 @@ class Netfilter:
   def handle_line(self, line):
     # a broken line must not end the watcher
     try:
-      excused = Regexes.temporary_failure(line)
-      if excused is not None:
-        ip = Regexes.parse_ip(excused)
-        if self.external(ip):
-          self.bans.excuse(ip)
-        return
       result = self.regexes.match(line)
       if result is None:
         return

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -6,20 +6,43 @@ import re
 class Regexes:
   """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
 
+  # Anchored to the line start: text clients control in other lines (login names, SMTP commands) must not match.
+  # Failed IMAP, POP3, Sieve and SMTP logins are reported by mailcowauth.php (rule 3), not by Dovecot or Postfix.
+  # Only the master passdb is checked by Dovecot itself (rule 8); the last address is the one Dovecot added.
   DEFAULTS = {
-    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
-    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
-    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
-    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
-    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
-    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
-    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
-    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
-    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+    1: r'^mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)$',
+    2: r'^Rspamd UI: Invalid password by ([0-9a-f\.:]+)$',
+    3: r'^MAILCOWAUTH: Login failed for user .+ with service \w* from IP ([0-9a-f\.:]+)$',
+    4: r'^warning: non-SMTP command from [^\[\]\s]*\[([0-9a-f\.:]+)\]:.+',
+    5: r'^NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    6: r'^(?:\[\d+\]: )?SOGo\w* Login from \'(?:[0-9a-f\.:]+, )*([0-9a-f\.:]+)\' for user .+ might not have worked',
+    7: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+    8: r'^auth: passwd-file\(.*,([0-9a-f\.:]+),master(?:,<[^<>]*>)?\): (?:Password mismatch|unknown user)',
   }
-  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
-  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
-  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+  # Former default rules (with any rule id) => rule id of their replacement, None: the rule is removed.
+  # Rules the admin changed are kept.
+  FORMER_DEFAULTS = {
+    r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)': 1,
+    r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)': 2,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed': 3,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+': 3,
+    r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+': 4,
+    r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+': 5,
+    r"SOGo.+ Login from '([0-9a-f\.:]+)' for user .+ might not have worked": 6,
+    r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked': 6,
+    r'-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'auth: static\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'auth: \w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)': None,
+  }
+  # Lines of the web UI are not sanitized like syslog lines: a newline in a login name must not hide the address
+  CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f]')
 
   def __init__(self, logger):
     self.logger = logger
@@ -42,6 +65,10 @@ class Regexes:
     if not isinstance(stored, dict):
       self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
       return
+    migrated = self.migrate(stored)
+    if migrated != stored:
+      self.store_migrated(r)
+      stored = migrated
 
     rules = []
     for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
@@ -54,6 +81,7 @@ class Regexes:
 
   def match(self, line):
     """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    line = self.CONTROL_CHARS.sub('_', line)
     for rule_id, regex in self.rules:
       result = regex.search(line)
       if result and result.lastindex and result.group(1):
@@ -61,10 +89,35 @@ class Regexes:
     return None
 
   @classmethod
-  def temporary_failure(cls, line):
-    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
-    result = cls.TEMPORARY_FAILURE.search(line)
-    return result.group(1) if result else None
+  def migrate(cls, stored):
+    """The stored rules with former default rules replaced by the current ones (or removed)"""
+    migrated = {}
+    for rule_id, regex in stored.items():
+      if isinstance(regex, str) and regex in cls.FORMER_DEFAULTS:
+        replacement = cls.FORMER_DEFAULTS[regex]
+        if replacement is None:
+          continue
+        regex = cls.DEFAULTS[replacement]
+      migrated[rule_id] = regex
+    return migrated
+
+  def store_migrated(self, r):
+    # WATCH keeps a save of the UI between reading and writing
+    def replace(pipe):
+      current = json.loads(pipe.get('F2B_REGEX') or '{}')
+      if not isinstance(current, dict):
+        return
+      migrated = self.migrate(current)
+      if migrated == current:
+        return
+      pipe.multi()
+      pipe.set('F2B_REGEX', json.dumps(migrated, ensure_ascii=False))
+
+    try:
+      r.transaction(replace, 'F2B_REGEX')
+      self.logger.logInfo('Replaced former default rules in F2B_REGEX')
+    except Exception as ex:
+      self.logger.logWarn('Could not replace former default rules in F2B_REGEX: %s' % ex)
 
   @staticmethod
   def parse_ip(text):

data/Dockerfiles/nginx/Dockerfile

@@ -3,7 +3,8 @@ LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
-RUN apk add --no-cache nginx \
+RUN apk upgrade --no-cache && \
+  apk add --no-cache nginx \
   python3 \
   py3-pip && \
   pip install --upgrade pip && \

data/Dockerfiles/nginx/bootstrap.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 from jinja2 import Environment, FileSystemLoader
 
@@ -44,6 +45,11 @@ def prepare_template_vars():
   ipv4_network = os.getenv("IPV4_NETWORK", "172.22.1")
   additional_server_names = os.getenv("ADDITIONAL_SERVER_NAMES", "")
   trusted_proxies = os.getenv("TRUSTED_PROXIES", "")
+  # nginx size syntax, invalid values fall back to the default
+  max_body_size = os.getenv("NGINX_MAX_BODY_SIZE", "128m").strip()
+  if not re.fullmatch(r'[0-9]+[kKmMgG]?', max_body_size):
+    print(f"Invalid NGINX_MAX_BODY_SIZE '{max_body_size}', using 128m")
+    max_body_size = "128m"
 
   template_vars = {
     'IPV4_NETWORK': ipv4_network,
@@ -60,6 +66,7 @@ def prepare_template_vars():
     'PHPFPMHOST': os.getenv("PHPFPMHOST", "php-fpm-mailcow"),
     'ENABLE_IPV6': os.getenv("ENABLE_IPV6", "true").lower() != "false",
     'HTTP_REDIRECT': os.getenv("HTTP_REDIRECT", "n").lower() in ("y", "yes"),
+    'NGINX_MAX_BODY_SIZE': max_body_size,
   }
 
   ssl_dir = '/etc/ssl/mail/'

data/Dockerfiles/postfix-tlspol/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /src
 ENV CGO_ENABLED=0 \
     GO111MODULE=on \
 		NOOPT=1 \
-    VERSION=1.14.0
+    VERSION=1.14.5
 
 RUN git clone --branch v${VERSION} https://github.com/Zuplu/postfix-tlspol && \
     cd /src/postfix-tlspol && \

data/Dockerfiles/postfix/Dockerfile

@@ -26,6 +26,7 @@ RUN groupadd -g 102 postfix \
 	postfix \
 	postfix-mysql \
 	postfix-pcre \
+	python3 \
 	redis-tools \
 	sasl2-bin \
 	sudo \
@@ -46,6 +47,7 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
+COPY mailcow-policy.py /usr/local/bin/mailcow-policy.py
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
@@ -53,6 +55,7 @@ RUN chmod +x /opt/postfix.sh \
   /usr/local/bin/rspamd-pipe-ham \
   /usr/local/bin/rspamd-pipe-spam \
   /usr/local/bin/whitelist_forwardinghosts.sh \
+  /usr/local/bin/mailcow-policy.py \
   /usr/local/sbin/stop-supervisor.sh
 RUN rm -rf /tmp/* /var/tmp/*
 

data/Dockerfiles/postfix/mailcow-policy.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+# Policy service of the mail policies (mailcow-policy in master.cf, spawned by Postfix): reads the policy requests
+# of smtpd and asks the mailcow UI for the action (nginx:8081/mailpolicy.php). Without an answer the recipient is
+# deferred; smtpd only asks while policies or internal aliases exist (mysql_mail_policy_maps.cf).
+
+import http.client
+import re
+import socket
+import sys
+from urllib.parse import urlencode
+
+# one line: DUNNO or a rejection with code and text
+valid_action = re.compile(rb'DUNNO|[45]\d\d [45]\.\d{1,3}\.\d{1,3} [\x20-\x7e]{1,200}')
+unavailable = b'DEFER_IF_PERMIT Mail policy check temporarily unavailable'
+# address of nginx, resolved once per process (a lookup costs more than the check)
+nginx = None
+
+def check(request):
+  global nginx
+  body = urlencode(sorted(request.items())).encode()
+  try:
+    if nginx is None:
+      nginx = socket.gethostbyname('nginx')
+    # http.client sets TCP_NODELAY, so the body is not held back until the headers are acknowledged
+    # below the timeout of Postfix (main.cf), so an answer always arrives
+    conn = http.client.HTTPConnection(nginx, 8081, timeout=8)
+    try:
+      conn.request('POST', '/mailpolicy.php', body, {'Host': 'nginx', 'Content-Type': 'application/x-www-form-urlencoded'})
+      response = conn.getresponse()
+      action = response.read()
+    finally:
+      conn.close()
+  except (OSError, http.client.HTTPException):
+    # nginx may have a new address after a restart
+    nginx = None
+    return unavailable
+  return action if response.status == 200 and valid_action.fullmatch(action) else unavailable
+
+def main():
+  request = {}
+  # attributes as bytes: addresses may contain any 8-bit characters
+  for line in sys.stdin.buffer:
+    line = line.rstrip(b'\r\n')
+    if line:
+      name, separator, value = line.partition(b'=')
+      if separator:
+        request[name] = value
+      continue
+    sys.stdout.buffer.write(b'action=' + check(request) + b'\n\n')
+    sys.stdout.buffer.flush()
+    request = {}
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/postfix/postfix.sh

@@ -15,6 +15,21 @@ until dig +short mailcow.email > /dev/null; do
   sleep 1
 done
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+done
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
+else
+  echo "DB schema is ${DBV_NOW}"
+fi
+
 cat <<EOF > /etc/aliases
 # Autogenerated by mailcow
 null: /dev/null
@@ -163,6 +178,45 @@ query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM tra
   AND destination = '%s';
 EOF
 
+# Archive copies (journal) to the archive set in the admin UI: on port 591 the sender archive-bounce@localhost
+# sends the whole mail through archive_smtp (master.cf), whatever the recipient domain
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT('FILTER archive_smtp:', nexthop) AS action FROM archive_transport
+  WHERE active = '1'
+  AND '%s' = 'archive-bounce@localhost'
+  LIMIT 1;
+EOF
+
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM archive_transport
+  WHERE nexthop = '%s'
+  AND active = '1'
+  AND username != ''
+  LIMIT 1;
+EOF
+
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT tls_policy FROM archive_transport
+  WHERE nexthop = '%s'
+  AND active = '1'
+  LIMIT 1;
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_resource_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}
@@ -224,6 +278,19 @@ query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports
   LIMIT 1;
 EOF
 
+# Mail policies: while policies or internal aliases exist, smtpd asks the policy service (mailcow_mail_policy in main.cf)
+cat <<EOF > /opt/postfix/conf/sql/mysql_mail_policy_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT 'mailcow_mail_policy' FROM DUAL WHERE '%s' != ''
+  AND (EXISTS (SELECT 1 FROM mail_policy_member JOIN mail_policy ON mail_policy.id = mail_policy_member.policy_id
+      WHERE mail_policy.active = '1')
+    OR EXISTS (SELECT 1 FROM alias WHERE internal = '1' AND active IN ('1', '2')));
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}

data/Dockerfiles/restic/Dockerfile

@@ -0,0 +1,28 @@
+FROM alpine:3.24
+
+LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, the runner connects through the socket
+ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
+
+RUN apk add --no-cache --update \
+  ca-certificates \
+  git \
+  mariadb-client \
+  openssh-client \
+  py3-mysqlclient \
+  py3-pynacl \
+  python3 \
+  restic \
+  tini \
+  tzdata \
+  # dump, export, manifest and the mailcow directory are below /mailcow (paths of the snapshot), only root may read them
+  && mkdir -m 700 /mailcow
+
+COPY restic_runner.py /usr/local/bin/restic_runner.py
+COPY sogo_restore.py /usr/local/bin/sogo_restore.py
+
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD ["/usr/bin/python3", "-u", "/usr/local/bin/restic_runner.py"]

data/Dockerfiles/restic/sogo_restore.py

@@ -0,0 +1,118 @@
+"""SOGo part of a mailbox restore (component sogo, overwrites), called by the restic runner for one mailbox.
+
+Runs `sogo-tool restore -f ALL <dir> <user>` and `sogo-tool restore -p <dir> <user>` in the sogo container through
+dockerapi on the user's file of the nightly sogo-tool backup: missing calendars, address books and entries come back,
+the names and shares of existing folders and all SOGo preferences are replaced by the backup's, entries changed since
+stay. Then the user's entries in SOGo's cache (memcached) are removed, so the change shows at once.
+"""
+
+import json
+import os
+import re
+import socket
+import ssl
+import urllib.request
+
+dockerapi_url = 'https://dockerapi:443'
+memcached_addr = ('memcached-mailcow', 11211)
+# dockerapi stops each of the two sogo-tool runs after 850 s
+restore_timeout = 1800
+# accepted by the dockerapi handler as well
+user_re = re.compile(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+')
+
+
+def _short(text, limit=300):
+  # sogo-tool may print database URLs: no credentials, one line, the end has the error
+  text = re.sub(r'([A-Za-z][A-Za-z0-9+.-]*://)[^\s/@]*@', r'\1***@', str(text))
+  text = ' '.join(text.split())
+  return text if len(text) <= limit else '...' + text[-limit:]
+
+
+def _api(path, body=None, timeout=60):
+  # dockerapi uses the mail certificate, its name does not match
+  ctx = ssl.create_default_context()
+  ctx.check_hostname = False
+  ctx.verify_mode = ssl.CERT_NONE
+  data = None if body is None else json.dumps(body).encode()
+  req = urllib.request.Request(dockerapi_url + path, data=data, headers={'Content-Type': 'application/json'},
+    method='GET' if body is None else 'POST')
+  with urllib.request.urlopen(req, timeout=timeout, context=ctx) as response:
+    return json.load(response)
+
+
+def _sogo_container():
+  project = os.environ.get('COMPOSE_PROJECT_NAME', '').lower()
+  by_name = None
+  for cid, info in _api('/containers/json').items():
+    if not cid.isalnum() or not isinstance(info, dict):
+      continue
+    labels = (info.get('Config') or {}).get('Labels') or {}
+    if labels.get('com.docker.compose.service') == 'sogo-mailcow':
+      if not project or labels.get('com.docker.compose.project', '').lower() == project:
+        return cid
+    elif by_name is None and 'sogo-mailcow' in str(info.get('Name', '')):
+      by_name = cid
+  if by_name:
+    return by_name
+  raise RuntimeError('sogo container not found')
+
+
+def _memcached_delete(keys):
+  keys = [k for k in keys if 0 < len(k.encode()) <= 250 and not re.search(r'[\s\x00-\x1f\x7f]', k)]
+  if not keys:
+    return
+  with socket.create_connection(memcached_addr, timeout=10) as s:
+    for key in keys:
+      s.sendall(b'delete ' + key.encode() + b'\r\n')
+      reply = b''
+      while not reply.endswith(b'\r\n'):
+        chunk = s.recv(64)
+        if not chunk:
+          raise RuntimeError('memcached closed the connection')
+        reply += chunk
+      if reply.strip() not in (b'DELETED', b'NOT_FOUND'):
+        raise RuntimeError('memcached: %s' % _short(reply.decode('ascii', 'replace')))
+
+
+def _cache_keys(db, username):
+  # preferences and the ACLs of the user's folders (<uid>/Calendar/personal+acl); never read the c_*location
+  # columns of sogo_folder_info (database credentials)
+  keys = ['%s+defaults' % username, '%s+settings' % username]
+  cur = db.cursor()
+  try:
+    cur.execute('SELECT c_path FROM sogo_folder_info WHERE c_path2 = %s', (username,))
+    keys += ['%s+acl' % row[0][len('/Users/'):] for row in cur.fetchall() if row[0].startswith('/Users/')]
+  finally:
+    cur.close()
+  return keys
+
+
+def restore(db, backup_dir, username, log):
+  """Restores the SOGo data of one user from <backup_dir>/<username> (staged readable for the sogo user only).
+  db: open MySQLdb connection; log: callable for short progress lines. Returns {"added": 0|1, "errors": [...]}."""
+  result = {'added': 0, 'errors': []}
+  if not user_re.fullmatch(username):
+    result['errors'].append('username not supported by the SOGo restore')
+    return result
+  path = os.path.join(backup_dir, username)
+  if not os.path.isfile(path) or os.path.islink(path):
+    # sogo-tool backup ALL skips users without SOGo folders
+    # a key the UI translates
+    result['note'] = 'no_sogo_data'
+    log('sogo: no SOGo data in the snapshot')
+    return result
+  try:
+    res = _api('/containers/%s/exec' % _sogo_container(), {'cmd': 'sogo', 'task': 'restore', 'dir': backup_dir,
+      'username': username}, timeout=restore_timeout)
+    if not isinstance(res, dict) or res.get('type') != 'success':
+      raise RuntimeError('sogo-tool restore failed: %s' % _short(res.get('msg', '') if isinstance(res, dict) else res))
+    result['added'] = 1
+    log('sogo: calendars, address books and preferences reset to the backup')
+  except Exception as e:
+    result['errors'].append(_short(e))
+  try:
+    # SOGo caches ACLs and preferences for 15 minutes
+    _memcached_delete(_cache_keys(db, username))
+  except Exception as e:
+    result['errors'].append('SOGo cache not cleared, changes may show up after 15 minutes: %s' % _short(e))
+  return result

data/Dockerfiles/rspamd/docker-entrypoint.sh

@@ -277,9 +277,34 @@ if [[ ! -z ${SPAMHAUS_DQS_KEY} ]]; then
       }
     }
     "DBL" {
-        # override the defaults for DBL defined in modules.d/rbl.conf
+        # full copy of the DBL rule from modules.d/rbl.conf, pointed at DQS:
+        # the override include replaces the stock rule instead of merging into it
         rbl = "${SPAMHAUS_DQS_KEY}.dbl.dq.spamhaus.net";
         disable_monitoring = true;
+        checks = ["emails", "dkim", "helo", "rdns", "replyto", "urls", "content_urls"];
+        no_ip = true;
+        ignore_defaults = true;
+        exclude_users = false;
+        emails_domainonly = true;
+        selector = {
+            from_smtp = "from('smtp'):domain";
+            from_mime = "from('mime'):domain";
+            mid = "header(Message-Id).regexp('@([^\.]+\.[^>]+)').last";
+        }
+        returncodes = {
+            DBL_SPAM = "127.0.1.2";
+            DBL_PHISH = "127.0.1.4";
+            DBL_MALWARE = "127.0.1.5";
+            DBL_BOTNET = "127.0.1.6";
+            DBL_ABUSE = "127.0.1.102";
+            DBL_ABUSE_REDIR = "127.0.1.103";
+            DBL_ABUSE_PHISH = "127.0.1.104";
+            DBL_ABUSE_MALWARE = "127.0.1.105";
+            DBL_ABUSE_BOTNET = "127.0.1.106";
+            DBL_PROHIBIT = "127.0.1.255";
+            DBL_BLOCKED_OPENRESOLVER = "127.255.255.254";
+            DBL_BLOCKED = "127.255.255.255";
+        }
     }
     "ZRD" {
         ignore_defaults = true;

data/Dockerfiles/sogo/Dockerfile

@@ -1,6 +1,6 @@
 # SOGo built from source to enable security patch application
 # Repository: https://github.com/Alinto/sogo
-# Version: SOGo-5.12.12
+# Version: SOGo-5.12.13
 #
 # Applied security patches:
 # -
@@ -12,8 +12,8 @@ FROM debian:bookworm
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SOGO_VERSION=SOGo-5.12.12
-ARG SOPE_VERSION=SOPE-5.12.12
+ARG SOGO_VERSION=SOGo-5.12.13
+ARG SOPE_VERSION=SOPE-5.12.13
 # Security patches to apply (space-separated commit hashes)
 ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$

data/Dockerfiles/watchdog/watchdog.sh

@@ -43,6 +43,19 @@ while ! mariadb-admin status --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u$
   sleep 2
 done
 
+# Wait for the schema update (DB_VERSION) at startup, do not restart them meanwhile.
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+done
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31m$(date) - DB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting checks anyway, check the php-fpm log (init_db)\e[0m"
+fi
+
 # Do not attempt to write to slave
 if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"

data/conf/dovecot/auth/mailcowauth.php

@@ -68,13 +68,17 @@ require_once 'functions.acl.inc.php';
 
 
 $isSOGoRequest = $post['real_rip'] == getenv('IPV4_NETWORK') . '.248';
+// real_rip is the address that connected to Dovecot (for proxied connections the proxy), client_ip the client (for logs)
+$client_ip = filter_var($post['client_ip'] ?? '', FILTER_VALIDATE_IP) ? $post['client_ip'] : $post['real_rip'];
 $result = false;
+// set when the password is right, but the account does not allow this login
+$denied = false;
 if ($isSOGoRequest) {
   // This is a SOGo Auth request. First check for SSO password.
   $sogo_sso_pass = file_get_contents("/etc/sogo-sso/sogo-sso.pass");
   if ($sogo_sso_pass === $post['password']){
     error_log('MAILCOWAUTH: SOGo SSO auth for user ' . $post['username']);
-    set_sasl_log($post['username'], $post['real_rip'], "SOGO");
+    set_sasl_log($post['username'], $client_ip, "SOGO");
     $result = true;
   }
 }
@@ -91,10 +95,10 @@ if ($result === false){
     'service' => $post['service'],
     'is_internal' => true,
     'remote_addr' => $post['real_rip']
-  ));
+  ), $denied);
   if ($result) {
-    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $post['real_rip']);
-    set_sasl_log($post['username'], $post['real_rip'], $service);
+    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+    set_sasl_log($post['username'], $client_ip, $service);
   }
 }
 if ($result === false){
@@ -102,26 +106,37 @@ if ($result === false){
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
   try {
-    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']), $denied);
   } catch (\Mailcow\Auth\IdpUnavailable $e) {
     // temporary failure for Dovecot, no failed login
-    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip . ": " . $e->getMessage());
     http_response_code(503); // Service Unavailable
     echo json_encode($return);
     session_destroy();
     exit;
   }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
-    set_sasl_log($post['username'], $post['real_rip'], $post['service']);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+    set_sasl_log($post['username'], $client_ip, $service);
   }
 }
 
 if ($result) {
   http_response_code(200); // OK
   $return['success'] = true;
+} elseif ($denied) {
+  // Logged, but no failed login for netfilter: e.g. a client of a shared mailbox or without IMAP access
+  error_log("MAILCOWAUTH: Login denied for user " . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+  http_response_code(401); // Unauthorized
 } else {
-  error_log("MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
+  // Same line for the log and netfilter, which counts it for IMAP, POP3, Sieve and SMTP (Postfix asks Dovecot);
+  // SOGo logins have their own rule
+  $failed = "MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " .
+    preg_replace('/[^a-z0-9]/', '', strtolower((string)$service)) . " from IP " . $client_ip;
+  error_log($failed);
+  if (!$isSOGoRequest) {
+    $redis->publish("F2B_CHANNEL", $failed);
+  }
   http_response_code(401); // Unauthorized
 }
 

data/conf/dovecot/auth/passwd-verify.lua

@@ -1,8 +1,4 @@
 function auth_password_verify(request, password)
-  if request.domain == nil then
-    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
-  end
-
   local json = require "cjson"
   local ltn12 = require "ltn12"
   local https = require "ssl.https"
@@ -11,7 +7,10 @@ function auth_password_verify(request, password)
   local req = {
     username = request.user,
     password = password,
+    -- address that connected to Dovecot, for proxied connections the proxy
     real_rip = request.real_rip,
+    -- address of the client, also for proxied connections
+    client_ip = request.rip,
     service = request.service
   }
   local req_json = json.encode(req)

data/conf/dovecot/dovecot.conf

@@ -51,13 +51,6 @@ recipient_delimiter = +
 auth_master_user_separator = *
 mail_shared_explicit_inbox = yes
 mail_prefetch_count = 30
-passdb {
-  driver = lua
-  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
-  result_success = return-ok
-  result_failure = continue
-  result_internalfail = continue
-}
 # try a master passwd
 passdb {
   driver = passwd-file
@@ -67,8 +60,9 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
+# each check asks mailcowauth.php, which reports a failed login to netfilter once
 # cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
-# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
+# of a passdb lookup (master logins) and reject the login as unknown user
 passdb {
   driver = lua
   args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
@@ -205,6 +199,9 @@ plugin {
   sieve_plugins = sieve_imapsieve sieve_extprograms
   sieve_vacation_send_from_recipient = yes
   sieve_redirect_envelope_from = recipient
+  # Notifications are sent with the address of the mailbox (not :from or postmaster), so mail policies apply;
+  # the old notify extension always sends as postmaster and is on by default, so sieve_extensions disables it
+  sieve_notify_mailto_envelope_from = recipient
   # From elsewhere to Spam folder
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY
@@ -221,7 +218,7 @@ plugin {
   quota_warning2 = storage=80%% quota-warning 80 %u
   sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
   sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
-  sieve_extensions = +notify +imapflags +vacation-seconds +editheader
+  sieve_extensions = -notify +imapflags +vacation-seconds +editheader
   sieve_max_script_size = 1M
   sieve_max_redirects = 100
   sieve_max_actions = 101

data/conf/nginx/templates/nginx.conf.j2

@@ -47,6 +47,13 @@ http {
         default "";
     }
 
+    # Client address for PHP: on the internal sogo-auth listener nginx itself is the client,
+    # the real address comes from the X-Real-IP header set by /sogo-auth-verify
+    map $sogo_auth_internal $mailcow_client_ip {
+        "1"     $http_x_real_ip;
+        default $remote_addr;
+    }
+
     {% if HTTP_REDIRECT %}
     # HTTP to HTTPS redirect
     server {
@@ -143,6 +150,15 @@ http {
         access_log /var/log/nginx/access.log;
         root /dynmaps;
 
+        # backup endpoint of the restic runner: large restore requests, long merges of mailboxes and domains
+        location = /backup.php {
+            client_max_body_size 32m;
+            fastcgi_read_timeout 600s;
+            fastcgi_pass {{ PHPFPMHOST }}:9001;
+            include fastcgi_params;
+            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        }
+
         location ~ \.php$ {
             try_files $uri =404;
             fastcgi_split_path_info ^(.+\.php)(/.+)$;
@@ -152,6 +168,13 @@ http {
             fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
             fastcgi_param PATH_INFO $fastcgi_path_info;
         }
+
+        location = /retention.php {
+            client_max_body_size 32m;
+            fastcgi_pass {{ PHPFPMHOST }}:9001;
+            include fastcgi_params;
+            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        }
     }
 
     # rspamd meta_exporter:

data/conf/nginx/templates/sites-default.conf.j2

@@ -22,7 +22,8 @@ add_header Referrer-Policy strict-origin;
 
 index index.php index.html;
 
-client_max_body_size 0;
+# PHP accepts no larger requests (post_max_size), SOGo webmail, CalDAV/CardDAV and ActiveSync use NGINX_MAX_BODY_SIZE
+client_max_body_size 64m;
 
 gzip on;
 gzip_disable "msie6";
@@ -93,11 +94,11 @@ location / {
 }
 
 location /qhandler {
-    rewrite ^/qhandler/(.*)/(.*) /qhandler.php?action=$1&hash=$2;
+    rewrite ^/qhandler/([^/]+)/(.+?)/?$ /qhandler.php?action=$1&hash=$2;
 }
 
 location /edit {
-    rewrite ^/edit/(.*)/(.*) /edit.php?$1=$2;
+    rewrite ^/edit/([^/]+)/(.+?)/?$ /edit.php?$1=$2;
 }
 
 location ~ ^/api/v1/(.*)$ {
@@ -116,7 +117,8 @@ location ~ \.php$ {
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
-    fastcgi_param HTTP_X_REAL_IP $remote_addr;
+    fastcgi_param REMOTE_ADDR $mailcow_client_ip;
+    fastcgi_param HTTP_X_REAL_IP $mailcow_client_ip;
     # trusted internal-auth marker; empty for external clients (see nginx.conf map)
     fastcgi_param SOGO_AUTH_INTERNAL $sogo_auth_internal;
     fastcgi_read_timeout 3600;
@@ -181,9 +183,16 @@ location /sogo-auth-verify {
     proxy_set_header  Content-Length "";
     proxy_pass        http://127.0.0.1:65510/sogo-auth;
     proxy_pass_request_body off;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
 }
 
 location ^~ /Microsoft-Server-ActiveSync {
+    # Outlook for iOS/Android (Microsoft's sync service) sends EAS commands without credentials
+    # and only authenticates after a 401 Basic challenge. SOGo answers those with 403, so challenge here.
+    if ($http_authorization = "") {
+        add_header WWW-Authenticate 'Basic realm="SOGo"' always;
+        return 401;
+    }
     auth_request /sogo-auth-verify;
     auth_request_set $user $upstream_http_x_user;
     auth_request_set $auth $upstream_http_x_auth;
@@ -204,7 +213,7 @@ location ^~ /Microsoft-Server-ActiveSync {
     proxy_busy_buffers_size 512k;
     proxy_set_header Host $http_host;
     client_body_buffer_size 512k;
-    client_max_body_size 0;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
 }
 
 location ^~ /SOGo {
@@ -255,7 +264,7 @@ location ^~ /SOGo {
     proxy_send_timeout 3600;
     proxy_read_timeout 3600;
     client_body_buffer_size 128k;
-    client_max_body_size 0;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
     break;
 }
 

data/conf/phpfpm/crons/identity-sync.php

@@ -60,6 +60,7 @@ $_SESSION['acl']['sogo_access'] = "1";
 $_SESSION['acl']['protocol_access'] = "1";
 $_SESSION['acl']['mailbox_relayhost'] = "1";
 $_SESSION['acl']['unlimited_quota'] = "1";
+$_SESSION['acl']['archive'] = "1";
 
 // One run at a time, released when the process ends
 $lock = fopen('/tmp/identity-sync.lock', 'c');

data/conf/postfix/local_transport

@@ -1,2 +1,3 @@
 /watchdog@localhost$/   watchdog_discard:
+/^archive-bounce@localhost$/  archive_discard:
 /localhost$/  local:

data/conf/postfix/main.cf

@@ -50,6 +50,10 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   $sender_dependent_default_transport_maps,
@@ -90,6 +94,7 @@ smtpd_hard_error_limit = ${stress?1}${stress:5}
 smtpd_helo_required = yes
 smtpd_proxy_timeout = 600s
 smtpd_recipient_restrictions = check_recipient_mx_access proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
+  check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   permit_sasl_authenticated,
   permit_mynetworks,
   check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
@@ -170,5 +175,11 @@ parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks
 # This Option is added to correctly set the X-Original-To Header when mails are send to lmtp (dovecot)
 lmtp_destination_recipient_limit=1
 
+# Mail policies: while policies or internal aliases exist, mysql_mail_policy_maps.cf returns this class and smtpd
+# asks the policy service for every recipient (mailcow-policy in master.cf); no answer defers the recipient
+smtpd_restriction_classes = mailcow_mail_policy
+mailcow_mail_policy = check_policy_service { unix:private/mailcow-policy, timeout=10s, try_limit=1, default_action=DEFER_IF_PERMIT }
+mailcow-policy_time_limit = 3600s
+
 # DO NOT EDIT ANYTHING BELOW #
 # Overrides #

data/conf/postfix/master.cf

@@ -61,13 +61,15 @@ submission inet n       -       n       -       -       smtpd
   -o non_smtpd_milters=
   -o syslog_name=postfix/quarantine
 
-# used to send bcc mails
+# used to send bcc mails and archive copies (sender archive-bounce@localhost goes to the archive)
+# the archive FILTER comes after the one of MX-based transports, Postfix uses the last FILTER
 591 inet n      -       n       -       -       smtpd
   -o smtpd_helo_restrictions=
   -o smtpd_client_restrictions=permit_mynetworks,reject
   -o smtpd_tls_auth_only=no
   -o smtpd_milters=
   -o non_smtpd_milters=
+  -o smtpd_recipient_restrictions=check_recipient_mx_access,proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,check_sender_access,proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,permit_mynetworks,reject
   -o syslog_name=postfix/bcc
 
 # enforced smtp connector
@@ -81,6 +83,27 @@ smtp_enforced_tls      unix  -       -       n       -       -       smtp
 smtp_via_transport_maps      unix  -       -       n       -       -       smtp
   -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf
 
+# archive copies (journal) to the archive set in the admin UI: only mails from archive-bounce@localhost on 591 get here
+# (FILTER of mysql_archive_sender_filter_maps), sent unchanged, TLS by default, refusals are retried (soft_bounce)
+# until maximal_queue_lifetime
+archive_smtp      unix  -       -       n       -       -       smtp
+  -o soft_bounce=yes
+  -o smtp_header_checks=
+  -o smtp_mime_header_checks=
+  -o smtp_nested_header_checks=
+  -o smtp_body_checks=
+  -o smtp_line_length_limit=0
+  -o disable_mime_output_conversion=yes
+  -o smtp_tls_security_level=encrypt
+  -o smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
+  -o smtp_sasl_auth_enable=yes
+  -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
+  -o smtp_sasl_tls_security_options=noanonymous
+  -o syslog_name=postfix/archive
+# notices about archive copies (delays, bounces) go to archive-bounce@localhost and are dropped here
+archive_discard   unix  -       -       n       -       -       discard
+  -o syslog_name=postfix/archive
+
 tlsproxy   unix  -       -       n       -       0       tlsproxy
 dnsblog    unix  -       -       n       -       0       dnsblog
 pickup     fifo  n       -       n       60      1       pickup
@@ -117,10 +140,14 @@ smtp_sender_cleanup unix n - y - 0 cleanup
 127.0.0.1:10027 inet n n n - 0 spawn user=nobody argv=/usr/local/bin/whitelist_forwardinghosts.sh
 # end whitelist_fwd
 
+# mail policies: policy service of smtpd, asks the mailcow UI (mailcow_mail_policy in main.cf)
+mailcow-policy unix - n n - 0 spawn user=nobody argv=/usr/local/bin/mailcow-policy.py
+
 # start watchdog-specific
-# logs to local7 (hidden)
+# logs to local7 (hidden), no mail policy check
 589 inet n      -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,reject
+  -o smtpd_recipient_restrictions=permit_mynetworks,reject
   -o syslog_name=watchdog
   -o syslog_facility=local7
   -o smtpd_milters=

data/conf/postfix/postscreen_access.cidr

@@ -1,6 +1,6 @@
-# Whitelist generated by Postwhite v3.4 on Thu Oct  1 00:35:20 UTC 2026
+# Whitelist generated by Postwhite v3.4 on Sun Nov  1 00:23:09 UTC 2026
 # https://github.com/stevejenkins/postwhite/
-# 2246 total rules
+# 2237 total rules
 2a00:1450:4000::/36	permit
 2a00:1450:4864::/56	permit
 2a01:111:f400::/48	permit
@@ -197,6 +197,9 @@
 44.246.68.102	permit
 44.246.77.92	permit
 45.14.148.0/22	permit
+45.128.172.192/27	permit
+45.128.172.224/28	permit
+45.128.174.0/25	permit
 45.143.132.0/24	permit
 45.143.133.0/24	permit
 45.143.134.0/24	permit
@@ -672,7 +675,6 @@
 85.215.255.47	permit
 85.215.255.48	permit
 85.215.255.49	permit
-86.61.88.25	permit
 87.238.80.0/21	permit
 87.248.103.12	permit
 87.248.103.21	permit
@@ -1345,11 +1347,11 @@
 107.20.210.250	permit
 107.22.191.150	permit
 108.174.0.0/24	permit
-108.174.0.215	permit
+108.174.0.93	permit
 108.174.3.0/24	permit
-108.174.3.215	permit
+108.174.3.93	permit
 108.174.6.0/24	permit
-108.174.6.215	permit
+108.174.6.93	permit
 108.175.18.45	permit
 108.175.30.45	permit
 108.179.144.0/20	permit
@@ -1532,11 +1534,6 @@
 147.154.63.0/24	permit
 147.154.126.0/24	permit
 147.154.191.0/24	permit
-147.243.1.47	permit
-147.243.1.48	permit
-147.243.1.153	permit
-147.243.128.24	permit
-147.243.128.26	permit
 148.105.0.0/16	permit
 148.105.8.0/21	permit
 148.116.32.128/25	permit
@@ -1546,7 +1543,7 @@
 149.97.173.180	permit
 149.118.160.128/25	permit
 150.136.21.199	permit
-150.171.109.178	permit
+150.171.109.66	permit
 150.230.98.160	permit
 151.145.38.14	permit
 152.67.105.195	permit
@@ -1562,7 +1559,6 @@
 155.248.237.141	permit
 157.56.120.128/26	permit
 157.58.30.128/25	permit
-157.58.196.96/29	permit
 157.58.249.3	permit
 157.137.32.128/25	permit
 157.137.96.128/25	permit
@@ -1651,7 +1647,6 @@
 167.89.75.126	permit
 167.89.75.136	permit
 167.89.75.164	permit
-167.220.67.232/29	permit
 168.107.248.128/25	permit
 168.110.160.128/25	permit
 168.110.248.128/25	permit
@@ -1979,8 +1974,6 @@
 207.46.22.35	permit
 207.46.50.72	permit
 207.46.50.82	permit
-207.46.52.71	permit
-207.46.52.79	permit
 207.67.38.0/24	permit
 207.67.98.192/27	permit
 207.97.204.96/29	permit
@@ -2138,8 +2131,6 @@
 212.227.126.227	permit
 213.199.128.139	permit
 213.199.128.145	permit
-213.199.138.181	permit
-213.199.138.191	permit
 216.17.150.242	permit
 216.17.150.251	permit
 216.24.224.0/20	permit
@@ -2228,23 +2219,23 @@
 2001:748:400:3301::4	permit
 2404:6800:4000::/36	permit
 2404:6800:4864::/56	permit
-2603:1061:14:1d0::1	permit
+2603:1061:14:41::1	permit
 2607:13c0:0001:0000:0000:0000:0000:7000/116	permit
 2607:13c0:0002:0000:0000:0000:0000:1000/116	permit
 2607:13c0:0004:0000:0000:0000:0000:0000/116	permit
 2607:f8b0:4000::/36	permit
 2607:f8b0:4864::/56	permit
 2620:109:c003:104::/64	permit
-2620:109:c003:104::215	permit
+2620:109:c003:104::5d	permit
 2620:109:c006:104::/64	permit
-2620:109:c006:104::215	permit
+2620:109:c006:104::5d	permit
 2620:109:c00d:104::/64	permit
 2620:10d:c090:400::8:1	permit
 2620:10d:c091:400::8:1	permit
 2620:10d:c09b:400::8:1	permit
 2620:10d:c09c:400::8:1	permit
 2620:119:50c0:207::/64	permit
-2620:119:50c0:207::215	permit
+2620:119:50c0:207::5d	permit
 2800:3f0:4000::/36	permit
 2800:3f0:4864::/56	permit
 49.12.4.251 permit # checks.mailcow.email

data/conf/rspamd/dynmaps/archive.php

@@ -0,0 +1,47 @@
+<?php
+// Archive decision for an accepted mail, asked by the MAILCOW_ARCHIVE symbol of rspamd
+// Request: JSON {from, rcpt[], user, bcc, qid, received, sha256}; answer: JSON {archive, rcpt, sender, headers}
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("ARCHIVE: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+// Init Redis, it keeps the global switch; without it nothing is archived
+$redis = new Redis();
+try {
+  $redis->connect('redis-mailcow', 6379);
+  $redis->auth(getenv("REDISPASS"));
+}
+catch (Exception $e) {
+  $redis = null;
+}
+
+$request = json_decode(file_get_contents('php://input'), true);
+if (!is_array($request)) {
+  http_response_code(400);
+  exit;
+}
+
+try {
+  $archive = \Mailcow\Archive\ArchiveModule::create($pdo, \Mailcow\Core\Context::system(), $redis);
+  echo json_encode($archive->journal->prepare($request));
+}
+catch (Throwable $e) {
+  error_log("ARCHIVE: " . $e->getMessage() . PHP_EOL);
+  http_response_code(502);
+}

data/conf/rspamd/dynmaps/backup.php

@@ -0,0 +1,97 @@
+<?php
+// Backup with restic: endpoint of the runner in the restic container (Mailcow\Backup)
+// GET ?domains=1: {"domains": [...]}, all domains
+// GET ?export=<domain>[&after=<username>]: a page of the export of a domain (the first with domain_data and aliases,
+//   every page with a part of the mailboxes and "next_after", the last username to ask for the next page with, null
+//   after the last), incl. password hashes, DKIM keys and secrets; the runner merges the pages into
+//   /mailcow/export/<domain>.json of the snapshot.
+// POST {"action": "restore_domain", "job": <id>, "data": <domain_data>}: recreates a deleted domain
+// POST {"action": "restore_aliases", "job": <id>, "aliases": [...]}: a part (at most 500) of the aliases of a domain restore
+// POST {"action": "restore", "job": <id>, "username": <mailbox>, "data": <its section of the export>}: one mailbox
+//   (created first when it was deleted); all answer {"result": {<component>: {"added", "skipped", "errors"}}}
+// All need the header X-Backup-Token: HMAC-SHA256 of "backup" with the database password as key
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'backup', $database_pass), (string)($_SERVER['HTTP_X_BACKUP_TOKEN'] ?? ''))) {
+  http_response_code(403);
+  exit;
+}
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("BACKUP: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+try {
+  // DKIM keys, rate limits and the domain map are kept in Redis
+  $redis = new Redis();
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  }
+  else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+  $config = array(
+    'timezone' => getenv('TZ') ?: 'UTC',
+    'alias_disable_external_domains' => ($ALIAS_DISABLE_EXTERNAL_DOMAINS ?? false) === true,
+  );
+  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
+    $body = json_decode((string)file_get_contents('php://input'), true);
+    if (!is_array($body)) {
+      http_response_code(400);
+      exit;
+    }
+    // The restore has the side effects of the UI: SOGo view and restart, Dovecot auth cache, group shares (legacy code)
+    require_once __DIR__ . '/../web/inc/functions.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+    // The legacy mailbox functions run as admin, without a session
+    $_SESSION = array(
+      'mailcow_cc_username' => 'admin',
+      'mailcow_cc_role' => 'admin',
+      'access_all_exception' => '0',
+      'acl' => array_fill_keys(array('tls_policy', 'quarantine_notification', 'quarantine_category', 'ratelimit', 'sogo_access',
+        'protocol_access', 'mailbox_relayhost', 'unlimited_quota'), '1'),
+    );
+    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
+    echo json_encode($backup->api->post($body), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+  }
+  else {
+    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
+    if (isset($_GET['export'])) {
+      echo json_encode($backup->api->export((string)$_GET['export'], (isset($_GET['after']) && is_string($_GET['after'])) ? $_GET['after'] : null),
+        JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
+    }
+    elseif (isset($_GET['domains'])) {
+      echo json_encode($backup->api->domains());
+    }
+    else {
+      http_response_code(400);
+    }
+  }
+}
+catch (\Mailcow\Core\UserError $e) {
+  http_response_code(($e->key === 'domain_not_found') ? 404 : 400);
+  echo json_encode(array('error' => $e->key, 'args' => $e->args));
+}
+catch (Throwable $e) {
+  error_log("BACKUP: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/mailpolicy.php

@@ -0,0 +1,39 @@
+<?php
+// Mail policy check for one recipient, asked by the Postfix policy service (mailcow-policy in master.cf)
+// Request: POST with the attributes of the Postfix policy request; answer: the action (DUNNO or the rejection)
+header('Content-Type: text/plain');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("MAILPOLICY: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+if (!isset($_POST['request']) || $_POST['request'] !== 'smtpd_access_policy') {
+  http_response_code(400);
+  exit;
+}
+$value = fn(string $key): string => is_string($_POST[$key] ?? null) ? $_POST[$key] : '';
+
+try {
+  $policy = \Mailcow\MailPolicy\MailPolicyModule::create($pdo, \Mailcow\Core\Context::system());
+  echo $policy->api->check($value('sender'), $value('recipient'), $value('sasl_username'), (int)$value('server_port'), $value('helo_name'));
+}
+catch (Throwable $e) {
+  // the policy service answers with a temporary error
+  error_log("MAILPOLICY: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/retention.php

@@ -0,0 +1,52 @@
+<?php
+// Retention rules for the nightly run in the dovecot container (retention_runner.py)
+// GET ?after=<username>&limit=<n>: a page of the plan, per mailbox the folders to clean up (JSON)
+// POST (JSON {"results": [...], "summary": [...], "run": {...}}): the results
+// Both need the header X-Retention-Token: HMAC-SHA256 of "retention" with the database password as key
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'retention', $database_pass), (string)($_SERVER['HTTP_X_RETENTION_TOKEN'] ?? ''))) {
+  http_response_code(403);
+  exit;
+}
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("RETENTION: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+try {
+  $retention = \Mailcow\Retention\RetentionModule::create($pdo, \Mailcow\Core\Context::system());
+  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
+    $body = json_decode((string)file_get_contents('php://input'), true);
+    if (!is_array($body) || !is_array($body['results'] ?? null)) {
+      http_response_code(400);
+      exit;
+    }
+    echo json_encode(['stored' => $retention->api->report($body['results'], is_array($body['summary'] ?? null) ? $body['summary'] : [],
+      is_array($body['run'] ?? null) ? $body['run'] : null)]);
+  }
+  else {
+    echo json_encode($retention->api->plan((string)($_GET['after'] ?? ''), (int)($_GET['limit'] ?? 0)));
+  }
+}
+catch (\Mailcow\Core\UserError $e) {
+  http_response_code(400);
+}
+catch (Throwable $e) {
+  error_log("RETENTION: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/settings.php

@@ -469,35 +469,5 @@ while ($row = array_shift($rows)) {
 }
 ?>
 
-<?php
-// Start internal aliases
-
-$stmt = $pdo->query("SELECT `id`, `address`, `domain` FROM `alias` WHERE `active` = '1' AND `internal` = '1'");
-$aliases = $stmt->fetchAll(PDO::FETCH_ASSOC);
-while ($alias = array_shift($aliases)) {
-  // build allowed_domains regex and add target domain and alias domains
-  $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `active` = '1' AND `target_domain` = :target_domain");
-  $stmt->execute(array(':target_domain' => $alias['domain']));
-  $allowed_domains = $stmt->fetchAll(PDO::FETCH_ASSOC);
-  $allowed_domains = array_map(function($item) {
-    return str_replace('.', '\.', $item['alias_domain']);
-  }, $allowed_domains);
-  $allowed_domains[] = str_replace('.', '\.', $alias['domain']);
-  $allowed_domains = implode('|', $allowed_domains);
-?>
-  internal_alias_<?=$alias['id'];?> {
-    priority = 10;
-    rcpt = "<?=$alias['address'];?>";
-    from = "/^((?!.*@(<?=$allowed_domains;?>)).)*$/";
-    apply "default" {
-      MAILCOW_INTERNAL_ALIAS = 9999.0;
-    }
-    symbols [
-      "MAILCOW_INTERNAL_ALIAS"
-    ]
-  }
-<?php
-}
-?>
 
 }

data/conf/rspamd/local.d/composites.conf

@@ -1,7 +1,3 @@
-MX_IMPLICIT {
-  expression = "MX_GOOD & MX_MISSING";
-  score = -0.01;
-}
 VIRUS_FOUND {
   expression = "CLAM_VIRUS & !MAILCOW_WHITE";
   score = 2000.0;

data/conf/rspamd/local.d/groups.conf

@@ -23,24 +23,6 @@ symbols {
   }
 }
 
-group "MX" {
-  "MX_INVALID" {
-    score = 0.5;
-    description = "No connectable MX";
-    one_shot = true;
-  }
-  "MX_MISSING" {
-    score = 2.0;
-    description = "No MX record";
-    one_shot = true;
-  }
-  "MX_GOOD" {
-    score = -0.01;
-    description = "MX was ok";
-    one_shot = true;
-  }
-}
-
 group "reputation" {
   symbols = {
     "IP_REPUTATION_HAM" {

data/conf/rspamd/local.d/multimap.conf

@@ -5,12 +5,6 @@ RCPT_MAILCOW_DOMAIN {
   symbols_set = ["RCPT_MAILCOW_DOMAIN"];
 }
 
-WHITELISTED_FWD_HOST {
-  type = "ip";
-  map = "redis://WHITELISTED_FWD_HOST";
-  symbols_set = ["WHITELISTED_FWD_HOST"];
-}
-
 BULK_HEADER {
   type = "content";
   map = "${LOCAL_CONFDIR}/custom/bulk_header.map";
@@ -151,20 +145,6 @@ BAD_LANG {
   score = 5.0;
 }
 
-BAZAAR_ABUSE_CH {
-  type = "selector";
-  selector = "attachments(hex,md5)";
-  map = "https://bazaar.abuse.ch/export/txt/md5/recent/";
-  score = 10.0;
-}
-
-URLHAUS_ABUSE_CH {
-  type = "selector";
-  selector = "urls";
-  map = "https://urlhaus.abuse.ch/downloads/text_online/";
-  score = 10.0;
-}
-
 SMTP_LIMITED_ACCESS {
   type = "user";
   map = "redis://SMTP_LIMITED_ACCESS";

data/conf/rspamd/local.d/mx_check.conf

@@ -1,6 +1,5 @@
 timeout = 8.0;
 symbol_bad_mx = "MX_INVALID";
-symbol_no_mx = "MX_MISSING";
 symbol_good_mx = "MX_GOOD";
 expire = 86400;
 key_prefix = "rmx";

data/conf/rspamd/local.d/options.inc

@@ -9,3 +9,5 @@ disable_monitoring = true;
 # instead of silently accepting the message without further processing.
 soft_reject_on_timeout = true;
 local_addrs = /etc/rspamd/custom/mailcow_networks.map;
+# Must not be lower than message_size_limit in postfix main.cf
+max_message = 134217728;

data/conf/rspamd/lua/rspamd.local.lua

@@ -150,41 +150,41 @@ rspamd_config:register_symbol({
     local function ipv6_to_segments(ip_str)
       -- Remove zone identifier if present (e.g., %eth0)
       ip_str = ip_str:gsub("%%.*$", "")
-      
+
       local segments = {}
-      
+
       -- Handle :: compression
       if ip_str:find('::') then
         local before, after = ip_str:match('^(.*)::(.*)$')
         before = before or ''
         after = after or ''
-        
+
         local before_parts = {}
         local after_parts = {}
-        
+
         if before ~= '' then
           for seg in before:gmatch('[^:]+') do
             table.insert(before_parts, tonumber(seg, 16) or 0)
           end
         end
-        
+
         if after ~= '' then
           for seg in after:gmatch('[^:]+') do
             table.insert(after_parts, tonumber(seg, 16) or 0)
           end
         end
-        
+
         -- Add before segments
         for _, seg in ipairs(before_parts) do
           table.insert(segments, seg)
         end
-        
+
         -- Add compressed zeros
         local zeros_needed = 8 - #before_parts - #after_parts
         for i = 1, zeros_needed do
           table.insert(segments, 0)
         end
-        
+
         -- Add after segments
         for _, seg in ipairs(after_parts) do
           table.insert(segments, seg)
@@ -195,12 +195,12 @@ rspamd_config:register_symbol({
           table.insert(segments, tonumber(seg, 16) or 0)
         end
       end
-      
+
       -- Ensure we have exactly 8 segments
       while #segments < 8 do
         table.insert(segments, 0)
       end
-      
+
       return segments
     end
 
@@ -208,46 +208,46 @@ rspamd_config:register_symbol({
     local function get_ipv6_variants(ip_str)
       local variants = {}
       local seen = {}
-      
+
       local function add_variant(v)
         if v and not seen[v] then
           table.insert(variants, v)
           seen[v] = true
         end
       end
-      
+
       -- For IPv4, just return the original
       if not ip_str:find(':') then
         add_variant(ip_str)
         return variants
       end
-      
+
       local segments = ipv6_to_segments(ip_str)
-      
+
       -- 1. Fully expanded form (all zeros shown as 0000)
       local expanded_parts = {}
       for _, seg in ipairs(segments) do
         table.insert(expanded_parts, string.format('%04x', seg))
       end
       add_variant(table.concat(expanded_parts, ':'))
-      
+
       -- 2. Standard form (no leading zeros, but all segments present)
       local standard_parts = {}
       for _, seg in ipairs(segments) do
         table.insert(standard_parts, string.format('%x', seg))
       end
       add_variant(table.concat(standard_parts, ':'))
-      
+
       -- 3. Find all possible :: compressions
       -- RFC 5952: compress the longest run of consecutive zeros
       -- But we need to check all possibilities since Redis might have any form
-      
+
       -- Find all zero runs
       local zero_runs = {}
       local in_run = false
       local run_start = 0
       local run_length = 0
-      
+
       for i = 1, 8 do
         if segments[i] == 0 then
           if not in_run then
@@ -266,21 +266,21 @@ rspamd_config:register_symbol({
           end
         end
       end
-      
+
       -- Don't forget the last run
       if in_run and run_length >= 1 then
         table.insert(zero_runs, {start = run_start, length = run_length})
       end
-      
+
       -- Generate variant for each zero run compression
       for _, run in ipairs(zero_runs) do
         local parts = {}
-        
+
         -- Before compression
         for i = 1, run.start - 1 do
           table.insert(parts, string.format('%x', segments[i]))
         end
-        
+
         -- The compression
         if run.start == 1 then
           table.insert(parts, '')
@@ -291,22 +291,22 @@ rspamd_config:register_symbol({
         else
           table.insert(parts, '')
         end
-        
+
         -- After compression
         for i = run.start + run.length, 8 do
           table.insert(parts, string.format('%x', segments[i]))
         end
-        
+
         local compressed = table.concat(parts, ':'):gsub('::+', '::')
         add_variant(compressed)
       end
-      
+
       return variants
     end
 
     local from_ip_string = tostring(ip)
     local ip_check_table = {}
-    
+
     -- Add all variants of the exact IP
     for _, variant in ipairs(get_ipv6_variants(from_ip_string)) do
       table.insert(ip_check_table, variant)
@@ -318,18 +318,18 @@ rspamd_config:register_symbol({
         maxbits = 32
         minbits = 8
     end
-    
+
     -- Add all CIDR notations with variants
     for i=maxbits,minbits,-1 do
       local masked_ip = ip:apply_mask(i)
       local cidr_base = masked_ip:to_string()
-      
+
       for _, variant in ipairs(get_ipv6_variants(cidr_base)) do
         local cidr = variant .. "/" .. i
         table.insert(ip_check_table, cidr)
       end
     end
-    
+
     local function keep_spam_cb(err, data)
       if err then
         rspamd_logger.infox(rspamd_config, "keep_spam query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
@@ -345,7 +345,7 @@ rspamd_config:register_symbol({
         end
       end
     end
-    
+
     table.insert(ip_check_table, 1, 'KEEP_SPAM')
     local redis_ret_user = rspamd_redis_make_request(task,
       redis_params, -- connect params
@@ -362,6 +362,61 @@ rspamd_config:register_symbol({
   priority = 19
 })
 
+rspamd_config:register_symbol({
+  name = 'WHITELISTED_FWD_HOST',
+  type = 'prefilter',
+  score = 0.0,
+  callback = function(task)
+    local rspamd_logger = require "rspamd_logger"
+    local redis_params = rspamd_parse_redis_server('fwd_host')
+    local ip = task:get_from_ip()
+
+    if ip == nil or not ip:is_valid() then
+      return false
+    end
+
+    local from_ip_string = tostring(ip)
+    local ip_check_table = {'WHITELISTED_FWD_HOST', from_ip_string}
+
+    local maxbits = 128
+    local minbits = 7
+    if ip:get_version() == 4 then
+        maxbits = 32
+        minbits = 8
+    end
+    for i=maxbits,minbits,-1 do
+      table.insert(ip_check_table, ip:apply_mask(i):to_string() .. "/" .. i)
+    end
+
+    local function fwd_host_cb(err, data)
+      if err or type(data) ~= 'table' then
+        rspamd_logger.infox(rspamd_config, "fwd_host query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
+        return
+      end
+      for _,v in pairs(data) do
+        if type(v) == 'string' then
+          task:insert_result('WHITELISTED_FWD_HOST', 1.0, from_ip_string)
+          task:set_flag('pass_all')
+          return
+        end
+      end
+    end
+
+    local redis_ret = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'WHITELISTED_FWD_HOST', -- hash key
+      false, -- is write
+      fwd_host_cb, --callback
+      'HMGET', -- command
+      ip_check_table -- arguments
+    )
+    if not redis_ret then
+      rspamd_logger.infox(rspamd_config, "cannot check fwd_host redis map")
+    end
+  end,
+  priority = 18
+})
+
 rspamd_config:register_symbol({
   name = 'TLS_HEADER',
   type = 'postfilter',
@@ -390,9 +445,8 @@ rspamd_config:register_symbol({
     local rspamd_logger = require "rspamd_logger"
     local redis_params = rspamd_parse_redis_server('taghandler')
     local rspamd_http = require "rspamd_http"
-    local rcpts = task:get_recipients('smtp')
+    local rcpts = task:get_recipients({'smtp', 'orig'})
     local lua_util = require "lua_util"
-    local tagged_rcpt = task:get_symbol("TAGGED_RCPT")
 
     local function remove_moo_tag()
       local moo_tag_header = task:get_header('X-Moo-Tag', false)
@@ -417,8 +471,9 @@ rspamd_config:register_symbol({
 
     -- Check if recipient has a tag (contains '+')
     local tag = nil
-    if tagged_rcpt ~= nil then
-      tag = tagged_rcpt
+    local base_user, tag_part = rcpt_user:match('^([^%+]+)%+(.+)$')
+    if base_user and tag_part then
+      tag = tag_part
       rspamd_logger.infox("TAG_MOO: found tag in recipient: %s (base: %s, tag: %s)", rcpt_addr, base_user, tag)
     end
 
@@ -441,7 +496,7 @@ rspamd_config:register_symbol({
 
     -- Check if we have a pre-result (e.g., from KEEP_SPAM or POSTMASTER_HANDLER)
     local allow_processing = false
-    
+
     if task.has_pre_result then
       local has_pre, pre_action = task:has_pre_result()
       if has_pre then
@@ -498,8 +553,7 @@ rspamd_config:register_symbol({
           else
             rspamd_logger.infox("TAG_MOO: user wants subject modified for tagged mail")
             local sbj = task:get_header('Subject') or ''
-            local tag_value = tag[1] and tag[1].options and tag[1].options[1] or ''
-            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag_value .. '] ' .. sbj)) .. '?='
+            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag .. '] ' .. sbj)) .. '?='
             task:set_milter_reply({
               remove_headers = {
                 ['Subject'] = 1,
@@ -765,13 +819,22 @@ rspamd_config:register_symbol({
   end
 })
 
+-- Expand a domain footer: replace only "{= key =}" placeholders with their values.
+-- Unknown placeholders render empty, matching the previous engine behaviour.
+local function expand_footer(template, replacements)
+  return (template:gsub('{=%s*([%w_]+)%s*=}', function(key)
+    local value = replacements[key]
+    if value == nil then return '' end
+    return tostring(value)
+  end))
+end
+
 rspamd_config:register_symbol({
   name = 'MOO_FOOTER',
   type = 'prefilter',
   callback = function(task)
     local cjson = require "cjson"
     local lua_mime = require "lua_mime"
-    local lua_util = require "lua_util"
     local rspamd_logger = require "rspamd_logger"
     local rspamd_http = require "rspamd_http"
     local envfrom = task:get_from(1)
@@ -844,10 +907,10 @@ rspamd_config:register_symbol({
               end
             end
             if footer.html and footer.html ~= "" then
-              footer.html = lua_util.jinja_template(footer.html, replacements, true)
+              footer.html = expand_footer(footer.html, replacements)
             end
             if footer.plain and footer.plain ~= "" then
-              footer.plain = lua_util.jinja_template(footer.plain, replacements, true)
+              footer.plain = expand_footer(footer.plain, replacements)
             end
 
             -- add footer
@@ -898,6 +961,9 @@ rspamd_config:register_symbol({
 
             if not seen_cte and rewrite.need_rewrite_ct then
               out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', new_cte)
+              task:set_milter_reply({
+                add_headers = {['Content-Transfer-Encoding'] = new_cte}
+              })
             end
 
             -- End of headers
@@ -947,3 +1013,134 @@ rspamd_config:register_symbol({
   end,
   priority = 1
 })
+
+-- Archive: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through Postfix (port 591)
+rspamd_config:register_symbol({
+  name = 'MAILCOW_ARCHIVE',
+  type = 'idempotent',
+  flags = 'empty,explicit_disable,ignore_passthrough',
+  callback = function(task)
+    local rspamd_http = require "rspamd_http"
+    local rspamd_logger = require "rspamd_logger"
+    local rspamd_text = require "rspamd_text"
+    local rspamd_util = require "rspamd_util"
+    local hash = require "rspamd_cryptobox_hash"
+    local lua_smtp = require "lua_smtp"
+    local cjson = require "cjson"
+
+    -- accepted mails only: no reject, soft reject (greylisting included), discard or quarantine
+    local action = task:get_metric_action('default')
+    local accepted = (action == 'no action' or action == 'add header' or action == 'rewrite subject' or action == 'greylist')
+    local pre_accept = false
+    if task.has_pre_result then
+      local has_pre, pre_action = task:has_pre_result()
+      pre_accept = (has_pre and pre_action == 'accept')
+    end
+    if not accepted and not pre_accept then
+      return
+    end
+
+    local rcpts = {}
+    for _, r in ipairs(task:get_recipients('smtp') or {}) do
+      if r.addr and r.addr ~= '' then
+        table.insert(rcpts, r.addr)
+      end
+    end
+    if #rcpts == 0 then
+      return
+    end
+    local from = task:get_from('smtp')
+    local from_addr = (from and from[1] and from[1].addr) or ''
+    local qid = task:get_queue_id() or ''
+    -- the BCC symbol sends copies under the same condition, they bypass rspamd
+    local bcc = (pre_accept or (action ~= 'greylist' and accepted)) and not task:has_symbol('ENCRYPTED_CHAT')
+    local redis_params = rspamd_parse_redis_server('archive')
+
+    -- not archived: counted and listed for the archive status in the UI
+    -- the Message-ID finds the mail in the archive log of the UI
+    local message_id = task:get_message_id()
+    if not message_id or message_id == 'undef' then
+      message_id = ''
+    end
+    local function dropped(reason)
+      rspamd_logger.errx(task, "archive: mail %s is not archived: %s", qid, reason)
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED', true, function() end, 'INCR', {'ARCHIVE_DROPPED'})
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LPUSH',
+        {'ARCHIVE_DROPPED_LOG', cjson.encode({time = os.time(), qid = qid, reason = tostring(reason), message_id = message_id})})
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LTRIM', {'ARCHIVE_DROPPED_LOG', '0', '99'})
+    end
+
+    local function archive()
+      -- the archive gets the mail as accepted, with CRLF line ends and a final CRLF (also what the hash covers)
+      local content = rspamd_text.fromstring(tostring(task:get_content()))
+      content:normalize_newlines('smtp')
+      if #content < 2 or content:at(#content) ~= 10 or content:at(#content - 1) ~= 13 then
+        content = rspamd_text.fromtable({content, '\r\n'})
+      end
+      local body = {
+        from = from_addr,
+        rcpt = rcpts,
+        user = task:get_user() or '',
+        bcc = bcc,
+        qid = qid,
+        received = rspamd_util.get_time(),
+        sha256 = hash.create_specific('sha256', content):hex()
+      }
+
+      local function sent_cb(ok, err)
+        if not ok then
+          dropped(err)
+        end
+      end
+
+      local function journal_cb(err, code, res_body)
+        if err or code ~= 200 then
+          dropped(string.format('owner lookup failed (%s, %s)', err, code))
+          return
+        end
+        local ok, res = pcall(cjson.decode, tostring(res_body))
+        if not ok or type(res) ~= 'table' then
+          dropped('invalid answer of the owner lookup')
+          return
+        end
+        if res.archive ~= true then
+          return
+        end
+        -- lua_smtp ends the data with CRLF.CRLF, so the last CRLF of the content is left to it
+        lua_smtp.sendmail({
+          task = task,
+          host = os.getenv("IPV4_NETWORK") .. '.253',
+          port = 591,
+          from = res.sender,
+          recipients = res.rcpt,
+          helo = 'archive',
+          timeout = 20,
+        }, {res.headers or '', content:span(1, #content - 2)}, sent_cb)
+      end
+
+      rspamd_http.request({
+        task = task,
+        url = 'http://nginx:8081/archive.php',
+        body = cjson.encode(body),
+        timeout = (#rcpts > 100) and 10.0 or 2.0,
+        callback = journal_cb,
+        headers = {['Content-Type'] = 'application/json'},
+      })
+    end
+
+    local function enabled_cb(err, data)
+      if not err and type(data) == 'string' and data == '1' then
+        archive()
+      end
+    end
+
+    rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'ARCHIVE_ENABLED', -- hash key
+      false, -- is write
+      enabled_cb, --callback
+      'GET', -- command
+      {'ARCHIVE_ENABLED'} -- arguments
+    )
+  end
+})

data/conf/rspamd/meta_exporter/pipe.php

@@ -109,7 +109,7 @@ foreach ($rcpts as $rcpt) {
   }
 
   // Skip if domain is excluded
-  if (in_array($parsed_rcpt['domain'], $exclude_domains)) {
+  if (is_array($exclude_domains) && in_array($parsed_rcpt['domain'], $exclude_domains)) {
     error_log(sprintf("QUARANTINE: Skipped domain %s", $parsed_rcpt['domain']) . PHP_EOL);
     continue;
   }
@@ -216,8 +216,9 @@ foreach ($rcpts as $rcpt) {
 foreach ($rcpt_final_mailboxes as $rcpt_final) {
   error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt_final . PHP_EOL);
   try {
-    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`)
-      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes)");
+    // qhash is the secret of the release/delete links in notifications, so it must not be derivable
+    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`, `qhash`)
+      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes, :qhash)");
     $stmt->execute(array(
       ':qid' => $qid,
       ':subject' => $subject,
@@ -229,11 +230,9 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
       ':ip' => $ip,
       ':msg' => $raw_data,
       ':action' => $action,
-      ':fuzzy_hashes' => $fuzzy
+      ':fuzzy_hashes' => $fuzzy,
+      ':qhash' => bin2hex(random_bytes(32))
     ));
-    $lastId = $pdo->lastInsertId();
-    $stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
-    $stmt_update->execute(array(':id' => $lastId));
     $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
       SELECT `id`
       FROM (

data/web/admin/mailbox.php

@@ -11,6 +11,8 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
+$js_minifier->add('/web/js/site/mail-policy.js');
+$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -46,6 +48,9 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
+  'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
+  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
+  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/restore.php

@@ -0,0 +1,8 @@
+<?php
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
+
+protect_route(['admin']);
+
+$role = "admin";
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/admin/system.php

@@ -13,6 +13,8 @@ $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 $js_minifier->add('/web/js/site/identity.js');
+$js_minifier->add('/web/js/site/archive.js');
+$js_minifier->add('/web/js/site/backup.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -116,11 +118,15 @@ $template_data = [
   'ui_theme' => customize('get', 'ui_theme'),
   'password_complexity' => password_complexity('get'),
   'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(),
+  'archive_settings' => \Mailcow\Archive\ArchiveModule::fromGlobals()->api->getSettings(),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
   'iam_settings' => $iam_settings,
   'mbox_templates' => $mbox_templates,
+  'backup_timezone' => getenv('TZ') ?: 'UTC',
+  'backup_hostname' => getenv('MAILCOW_HOSTNAME') ?: '',
+  'backup_demo_mode' => $DEMO_MODE,
   'lang_admin' => json_encode($lang['admin']),
   'lang_datatables' => json_encode($lang['datatables'])
 ];

data/web/autodiscover-json.php

@@ -8,7 +8,7 @@ if(file_exists('inc/vars.local.inc.php')) {
 $autodiscover_config = array_merge($default_autodiscover_config, $autodiscover_config);
 
 header('Content-type: application/json');
-if (strtolower($_GET['Protocol']) == 'activesync' && getenv('SKIP_SOGO') != "y") {
+if (strtolower($_GET['Protocol']) == 'activesync' && $autodiscover_config['autodiscoverType'] == 'activesync' && getenv('SKIP_SOGO') != "y") {
   echo '{"Protocol":"ActiveSync","Url":"' . $autodiscover_config['activesync']['url'] . '"}';
 }
 elseif (strtolower($_GET['Protocol']) == 'autodiscoverv1') {

data/web/autodiscover.php

@@ -31,7 +31,7 @@ error_reporting(0);
 
 $data = trim(file_get_contents("php://input"));
 
-if (strpos($data, 'autodiscover/outlook/responseschema') !== false) {
+if (strpos($data, 'autodiscover/outlook/responseschema') !== false && $autodiscover_config['autodiscoverType'] == 'activesync') {
   $autodiscover_config['autodiscoverType'] = 'imap';
   if ($autodiscover_config['useEASforOutlook'] == 'yes' &&
     // Office for macOS does not support EAS
@@ -144,41 +144,15 @@ try {
 }
 
 $username = trim((string)$email);
-try {
-  $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox` 
-    INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
-    WHERE `mailbox`.`username` = :username 
-    AND `mailbox`.`active` = '1'
-    AND `domain`.`active` = '1'");
-  $stmt->execute(array(':username' => $username));
-  $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
-}
-catch(PDOException $e) {
-  // Database error - return error response with complete XML
-  list($usec, $sec) = explode(' ', microtime());
-?>
-  <Response>
-    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
-      <ErrorCode>500</ErrorCode>
-      <Message>Database Error</Message>
-      <DebugData />
-    </Error>
-  </Response>
-</Autodiscover>
-<?php
-  exit(0);
-}
-
-// Mailbox not found or not active - return generic error to prevent user enumeration
-if (empty($MailboxData)) {
+if (!filter_var($username, FILTER_VALIDATE_EMAIL)) {
   try {
     $json = json_encode(
       array(
         "time" => time(),
         "ua" => $_SERVER['HTTP_USER_AGENT'],
-        "user" => $email,
+        "user" => "none",
         "ip" => $_SERVER['REMOTE_ADDR'],
-        "service" => "Error: mailbox not found or inactive"
+        "service" => "Error: invalid email address in request"
       )
     );
     $redis->lPush('AUTODISCOVER_LOG', $json);
@@ -202,13 +176,33 @@ if (empty($MailboxData)) {
 <?php
   exit(0);
 }
+try {
+  $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox`
+    INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
+    WHERE `mailbox`.`username` = :username
+    AND `mailbox`.`active` = '1'
+    AND `domain`.`active` = '1'");
+  $stmt->execute(array(':username' => $username));
+  $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
+}
+catch(PDOException $e) {
+  // Database error - return error response with complete XML
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>500</ErrorCode>
+      <Message>Database Error</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
 
-if (!empty($MailboxData['name'])) {
-  $displayname = $MailboxData['name'];
-}
-else {
-  $displayname = $email;
-}
+$email = $username;
+$displayname = $email;
 try {
   $json = json_encode(
     array(
@@ -216,11 +210,16 @@ try {
       "ua" => $_SERVER['HTTP_USER_AGENT'],
       "user" => $email,
       "ip" => $_SERVER['REMOTE_ADDR'],
-      "service" => $autodiscover_config['autodiscoverType']
+      // The answer is the same, only the log tells unknown or inactive mailboxes apart
+      "service" => (empty($MailboxData)) ? "Error: mailbox not found or inactive" : $autodiscover_config['autodiscoverType']
     )
   );
   $redis->lPush('AUTODISCOVER_LOG', $json);
   $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+  if (empty($MailboxData)) {
+    $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+    error_log("Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+  }
 }
 catch (RedisException $e) {
   $_SESSION['return'][] = array(
@@ -243,7 +242,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['imap']['server'];?></Server>
         <Port><?=$autodiscover_config['imap']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -253,7 +252,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['smtp']['server'];?></Server>
         <Port><?=$autodiscover_config['smtp']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -265,15 +264,15 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     ?>
       <Protocol>
         <Type>CalDAV</Type>
-        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
+        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
       </Protocol>
       <Protocol>
         <Type>CardDAV</Type>
-        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
+        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
       </Protocol>
     <?php
     }
@@ -288,7 +287,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     <Culture>en:en</Culture>
     <User>
       <DisplayName><?=htmlspecialchars($displayname, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></DisplayName>
-      <EMailAddress><?=$email;?></EMailAddress>
+      <EMailAddress><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></EMailAddress>
     </User>
     <Action>
       <Settings>

data/web/css/build/013-datatables.css

@@ -15,6 +15,10 @@ table.dataTable>tbody>tr.child ul.dtr-details>li {
     border-bottom: 1px solid rgba(0, 0, 0, 0.129);
     padding: 0.5em 0;
 }
+/* like the first entry has no top padding, the last one needs no bottom padding */
+table.dataTable>tbody>tr.child ul.dtr-details>li:last-child {
+    padding-bottom: 0;
+}
 table.dataTable.dtr-inline.collapsed>tbody>tr>td.dtr-control:before:hover, 
 table.dataTable.dtr-inline.collapsed>tbody>tr>th.dtr-control:before:hover {
     background-color: var(--mc-dtr-hover-bg);
@@ -28,7 +32,8 @@ table.dataTable td.dt-control:before {
     color: var(--mc-dtr-color);
     height: 1em;
     width: 1em;
-    line-height: 1.25em;
+    line-height: 1em;
+    margin-top: calc(-0.5em - 1.5px);
     border-radius: 0px;
     box-shadow: none;
     font-size: 14px;
@@ -39,11 +44,29 @@ table.dataTable.dtr-inline.collapsed>tbody>tr.parent>th.dtr-control:before,
 table.dataTable td.dt-control:before {
     background-color: var(--mc-dtr-active-bg) !important;
 }
+/* The default initial order targets column 0, which is usually the non-orderable expand column */
+table.dataTable thead > tr > th.sorting_disabled.sorting_asc,
+table.dataTable thead > tr > th.sorting_disabled.sorting_desc {
+    padding-right: 0.5rem;
+    cursor: auto;
+}
+table.dataTable thead > tr > th.sorting_disabled.sorting_asc:before,
+table.dataTable thead > tr > th.sorting_disabled.sorting_asc:after,
+table.dataTable thead > tr > th.sorting_disabled.sorting_desc:before,
+table.dataTable thead > tr > th.sorting_disabled.sorting_desc:after {
+    display: none;
+}
 table.dataTable.table-striped>tbody>tr>td {
     vertical-align: middle;
 }
 table.dataTable.table-striped>tbody>tr>td>input[type="checkbox"] {
-    margin-top: 7px;
+    /* centers the 1em checkbox in the 1.5em line, level with the expand control */
+    margin-top: 0.25em;
+}
+/* the vertical margin is meant for the tag editor in edit forms, in tables it only adds height */
+table.dataTable .tag-badge {
+    margin-top: 0;
+    margin-bottom: 0;
 }
 td.dtr-col-lg {
     min-width: 350px;

data/web/css/build/014-mailcow.css

@@ -139,6 +139,13 @@ body {
   display: inline-block;
   font-size: inherit;
 }
+.badge .bi {
+  font-size: inherit;
+}
+/* keeps icons from making a badge taller than one with text only */
+.badge .bi::before {
+  line-height: 0;
+}
 .btn-group-xs > .btn, .btn-xs {
   padding: .25rem .4rem;
   font-size: .875rem;
@@ -343,6 +350,7 @@ code {
 }
 .tag-badge .bi {
   font-size: 12px;
+  line-height: 0;
 }
 .tag-badge.btn-badge:hover {
   filter: brightness(0.9);

data/web/css/build/015-responsive.css

@@ -191,7 +191,7 @@
       height: 2rem;
       width: 2rem;
       line-height: 2rem;
-      margin-top: -15px;
+      margin-top: calc(-1rem - 1.5px);
   }
 
   li .dtr-data {

data/web/css/site/admin.css

@@ -1,83 +1,133 @@
-.pagination a {
-  text-decoration: none !important;
-}
-.panel.panel-default {
-  overflow: visible !important;
-}
-.table-responsive {
-  overflow: visible !important;
-}
-.table-responsive {
-  overflow-x: scroll !important;
-}
-body {
-  overflow-y:scroll;
-}
-/* Fix modal moving content left */
-body.modal-open {
-  overflow-y:scroll;
-  padding-right: inherit !important;
-}
-@media (min-width: 992px) {
-  .container {
-      width: 80%;
-  }
-}
-.mass-actions-admin {
-  user-select: none;
-}
-.inputMissingAttr {
-  border-color: #FF4136;
-}
-.rotate {
-  -moz-transition: all 0.3s linear;
-  -webkit-transition: all 0.3s linear;
-  transition: all 0.3s linear;
-}
-.rotate.animation {
-  -ms-transform:rotateX(180deg);
-  -moz-transform:rotateX(180deg);
-  -webkit-transform:rotateX(180deg);
-  transform:rotateX(180deg);
-}
-.anchor {
-  display: block;
-  height: 65px;
-  margin-top: -65px;
-  visibility: hidden;
-}
-.thumbnail img {
-  min-height:100px;
-  height:100px;
-}
-.nav-tabs>li>a {
-  z-index: 1;
-}
-.table-condensed .input-sm {
-  width: 100%!important;
-}
-.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
-  padding: 3px;
-}
-table tbody tr td input[type="checkbox"] {
-  cursor: pointer;
-}
-#quarantine_template {
-  margin:20px;
-}
-.regex-input {
-  font-family: Consolas,monaco,monospace;
-  font-size: 1rem;
-}
-.label-keys {
-  font-size:100%;
-  margin: 0px !important;
-  white-space: normal !important;
-}
-.key-action {
-  font-weight:bold;
-  color:white !important;
-}
-.dkim-label {
-  margin: 0 0 8px !important;
-}
+.pagination a {
+  text-decoration: none !important;
+}
+.panel.panel-default {
+  overflow: visible !important;
+}
+.table-responsive {
+  overflow: visible !important;
+}
+.table-responsive {
+  overflow-x: scroll !important;
+}
+body {
+  overflow-y:scroll;
+}
+/* Fix modal moving content left */
+body.modal-open {
+  overflow-y:scroll;
+  padding-right: inherit !important;
+}
+@media (min-width: 992px) {
+  .container {
+      width: 80%;
+  }
+}
+.mass-actions-admin {
+  user-select: none;
+}
+.inputMissingAttr {
+  border-color: #FF4136;
+}
+.rotate {
+  -moz-transition: all 0.3s linear;
+  -webkit-transition: all 0.3s linear;
+  transition: all 0.3s linear;
+}
+.rotate.animation {
+  -ms-transform:rotateX(180deg);
+  -moz-transform:rotateX(180deg);
+  -webkit-transform:rotateX(180deg);
+  transform:rotateX(180deg);
+}
+.anchor {
+  display: block;
+  height: 65px;
+  margin-top: -65px;
+  visibility: hidden;
+}
+.thumbnail img {
+  min-height:100px;
+  height:100px;
+}
+.nav-tabs>li>a {
+  z-index: 1;
+}
+.table-condensed .input-sm {
+  width: 100%!important;
+}
+.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
+  padding: 3px;
+}
+table tbody tr td input[type="checkbox"] {
+  cursor: pointer;
+}
+#quarantine_template {
+  margin:20px;
+}
+.regex-input {
+  font-family: Consolas,monaco,monospace;
+  font-size: 1rem;
+}
+.label-keys {
+  font-size:100%;
+  margin: 0px !important;
+  white-space: normal !important;
+}
+.key-action {
+  font-weight:bold;
+  color:white !important;
+}
+.dkim-label {
+  margin: 0 0 8px !important;
+}
+/* backup schedule: the weekday toggles and the remove buttons of the entries react to the mouse, in every theme */
+.backup-schedule-days .btn:hover {
+  background-image: linear-gradient(rgba(128, 128, 128, .3), rgba(128, 128, 128, .3));
+}
+.backup-schedule-remove {
+  opacity: .6;
+}
+.backup-schedule-remove:hover, .backup-schedule-remove:focus-visible {
+  opacity: 1;
+  color: var(--bs-danger) !important;
+}
+/* result and log of a backup job: the border of the theme in light and dark mode (Bootstrap's .border stays light) */
+.backup-pre {
+  border: 1px solid var(--mc-border);
+}
+/* result of a restore job: an entry to open per mailbox, numbers in columns */
+.backup-result {
+  border: 1px solid var(--mc-border);
+}
+.backup-result > details + details {
+  border-top: 1px solid var(--mc-border);
+}
+.backup-result summary {
+  list-style: none;
+  cursor: pointer;
+}
+.backup-result summary::-webkit-details-marker {
+  display: none;
+}
+.backup-result summary:hover {
+  background-image: linear-gradient(rgba(128, 128, 128, .12), rgba(128, 128, 128, .12));
+}
+.backup-result summary .bi-chevron-right {
+  transition: transform .15s;
+}
+.backup-result details[open] > summary .bi-chevron-right {
+  transform: rotate(90deg);
+}
+.backup-result-body {
+  padding-left: 2.6rem !important;
+}
+.backup-result-body .table {
+  --bs-table-bg: transparent;
+}
+.backup-result-body .num {
+  width: 8.5rem;
+  text-align: right;
+  font-variant-numeric: tabular-nums;
+  white-space: nowrap;
+}

data/web/css/themes/mailcow-darkmode.css

@@ -495,4 +495,16 @@ input::placeholder {
 }
 .btn-light, .btn-light:hover {
     background-image: none;
-}
+}
+/* native time picker icon and popup in dark mode */
+input[type="time"] {
+    color-scheme: dark;
+}
+/* switches while off: a light knob and a visible border (Bootstrap's dark knob is for light pages, the dark mode
+   does not set data-bs-theme) */
+.form-switch .form-check-input:not(:checked):not(:focus) {
+    --bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.6%29'/%3e%3c/svg%3e");
+}
+.form-switch .form-check-input:not(:checked) {
+    border-color: var(--mc-btn-outline-border);
+}

data/web/domainadmin/mailbox.php

@@ -11,6 +11,8 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
+$js_minifier->add('/web/js/site/mail-policy.js');
+$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -45,6 +47,9 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
+  'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
+  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
+  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/domainadmin/restore.php

@@ -0,0 +1,8 @@
+<?php
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.domainadmin.inc.php';
+
+protect_route(['domainadmin']);
+
+$role = "domainadmin";
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/edit.php

@@ -65,7 +65,8 @@ if (isset($_SESSION['mailcow_cc_role'])) {
             'mta_sts' => $mta_sts,
             'mailboxes' => mailbox('get', 'mailboxes', $_GET["domain"]),
             'aliases' => mailbox('get', 'aliases', $_GET["domain"], 'address'),
-            'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"])
+            'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"]),
+            'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled()
           ];
       }
     }
@@ -75,6 +76,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
         $template_data = [
           'template' => $domain_template,
           'rl' => ['frame' => $domain_template['attributes']['rl_frame']],
+          'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
         ];
         $template = 'edit/domain-templates.twig';
         $result = true;
@@ -85,6 +87,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           $template_data = [
             'template' => $mailbox_template,
             'rl' => ['frame' => $mailbox_template['attributes']['rl_frame']],
+            'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
           ];
           $template = 'edit/mailbox-templates.twig';
           $result = true;
@@ -115,6 +118,28 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
         ];
     }
+    elseif (isset($_GET['mail-policy']) &&
+      is_numeric($_GET['mail-policy'])) {
+        $mail_policy = \Mailcow\MailPolicy\MailPolicyModule::fromGlobals();
+        $result = $mail_policy->api->getPolicy($_GET['mail-policy']);
+        $template = 'edit/mail-policy.twig';
+        // Choices: mailboxes and groups of the domain of the policy
+        $template_data = [
+          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
+          'mail_policy_groups' => !empty($result) ? $mail_policy->policies->groupChoices([$result['domain']]) : [],
+        ];
+    }
+    elseif (isset($_GET['retention-rule']) &&
+      is_numeric($_GET['retention-rule'])) {
+        $retention = \Mailcow\Retention\RetentionModule::fromGlobals();
+        $result = $retention->api->getRule($_GET['retention-rule']);
+        $template = 'edit/retention-rule.twig';
+        // Choices: mailboxes and groups of the domain of the rule
+        $template_data = [
+          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
+          'retention_groups' => !empty($result) ? $retention->rules->groupChoices([$result['domain']]) : [],
+        ];
+    }
     elseif (isset($_GET['api-key']) &&
       is_numeric($_GET['api-key'])) {
         $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
@@ -162,6 +187,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'user_acls' => acl('get', 'user', $mailbox),
           'mailbox_details' => $result,
           'iam_settings' => $iam_settings,
+          'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
         ];
       }
     }
@@ -227,7 +253,8 @@ if (isset($_SESSION['mailcow_cc_role'])) {
     if (isset($_GET['syncjob']) &&
       is_numeric($_GET['syncjob'])) {
         $id = $_GET["syncjob"];
-        $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id);
+        // Without the syncjobs ACL a job cannot be saved, so do not show the form
+        $result = (($_SESSION['acl']['syncjobs'] ?? '0') == '1') ? \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id) : [];
         $template = 'edit/syncjob.twig';
       }
     elseif (isset($_GET['syncjob_source']) &&

data/web/inc/footer.inc.php

@@ -74,6 +74,9 @@ $globalVariables = [
   'lang_danger' => json_encode($lang['danger']),
   'lang_syncjobs' => json_encode($lang['syncjobs']),
   'lang_groups' => json_encode($lang['groups']),
+  'lang_mail_policy' => json_encode($lang['mail_policy'] ?? []),
+  'lang_retention' => json_encode($lang['retention'] ?? []),
+  'lang_backup' => json_encode($lang['backup'] ?? []),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.admin.inc.php

@@ -74,6 +74,15 @@ function admin($_action, $_data = null) {
         $usernames = $_data['username'];
       }
       foreach ($usernames as $username) {
+        // A pending password update only allows changing the own password
+        if (!empty($_SESSION['pending_pw_update']) && $username != $_SESSION['mailcow_cc_username']) {
+          $_SESSION['return'][] = array(
+            'type' => 'danger',
+            'log' => array(__FUNCTION__, $_action, $_data_log),
+            'msg' => 'access_denied'
+          );
+          continue;
+        }
         $is_now = admin('details', $username);
         if (!empty($is_now)) {
           $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];

data/web/inc/functions.app_passwd.inc.php

@@ -5,6 +5,15 @@ function app_passwd($_action, $_data = null) {
   $_data_log = $_data;
   !isset($_data_log['app_passwd']) ?: $_data_log['app_passwd'] = '*';
   !isset($_data_log['app_passwd2']) ?: $_data_log['app_passwd2'] = '*';
+  if (in_array($_action, array('add', 'edit', 'delete')) &&
+    (!isset($_SESSION['acl']['app_passwds']) || $_SESSION['acl']['app_passwds'] != "1")) {
+    $_SESSION['return'][] = array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $_action, $_data_log),
+      'msg' => 'access_denied'
+    );
+    return false;
+  }
   if (isset($_data['username']) && filter_var($_data['username'], FILTER_VALIDATE_EMAIL)) {
     if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data['username'])) {
       $_SESSION['return'][] = array(

data/web/inc/functions.auth.inc.php

@@ -6,6 +6,8 @@ function check_login($user, $pass, $extra = null) {
   $is_internal = $extra['is_internal'];
   $role = $extra['role'];
   $extra['service'] = !isset($extra['service']) ? 'NONE' : $extra['service'];
+  // set when the password is right, but the account does not allow this login
+  $denied = false;
 
   // Try validate admin
   if (!isset($role) || $role == "admin") {
@@ -26,7 +28,7 @@ function check_login($user, $pass, $extra = null) {
 
   // Try validate app password
   if (!isset($role) || $role == "app") {
-    $result = apppass_login($user, $pass, $extra);
+    $result = apppass_login($user, $pass, $extra, $denied);
     if ($result !== false) {
       $real_rip = ($_SERVER['HTTP_X_REAL_IP'] ?? $_SERVER['REMOTE_ADDR']);
       set_sasl_log($user, $real_rip, $extra['service'], $pass);
@@ -37,7 +39,7 @@ function check_login($user, $pass, $extra = null) {
   // Try validate user
   if (!isset($role) || $role == "user") {
     try {
-      $result = user_login($user, $pass, $extra);
+      $result = user_login($user, $pass, $extra, $denied);
     } catch (\Mailcow\Auth\IdpUnavailable $e) {
       // no failed login, netfilter must not count it
       error_log("mailcow UI: Identity provider unavailable for " . $user . " by " . $_SERVER['REMOTE_ADDR'] . ": " . $e->getMessage());
@@ -54,19 +56,20 @@ function check_login($user, $pass, $extra = null) {
       return $result;
     }
   }
-
   // skip log and only return false if it's an internal request
   if ($is_internal == true) return false;
 
+  // A denied login is logged, but not counted by netfilter (e.g. EAS/DAV clients of accounts without access)
+  $msg = ($denied ? "mailcow UI: Login denied for " : "mailcow UI: Invalid password for ") . $user . " by " . $_SERVER['REMOTE_ADDR'];
   if (!isset($_SESSION['ldelay'])) {
     $_SESSION['ldelay'] = "0";
-    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
-    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
+    error_log($msg);
   }
   elseif (!isset($_SESSION['mailcow_cc_username'])) {
     $_SESSION['ldelay'] = $_SESSION['ldelay']+0.5;
-    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
-    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
+    error_log($msg);
   }
   $_SESSION['return'][] =  array(
     'type' => 'danger',
@@ -207,7 +210,7 @@ function domainadmin_login($user, $pass){
 
   return false;
 }
-function user_login($user, $pass, $extra = null){
+function user_login($user, $pass, $extra = null, &$denied = false){
   global $pdo;
   global $iam_provider;
   global $iam_settings;
@@ -261,11 +264,14 @@ function user_login($user, $pass, $extra = null){
         if ($extra['service'] != 'NONE') {
           $key = strtolower($extra['service']) . "_access";
           if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
+            $denied = true;
             return false;
           }
         }
         return true;
       }
+      // the IdP accepted the password, the mailbox is inactive
+      $denied = true;
     }
     clear_session();
     return false;
@@ -273,18 +279,16 @@ function user_login($user, $pass, $extra = null){
 
   // check if user has access to service (imap, smtp, pop3, sieve) if service is set
   $row['attributes'] = json_decode($row['attributes'], true);
-  // Shared mailboxes have no login of their own, members open them through group shares
-  if (($row['attributes']['shared'] ?? '0') == '1') {
-    return false;
-  }
-  if ($extra['service'] != 'NONE') {
-    $key = strtolower($extra['service']) . "_access";
-    if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
-      return false;
+  $key = strtolower($extra['service']) . "_access";
+  // Shared mailboxes have no login of their own, members open them through group shares.
+  // Checked before asking an IdP (no template side effects).
+  if (($row['attributes']['shared'] ?? '0') == '1'
+    || ($extra['service'] != 'NONE' && isset($row['attributes'][$key]) && $row['attributes'][$key] != '1')
+    || $row['active'] != '1' || $row['d_active'] != '1') {
+    // Denied, not failed, unless the password is wrong; IdP passwords are not checked for that
+    if ($row['authsource'] != 'mailcow' || verify_hash($row['password'], $pass) !== false) {
+      $denied = true;
     }
-  }
-  // IdP accounts: refuse inactive mailboxes and domains before asking the IdP (no template side effects)
-  if ($row['authsource'] != 'mailcow' && ($row['active'] != '1' || $row['d_active'] != '1')) {
     return false;
   }
   switch ($row['authsource']) {
@@ -303,6 +307,7 @@ function user_login($user, $pass, $extra = null){
           $stmt->execute(array(':user' => $user));
           $row = $stmt->fetch(PDO::FETCH_ASSOC);
           if (empty($row)) {
+            $denied = true;
             return false;
           }
 
@@ -361,6 +366,7 @@ function user_login($user, $pass, $extra = null){
         $stmt->execute(array(':user' => $user));
         $row = $stmt->fetch(PDO::FETCH_ASSOC);
         if (empty($row)) {
+          $denied = true;
           return false;
         }
 
@@ -403,9 +409,6 @@ function user_login($user, $pass, $extra = null){
       return $result;
     break;
     case 'mailcow':
-      if ($row['active'] != 1 || $row['d_active'] != 1) {
-        return false;
-      }
       // verify password
       if (verify_hash($row['password'], $pass) !== false) {
 
@@ -446,13 +449,15 @@ function user_login($user, $pass, $extra = null){
           }
           return "user";
         }
+        // TFA accounts use app passwords in mail clients
+        $denied = true;
       }
     break;
   }
 
   return false;
 }
-function apppass_login($user, $pass, $extra = null){
+function apppass_login($user, $pass, $extra = null, &$denied = false){
   global $pdo;
 
   $is_internal = $extra['is_internal'];
@@ -473,15 +478,12 @@ function apppass_login($user, $pass, $extra = null){
     return false;
   }
 
-  // fetch app password data
-  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id` FROM `app_passwd`
+  // fetch app password data, inactive ones too: their right password is denied, not failed
+  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id`,
+    `mailbox`.`active` as `mailbox_active`, `domain`.`active` as `domain_active`, `mailbox`.`attributes` as `mailbox_attributes` FROM `app_passwd`
     INNER JOIN `mailbox` ON `mailbox`.`username` = `app_passwd`.`mailbox`
     INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
     WHERE `mailbox`.`kind` NOT REGEXP 'location|thing|group'
-      AND `mailbox`.`active` = '1'
-      AND `domain`.`active` = '1'
-      AND `app_passwd`.`active` = '1'
-      AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
       AND `app_passwd`.`mailbox` = :user"
   );
   // fetch password data
@@ -491,16 +493,25 @@ function apppass_login($user, $pass, $extra = null){
   $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
 
   foreach ($rows as $row) {
-    if ($extra['service'] != 'NONE' && $row[strtolower($extra['service']) . '_access'] != '1'){
+    // verify password
+    if (verify_hash($row['password'], $pass) === false) {
       continue;
     }
 
-    // verify password
-    if (verify_hash($row['password'], $pass) !== false) {
-      $_SESSION['app_passwd_id'] = $row['app_passwd_id'];
-      unset($_SESSION['ldelay']);
-      return "user";
+    $key = strtolower($extra['service']) . '_access';
+    $mailbox_attributes = json_decode($row['mailbox_attributes'], true);
+    // the protocols of the mailbox only limit direct user access, app passwords have their own
+    if ($row['active'] != '1' || $row['mailbox_active'] != '1' || $row['domain_active'] != '1'
+      || ($mailbox_attributes['shared'] ?? '0') == '1'
+      || ($extra['service'] != 'NONE' && $row[$key] != '1')) {
+      // right password, but the app password does not allow this login
+      $denied = true;
+      continue;
     }
+
+    $_SESSION['app_passwd_id'] = $row['app_passwd_id'];
+    unset($_SESSION['ldelay']);
+    return "user";
   }
 
   return false;
@@ -538,7 +549,8 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
 
   // get the mailcow_password attribute from keycloak user
   $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users";
-  $queryParams = array('email' => $user, 'exact' => true);
+  // as string, http_build_query() would send exact=1 and Keycloak would search for substrings
+  $queryParams = array('email' => $user, 'exact' => 'true');
   $queryString = http_build_query($queryParams);
   $curl = curl_init();
   curl_setopt($curl, CURLOPT_TIMEOUT, 7);
@@ -559,7 +571,10 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   if ($code != 200) {
     throw new \Mailcow\Auth\IdpUnavailable('Keycloak returned HTTP ' . $code);
   }
-  // exact=true can still return several users with that email
+  // exact=true can still return several users with that email, only the exact address counts
+  if (is_array($users)) {
+    $users = array_values(array_filter($users, fn($u) => is_array($u) && strcasecmp((string)($u['email'] ?? ''), $user) === 0));
+  }
   if (!is_array($users) || count($users) != 1) {
     return false;
   }
@@ -586,6 +601,13 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
 
   // validate mailcow_password
   $mailcow_password = $user_res['attributes']['mailcow_password'][0];
+  // Keycloak returns binary LDAP attributes like userPassword Base64 encoded
+  if (substr($mailcow_password, 0, 1) != '{') {
+    $decoded_password = base64_decode($mailcow_password, true);
+    if ($decoded_password !== false && substr($decoded_password, 0, 1) == '{') {
+      $mailcow_password = $decoded_password;
+    }
+  }
   if (!verify_hash($mailcow_password, $pass)) {
     return false;
   }
@@ -710,12 +732,15 @@ function ldap_mbox_login($user, $pass, $extra = null){
       ->select(array_values(array_filter(array_merge([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn'],
         \Mailcow\Identity\Directory\LdapDirectory::idAttributes($iam_settings['id_attribute'] ?? '')))));
 
-    $user_res = $ldap_query->firstOrFail();
-  } catch (\LdapRecord\Query\ObjectNotFoundException $e) {
-    return false;
+    $entries = $ldap_query->get();
   } catch (Exception $e) {
     throw new \Mailcow\Auth\IdpUnavailable('LDAP search failed: ' . $e->getMessage(), 0, $e);
   }
+  // the login name must identify exactly one entry, several matches are ambiguous
+  if (!is_array($entries) || count($entries) != 1) {
+    return false;
+  }
+  $user_res = reset($entries);
   $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
   if (!$idp_login->verifyLdapPassword($iam_provider, (string)$user_res['dn'], (string)$pass)) {
     return false;

data/web/inc/functions.fail2ban.inc.php

@@ -145,6 +145,28 @@ function fail2ban($_action, $_data = null, $_extra = null) {
           );
           return true;
         }
+        // Queue all automatic bans for unbanning, the denylist stays
+        elseif ($_data['action'] == "unban-all") {
+          try {
+            foreach ($redis->hKeys('F2B_ACTIVE_BANS') as $network) {
+              $redis->hSet('F2B_QUEUE_UNBAN', $network, 1);
+            }
+          }
+          catch (RedisException $e) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log),
+              'msg' => array('redis_error', $e)
+            );
+            return false;
+          }
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_data_log),
+            'msg' => 'f2b_modified'
+          );
+          return true;
+        }
 
         // Start actions in dependency of network
         if (!empty($_data['network'])) {

data/web/inc/functions.fwdhost.inc.php

@@ -35,6 +35,13 @@ function fwdhost($_action, $_data = null) {
         );
         return false;
       }
+      $hosts = array_unique(array_map(function ($host) {
+        $net = explode('/', $host, 2);
+        if (filter_var($net[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
+          $net[0] = inet_ntop(inet_pton($net[0]));
+        }
+        return implode('/', $net);
+      }, $hosts));
       foreach ($hosts as $host) {
         try {
           $redis->hSet('WHITELISTED_FWD_HOST', $host, $source);

data/web/inc/functions.inc.php

@@ -539,6 +539,7 @@ function sys_mail($_data) {
       $mail->Body = $mass_text;
     }
     $mail->XMailer = 'MooMassMail';
+    $archive = \Mailcow\Archive\ArchiveModule::fromGlobals();
     foreach ($rcpts as $rcpt) {
       $mail->AddAddress($rcpt);
       if (!$mail->send()) {
@@ -548,6 +549,10 @@ function sys_mail($_data) {
           'msg' => 'Mailer error (RCPT "' . htmlspecialchars($rcpt) . '"): ' . str_replace('https://github.com/PHPMailer/PHPMailer/wiki/Troubleshooting', '', $mail->ErrorInfo)
         );
       }
+      else {
+        // Delivered by LMTP without rspamd, so the archive gets it here
+        $archive->journal->submitDelivered($mail->getSentMIMEMessage(), $mass_from, array($rcpt), 'massmail');
+      }
       $mail->ClearAllRecipients();
     }
   }
@@ -1394,6 +1399,18 @@ function set_tfa($_data) {
     case "webauthn":
         $key_id = (!isset($_data["key_id"])) ? 'unidentified' : $_data["key_id"];
 
+        // Refuse incomplete WebAuthn registrations. Without this guard a request
+        // that does not contain the parsed credential data (e.g. a plain form
+        // POST) would silently insert an empty keyHandle while reporting success.
+        if (empty($_data['registration']) || empty($_data['registration']->credentialId)) {
+          $_SESSION['return'][] = array(
+            'type' => 'danger',
+            'log' => array(__FUNCTION__, $_data_log),
+            'msg' => 'webauthn_registration_failed'
+          );
+          return false;
+        }
+
         $stmt = $pdo->prepare("INSERT INTO `tfa` (`username`, `key_id`, `authmech`, `keyHandle`, `publicKey`, `certificate`, `counter`, `active`)
         VALUES (?, ?, 'webauthn', ?, ?, ?, ?, '1')");
         $stmt->execute(array(
@@ -1416,11 +1433,11 @@ function set_tfa($_data) {
       // Block TFA removal if force_tfa policy is active
       $is_forced_tfa = false;
       if ($_SESSION['mailcow_cc_role'] === 'user') {
-        $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
+        $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
         $stmt_check->execute(array($username));
         $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
       } else {
-        $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
+        $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
         $stmt_check->execute(array($username));
         $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
       }
@@ -1687,11 +1704,11 @@ function unset_tfa_key($_data) {
     // Block key removal if force_tfa policy is active
     $is_forced_tfa = false;
     if ($_SESSION['mailcow_cc_role'] === 'user') {
-      $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
+      $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
       $stmt_check->execute(array($username));
       $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
     } else {
-      $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
+      $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
       $stmt_check->execute(array($username));
       $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
     }
@@ -1857,6 +1874,40 @@ function get_tfa($username = null, $id = null) {
     }
 }
 function verify_tfa_login($username, $_data) {
+  global $redis;
+  global $TFA_MAX_FAILS;
+  global $TFA_FAIL_WINDOW;
+
+  if (empty($username)) {
+    return false;
+  }
+  // Count failed second factors per account, so codes cannot be guessed from many addresses
+  $fails_key = 'TFA_FAILS_' . $username;
+  if (intval($redis->get($fails_key)) >= $TFA_MAX_FAILS) {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $username, '*'),
+      'msg' => 'tfa_login_locked'
+    );
+    $verified = false;
+  }
+  else {
+    $verified = verify_tfa_token($username, $_data);
+  }
+  if ($verified !== true) {
+    // The window starts with the first failure and is not extended by further attempts
+    if ($redis->incr($fails_key) == 1) {
+      $redis->expire($fails_key, $TFA_FAIL_WINDOW);
+    }
+    // Same message format as failed passwords, so the default fail2ban rule counts it
+    error_log("mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
+    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
+    return false;
+  }
+  $redis->del($fails_key);
+  return true;
+}
+function verify_tfa_token($username, $_data) {
   global $pdo;
   global $yubi;
   global $u2f;
@@ -2386,7 +2437,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
           case "use_ssl":
           case "use_tls":
           case "login_provisioning":
-          case "ignore_ssl_errors":
+          case "ignore_ssl_error":
             $settings[$row["key"]] = boolval($row["value"]);
           break;
           default:
@@ -2410,7 +2461,7 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
       if ($settings["authsource"] == "ldap"){
         $settings['use_ssl'] = !isset($settings['use_ssl']) ? false : $settings['use_ssl'];
         $settings['use_tls'] = !isset($settings['use_tls']) ? false : $settings['use_tls'];
-        $settings['ignore_ssl_errors'] = !isset($settings['ignore_ssl_errors']) ? false : $settings['ignore_ssl_errors'];
+        $settings['ignore_ssl_error'] = !isset($settings['ignore_ssl_error']) ? false : $settings['ignore_ssl_error'];
       }
       return $settings;
     break;
@@ -3013,6 +3064,18 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
+      // A refresh keeps the logged-in mailbox: a changed IdP email or another IdP user ends the session
+      if (strtolower($info['email']) !== strtolower($_SESSION['mailcow_cc_username'] ?? '') ||
+        !\Mailcow\Auth\AuthModule::fromGlobals()->idp->verifyLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
+        clear_session();
+        $_SESSION['return'][] =  array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $info['email'], 'The IdP user does not match the logged-in mailbox'),
+          'msg' => 'refresh_login_failed'
+        );
+        return false;
+      }
+
       // The mailbox may have been deleted or deactivated since the login
       $stmt = $pdo->prepare("SELECT `mailbox`.`authsource` FROM `mailbox`
         INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
@@ -3060,7 +3123,17 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
     case "get-keycloak-admin-token":
       // get access_token for service account of mailcow client
       if ($iam_settings['authsource'] !== 'keycloak') return false;
-      if (isset($iam_settings['access_token'])) {
+      // only introspect a cached access_token that is not (about to be) expired,
+      // introspecting an expired token makes Keycloak log INTROSPECT_TOKEN_ERROR
+      $token_not_expired = false;
+      if (!empty($iam_settings['access_token'])) {
+        $token_parts = explode('.', $iam_settings['access_token']);
+        if (count($token_parts) === 3) {
+          $token_payload = json_decode(base64_decode(strtr($token_parts[1], '-_', '+/')), true);
+          $token_not_expired = is_array($token_payload) && isset($token_payload['exp']) && intval($token_payload['exp']) > time() + 30;
+        }
+      }
+      if ($token_not_expired) {
         // check if access_token is valid
         $url = "{$iam_settings['server_url']}/realms/{$iam_settings['realm']}/protocol/openid-connect/token/introspect";
         $req = http_build_query(array(
@@ -3111,6 +3184,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         ':key' => 'access_token',
         ':value' => $res['access_token']
       ));
+      // keep the cached token in sync, so subsequent calls in the same run reuse it
+      $iam_settings['access_token'] = $res['access_token'];
       return $res['access_token'];
     break;
   }

data/web/inc/functions.mailbox.inc.php

@@ -366,6 +366,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $relay_unknown_only = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $DOMAIN_DEFAULT_ATTRIBUTES['relay_unknown_only'];
           $backupmx = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $DOMAIN_DEFAULT_ATTRIBUTES['backupmx'];
           $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $DOMAIN_DEFAULT_ATTRIBUTES['gal'];
+          $archive = (intval((isset($_data['archive'])) ? $_data['archive'] : ($DOMAIN_DEFAULT_ATTRIBUTES['archive'] ?? 0)) == 1) ? 1 : 0;
           if ($relay_all_recipients == 1) {
             $backupmx = '1';
           }
@@ -421,8 +422,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             ':domain' => '%@' . $domain
           ));
           // save domain
-          $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`)
-            VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients)");
+          $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`, `archive`)
+            VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients, :archive)");
           $stmt->execute(array(
             ':domain' => $domain,
             ':description' => $description,
@@ -435,7 +436,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             ':gal' => $gal,
             ':active' => $active,
             ':relay_unknown_only' => $relay_unknown_only,
-            ':relay_all_recipients' => $relay_all_recipients
+            ':relay_all_recipients' => $relay_all_recipients,
+            ':archive' => $archive
           ));
           // save tags
           foreach($tags as $index => $tag){
@@ -940,6 +942,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : 0;
           // Shared mailbox: no login of its own, members open it through group shares
           $shared = (isset($_data['shared'])) ? intval($_data['shared']) : 0;
+          // Archiving comes from the template unless the caller may set it
+          $archive = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($MAILBOX_DEFAULT_ATTRIBUTES['archive'] ?? 'inherit'));
           $quarantine_notification = (isset($_data['quarantine_notification'])) ? strval($_data['quarantine_notification']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_notification']);
           $quarantine_category = (isset($_data['quarantine_category'])) ? strval($_data['quarantine_category']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_category']);
           // Validate quarantine_category
@@ -974,6 +978,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'dav_access' => strval($dav_access),
               'relayhost' => strval($relayhost),
               'shared' => strval($shared),
+              'archive' => $archive,
               'passwd_update' => time(),
               'mailbox_format' => strval($MAILBOX_DEFAULT_ATTRIBUTES['mailbox_format']),
               'quarantine_notification' => strval($quarantine_notification),
@@ -1525,6 +1530,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attr['backupmx']                   = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0;
           $attr['relay_all_recipients']       = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0;
           $attr['relay_unknown_only']          = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0;
+          $attr['archive']                    = (isset($_data['archive']) && intval($_data['archive']) == 1) ? 1 : 0;
           $attr['dkim_selector']              = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim";
           $attr['key_size']                   = isset($_data['key_size']) ? intval($_data['key_size']) : 2048;
 
@@ -1600,6 +1606,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attr["rl_value"]                    = (!empty($_data['rl_value'])) ? $_data['rl_value'] : "";
           $attr["force_pw_update"]             = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update']);
           $attr["force_tfa"]                   = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_tfa']);
+          $attr["archive"]                     = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? $MAILBOX_DEFAULT_ATTRIBUTES['archive']);
           $attr["sogo_access"]                 = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['sogo_access']);
           $attr["active"]                      = isset($_data['active']) ? intval($_data['active']) : 1;
           $attr["tls_enforce_in"]              = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_in']);
@@ -2432,6 +2439,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               if ($address != $is_now['address']) {
                 // Send-as grants of groups follow the address
                 \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
+                \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
               }
             }
             $_SESSION['return'][] = array(
@@ -2488,6 +2496,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $gal                  = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal'];
                 $description          = (!empty($_data['description']) && isset($_SESSION['acl']['domain_desc']) && $_SESSION['acl']['domain_desc'] == "1") ? $_data['description'] : $is_now['description'];
                 (int)$relayhost       = (isset($_data['relayhost']) && isset($_SESSION['acl']['domain_relayhost']) && $_SESSION['acl']['domain_relayhost'] == "1") ? intval($_data['relayhost']) : intval($is_now['relayhost']);
+                $archive              = (isset($_data['archive']) && hasACLAccess("archive")) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']);
                 $tags                 = (is_array($_data['tags']) ? $_data['tags'] : array());
               }
               else {
@@ -2501,11 +2510,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 
               $stmt = $pdo->prepare("UPDATE `domain` SET
               `description` = :description,
-              `gal` = :gal
+              `gal` = :gal,
+              `archive` = :archive
                 WHERE `domain` = :domain");
               $stmt->execute(array(
                 ':description' => $description,
                 ':gal' => $gal,
+                ':archive' => $archive,
                 ':domain' => $domain
               ));
               // save tags
@@ -2524,6 +2535,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':domain' => $domain,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
               }
 
               $_SESSION['return'][] = array(
@@ -2538,6 +2553,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $active               = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];
                 $backupmx             = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $is_now['backupmx'];
                 $gal                  = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal'];
+                $archive              = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']);
                 $relay_all_recipients = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : $is_now['relay_all_recipients'];
                 $relay_unknown_only   = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $is_now['relay_unknown_only'];
                 $relayhost            = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : $is_now['relayhost'];
@@ -2652,6 +2668,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               `relay_unknown_only` = :relay_unknown_only,
               `backupmx` = :backupmx,
               `gal` = :gal,
+              `archive` = :archive,
               `active` = :active,
               `quota` = :quota,
               `defquota` = :defquota,
@@ -2666,6 +2683,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':relay_unknown_only' => $relay_unknown_only,
                 ':backupmx' => $backupmx,
                 ':gal' => $gal,
+                ':archive' => $archive,
                 ':active' => $active,
                 ':quota' => $quota,
                 ':defquota' => $defquota,
@@ -2692,6 +2710,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':domain' => $domain,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
               }
 
               $_SESSION['return'][] = array(
@@ -2753,6 +2775,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $attr['backupmx']                   = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0;
             $attr['relay_all_recipients']       = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0;
             $attr['relay_unknown_only']          = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0;
+            $attr['archive']                    = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['attributes']['archive'] ?? 0);
             $attr['dkim_selector']              = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim";
             $attr['key_size']                   = isset($_data['key_size']) ? intval($_data['key_size']) : 2048;
 
@@ -2814,6 +2837,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               (int)$eas_access     = (isset($_data['eas_access']) && hasACLAccess("protocol_access")) ? intval($_data['eas_access']) : intval($is_now['attributes']['eas_access']);
               (int)$dav_access    = (isset($_data['dav_access']) && hasACLAccess("protocol_access")) ? intval($_data['dav_access']) : intval($is_now['attributes']['dav_access']);
               (int)$relayhost       = (isset($_data['relayhost']) && hasACLAccess("mailbox_relayhost")) ? intval($_data['relayhost']) : intval($is_now['attributes']['relayhost']);
+              $archive              = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($is_now['attributes']['archive'] ?? null));
               (int)$shared          = (isset($_data['shared'])) ? intval($_data['shared']) : intval($is_now['attributes']['shared'] ?? 0);
               (int)$quota_m         = (isset_has_content($_data['quota'])) ? intval($_data['quota']) : ($is_now['quota'] / 1048576);
               $name                 = (!empty($_data['name'])) ? ltrim(rtrim($_data['name'], '>'), '<') : $is_now['name'];
@@ -3114,6 +3138,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   `attributes` = JSON_SET(`attributes`, '$.smtp_access', :smtp_access),
                   `attributes` = JSON_SET(`attributes`, '$.eas_access', :eas_access),
                   `attributes` = JSON_SET(`attributes`, '$.dav_access', :dav_access),
+                  `attributes` = JSON_SET(`attributes`, '$.archive', :archive),
                   `attributes` = JSON_SET(`attributes`, '$.recovery_email', :recovery_email),
                   `attributes` = JSON_SET(`attributes`, '$.attribute_hash', :attribute_hash)
                     WHERE `username` = :username");
@@ -3131,6 +3156,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':smtp_access' => $smtp_access,
                 ':eas_access' => $eas_access,
                 ':dav_access' => $dav_access,
+                ':archive' => $archive,
                 ':recovery_email' => $pw_recovery_email,
                 ':relayhost' => $relayhost,
                 ':shared' => strval($shared),
@@ -3170,6 +3196,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':username' => $username,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `mailbox` SET `modified` = NOW() WHERE `username` = :username");
+                $stmt->execute(array(
+                  ':username' => $username,
+                ));
               } catch (Exception $e) {
               }
             }
@@ -3295,7 +3325,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'mailbox_group_imap_grant' => ['owner'],
               'mailbox_group_sogo_grant' => ['owner'],
               'mailbox_group_sogo_acl' => ['c_uid'],
-              'mailbox_group_subscription' => ['username', 'owner']
+              'mailbox_group_subscription' => ['username', 'owner'],
+              'mail_policy_member' => ['username'],
+              'retention_rule_member' => ['username']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3311,6 +3343,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
             // Group memberships and send-as grants follow, group aliases are rebuilt
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
+            // Address entries of mail policies follow
+            \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($old_username, $new_username);
 
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
@@ -3534,6 +3568,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $attr["rl_value"]                    = (!empty($_data['rl_value'])) ? $_data['rl_value'] : $is_now['rl_value'];
             $attr["force_pw_update"]             = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : $is_now['force_pw_update'];
             $attr["force_tfa"]                   = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : $is_now['force_tfa'];
+            $attr["archive"]                     = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? ($is_now['attributes']['archive'] ?? null));
             $attr["sogo_access"]                 = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : $is_now['sogo_access'];
             $attr["active"]                      = isset($_data['active']) ? intval($_data['active']) : $is_now['active'];
             $attr["tls_enforce_in"]              = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : $is_now['tls_enforce_in'];
@@ -3988,6 +4023,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $stmt->execute(array(':goto' => '(^|,)'.preg_quote($_data, '/').'($|,)'));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($row = array_shift($rows)) {
+            if (!hasAliasObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $row['address'])) {
+              continue;
+            }
             // Keep old array for backward compatibility
             $data['fixed_sender_aliases'][] = $row['address'];
             // Split into allowed/blocked for proper display
@@ -4682,6 +4720,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               `relay_unknown_only`,
               `backupmx`,
               `gal`,
+              `archive`,
               `active`
                 FROM `domain` WHERE `domain`= :domain");
           $stmt->execute(array(
@@ -4742,6 +4781,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $domaindata['backupmx_int'] = $row['backupmx'];
           $domaindata['gal'] = $row['gal'];
           $domaindata['gal_int'] = $row['gal'];
+          $domaindata['archive'] = $row['archive'];
           $domaindata['rl'] = $rl;
           $domaindata['active'] = $row['active'];
           $domaindata['active_int'] = $row['active'];
@@ -4829,6 +4869,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           if (preg_match('/y|yes/i', getenv('MASTER'))) {
             $stmt = $pdo->prepare("SELECT
               `domain`.`backupmx`,
+              `domain`.`active` AS `domain_active`,
+              `domain`.`archive` AS `domain_archive`,
               `mailbox`.`username`,
               `mailbox`.`name`,
               `mailbox`.`active`,
@@ -4851,6 +4893,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           else {
             $stmt = $pdo->prepare("SELECT
               `domain`.`backupmx`,
+              `domain`.`active` AS `domain_active`,
+              `domain`.`archive` AS `domain_archive`,
               `mailbox`.`username`,
               `mailbox`.`name`,
               `mailbox`.`active`,
@@ -4890,6 +4934,18 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $mailboxdata['created'] = $row['created'];
           $mailboxdata['modified'] = $row['modified'];
           $mailboxdata['authsource'] = ($row['authsource']) ? $row['authsource'] : 'mailcow';
+          // Mails of the mailbox go to the archive: global switch, mailbox setting ("inherit" follows the domain), receives mail
+          static $archive_enabled = null;
+          if ($archive_enabled === null) {
+            try {
+              $archive_enabled = \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled();
+            }
+            catch (PDOException $e) {
+              $archive_enabled = false;
+            }
+          }
+          $mailboxdata['archive'] = ($archive_enabled && $row['active'] != 0 && $row['domain_active'] == 1 && $row['backupmx'] == 0
+            && \Mailcow\Archive\Model\Mailbox::effective($mailboxdata['attributes']['archive'] ?? null, $row['domain_archive'])) ? 1 : 0;
 
           if ($mailboxdata['percent_in_use'] === '- ') {
             $mailboxdata['percent_class'] = "info";
@@ -4923,6 +4979,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 			      else if ($SaslLogs['service'] == 'SSO') {
               $last_sso_login = strtotime($SaslLogs['datetime']);
             }
+            else if ($SaslLogs['service'] == 'SOGO') {
+              $last_sogo_login = strtotime($SaslLogs['datetime']);
+            }
           }
           if (!isset($last_imap_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
             $last_imap_login = 0;
@@ -4936,10 +4995,14 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 		      if (!isset($last_sso_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
             $last_sso_login = 0;
           }
+          if (!isset($last_sogo_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
+            $last_sogo_login = 0;
+          }
           $mailboxdata['last_imap_login'] = $last_imap_login;
           $mailboxdata['last_smtp_login'] = $last_smtp_login;
           $mailboxdata['last_pop3_login'] = $last_pop3_login;
           $mailboxdata['last_sso_login'] = $last_sso_login;
+          $mailboxdata['last_sogo_login'] = $last_sogo_login;
 
           if (!isset($_extra) || $_extra != 'reduced') {
             $rl = ratelimit('get', 'mailbox', $_data);
@@ -4977,7 +5040,16 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(':username' => $_data));
             $sender_acl_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
             foreach ($sender_acl_rows as $sender_acl_row) {
-              $mailboxdata['sender_acl'][] = $sender_acl_row['send_as'];
+              $send_as = $sender_acl_row['send_as'];
+              if ($_SESSION['mailcow_cc_role'] == 'domainadmin' && $send_as != '*') {
+                $in_scope = (substr($send_as, 0, 1) == '@')
+                  ? hasDomainAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], substr($send_as, 1))
+                  : hasAliasObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $send_as);
+                if (!$in_scope) {
+                  continue;
+                }
+              }
+              $mailboxdata['sender_acl'][] = $send_as;
             }
           }
           $stmt = $pdo->prepare("SELECT `tag_name`
@@ -5348,6 +5420,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ':domain' => $domain,
             ));
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
+            \Mailcow\Retention\RetentionModule::fromGlobals()->api->domainDeleted($domain);
             $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,
@@ -5916,6 +5989,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':domain' => $domain,
                 ':tag_name' => $tag,
               ));
+              if ($stmt->rowCount() > 0) {
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
+              }
             }
           }
 
@@ -5968,6 +6047,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':username' => $username,
                 ':tag_name' => $tag,
               ));
+              if ($stmt->rowCount() > 0) {
+                $stmt = $pdo->prepare("UPDATE `mailbox` SET `modified` = NOW() WHERE `username` = :username");
+                $stmt->execute(array(
+                  ':username' => $username,
+                ));
+              }
             }
           }
 

data/web/inc/functions.quarantine.inc.php

@@ -154,6 +154,7 @@ function quarantine($_action, $_data = null) {
             $mail->addAttachment($msg_tmpf, $detail_row['qid'] . '.eml');
             $mail->Body = sprintf($lang['quarantine']['release_body']);
             $mail->send();
+            $released_msg = $mail->getSentMIMEMessage();
             unlink($msg_tmpf);
           }
           catch (phpmailerException $e) {
@@ -170,6 +171,7 @@ function quarantine($_action, $_data = null) {
         }
         elseif ($release_format == 'raw') {
           $detail_row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $detail_row['msg']);
+          $released_msg = $detail_row['msg'];
           // dot-stuffing per RFC5321 4.5.2: escape leading dots
           $detail_row['msg'] = preg_replace('~^\.~m', '..', $detail_row['msg']);
           $postfix_talk = array(
@@ -213,6 +215,10 @@ function quarantine($_action, $_data = null) {
           }
           fclose($smtp_connection);
         }
+        // Port 590 bypasses rspamd, so the archive gets released mails here
+        if (isset($released_msg)) {
+          \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($detail_row['rcpt']), 'quarantine');
+        }
         $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE id = :id");
         $stmt->execute(array(
           ':id' => $row['id']
@@ -453,6 +459,7 @@ function quarantine($_action, $_data = null) {
               $mail->addAttachment($msg_tmpf, $row['qid'] . '.eml');
               $mail->Body = sprintf($lang['quarantine']['release_body']);
               $mail->send();
+              $released_msg = $mail->getSentMIMEMessage();
               unlink($msg_tmpf);
             }
             catch (phpmailerException $e) {
@@ -467,6 +474,7 @@ function quarantine($_action, $_data = null) {
           }
           elseif ($release_format == 'raw') {
             $row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $row['msg']);
+            $released_msg = $row['msg'];
             // dot-stuffing per RFC5321 4.5.2: escape leading dots
             $row['msg'] = preg_replace('~^\.~m', '..', $row['msg']);
             $postfix_talk = array(
@@ -506,6 +514,11 @@ function quarantine($_action, $_data = null) {
             }
             fclose($smtp_connection);
           }
+          // Port 590 bypasses rspamd, so the archive gets released mails here
+          if (isset($released_msg)) {
+            \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($row['rcpt']), 'quarantine');
+            unset($released_msg);
+          }
           $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE `id` = :id");
           $stmt->execute(array(
             ':id' => $id
@@ -767,8 +780,9 @@ function quarantine($_action, $_data = null) {
       else {
         $domains = array_merge(mailbox('get', 'domains'), mailbox('get', 'alias_domains'));
         foreach ($domains as $domain) {
-          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE `rcpt` REGEXP :domain');
-          $stmt->execute(array(':domain' => '@' . $domain . '$'));
+          // Compare the domain part exactly, a regex would treat dots in the domain as wildcards
+          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE SUBSTRING_INDEX(`rcpt`, "@", -1) = :domain');
+          $stmt->execute(array(':domain' => $domain));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while($row = array_shift($rows)) {
             $q_meta[] = $row;

data/web/inc/init_db.inc.php

@@ -242,6 +242,7 @@ function init_db_schema()
           "gal" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "relay_all_recipients" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "relay_unknown_only" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
           "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
           "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
@@ -491,6 +492,257 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
+      // Mail policies: whom the assigned mailboxes and groups may send to and receive from (Mailcow\MailPolicy)
+      "mail_policy" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "name" => "VARCHAR(100) NOT NULL",
+          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          // all, internal or list; the entries are JSON lists of addresses, domains and group:<id>
+          "send_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
+          "send_allow" => "MEDIUMTEXT NOT NULL",
+          "send_block" => "MEDIUMTEXT NOT NULL",
+          "receive_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
+          "receive_allow" => "MEDIUMTEXT NOT NULL",
+          "receive_block" => "MEDIUMTEXT NOT NULL",
+          "reject_text" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "domain_name" => array("domain", "name")
+          ),
+          "fkey" => array(
+            "fk_mail_policy_domain" => array(
+              "col" => "domain",
+              "ref" => "domain.domain",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // a mailbox or a group (its members and its address) a policy is assigned to
+      "mail_policy_member" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "policy_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NULL DEFAULT NULL",
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "policy_mailbox" => array("policy_id", "username"),
+            "policy_group" => array("policy_id", "group_id")
+          ),
+          "key" => array(
+            "username" => array("username"),
+            "group_id" => array("group_id")
+          ),
+          "fkey" => array(
+            "fk_mail_policy_member_policy" => array(
+              "col" => "policy_id",
+              "ref" => "mail_policy.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mail_policy_member_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mail_policy_member_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "retention_rule" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "name" => "VARCHAR(100) NOT NULL",
+          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          // JSON list: trash, junk, sent, drafts, archive, inbox, all, folder:<name>
+          "folders" => "TEXT NOT NULL",
+          "days" => "INT NOT NULL DEFAULT '30'",
+          "keep_flagged" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // simulate (count only) or delete
+          "mode" => "VARCHAR(16) NOT NULL DEFAULT 'simulate'",
+          "all_mailboxes" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "domain_name" => array("domain", "name")
+          ),
+          "fkey" => array(
+            "fk_retention_rule_domain" => array(
+              "col" => "domain",
+              "ref" => "domain.domain",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // a mailbox or a group (its members of the rule's domain) a retention rule is assigned to
+      "retention_rule_member" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "rule_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NULL DEFAULT NULL",
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "rule_mailbox" => array("rule_id", "username"),
+            "rule_group" => array("rule_id", "group_id")
+          ),
+          "key" => array(
+            "username" => array("username"),
+            "group_id" => array("group_id")
+          ),
+          "fkey" => array(
+            "fk_retention_rule_member_rule" => array(
+              "col" => "rule_id",
+              "ref" => "retention_rule.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_retention_rule_member_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_retention_rule_member_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // results of the nightly run: per mailbox and rule with found mail or an error (folder: the folder names), per
+      // rule a summary (username '') and the whole run (username '', rule_id NULL); kept 90 days, of simulating
+      // rules only the last run
+      "retention_log" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "rule_id" => "INT NULL DEFAULT NULL",
+          "rule_name" => "VARCHAR(100) NOT NULL DEFAULT ''",
+          "folder" => "TEXT NULL DEFAULT NULL",
+          "days" => "INT NOT NULL DEFAULT '0'",
+          "mailboxes" => "INT NOT NULL DEFAULT '0'",
+          "matched" => "INT NOT NULL DEFAULT '0'",
+          "deleted" => "INT NOT NULL DEFAULT '0'",
+          "errors" => "INT NOT NULL DEFAULT '0'",
+          "simulated" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "error" => "TEXT NULL DEFAULT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "domain_created" => array("domain", "created"),
+            "rule_created" => array("rule_id", "username", "created"),
+            "created" => array("created")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_setting" => array(
+        "cols" => array(
+          "name" => "VARCHAR(64) NOT NULL",
+          "value" => "TEXT NOT NULL",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("name")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_job" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "type" => "ENUM('backup', 'check', 'prune', 'init', 'restore') NOT NULL",
+          "state" => "ENUM('queued', 'running', 'done', 'failed', 'cancelled') NOT NULL DEFAULT 'queued'",
+          "step" => "VARCHAR(32) NULL DEFAULT NULL",
+          "params" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "result" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "log" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "error" => "TEXT NULL DEFAULT NULL",
+          "domain" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "started" => "DATETIME NULL DEFAULT NULL",
+          "finished" => "DATETIME NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "state_type" => array("state", "type"),
+            "domain_created" => array("domain", "created")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_snapshot" => array(
+        "cols" => array(
+          "id" => "VARCHAR(64) NOT NULL",
+          "short_id" => "VARCHAR(16) NOT NULL",
+          "time" => "DATETIME NOT NULL",
+          "hostname" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "tags" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "summary" => "TEXT NULL DEFAULT NULL",
+          "domains" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "content" => "MEDIUMBLOB NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "time" => array("time")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
       "mailbox_group_imap_grant" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -703,7 +955,8 @@ function init_db_schema()
             "group_id" => array("group_id")
           ),
           "key" => array(
-            "domain" => array("domain")
+            "domain" => array("domain"),
+            "internal" => array("internal", "active")
           ),
           "fkey" => array(
             "fk_alias_mailbox_group" => array(
@@ -1053,7 +1306,10 @@ function init_db_schema()
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "mail_policies" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "retention" => "TINYINT(1) NOT NULL DEFAULT '0'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1098,6 +1354,7 @@ function init_db_schema()
           "host1" => "VARCHAR(255) NOT NULL",
           "port1" => "SMALLINT UNSIGNED NOT NULL",
           "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'",
+          "verify_cert" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "auth_type" => "ENUM('PLAIN','LOGIN','CRAM-MD5','XOAUTH2') DEFAULT 'PLAIN'",
           "oauth_flow" => "ENUM('client_credentials','authorization_code') NOT NULL DEFAULT 'client_credentials'",
           "oauth_token_endpoint" => "VARCHAR(500) DEFAULT NULL",
@@ -1276,6 +1533,42 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
+      "archive_settings" => array(
+        "cols" => array(
+          "name" => "VARCHAR(64) NOT NULL",
+          "value" => "TEXT NOT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("name")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // Archive target the Postfix maps read (FILTER of the copies on port 591, SASL, TLS policy), one row kept in line with archive_settings
+      "archive_transport" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "address" => "VARCHAR(255) NOT NULL",
+          "nexthop" => "VARCHAR(255) NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "password" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "tls_policy" => "VARCHAR(64) NOT NULL DEFAULT 'encrypt'",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "address" => array("address"),
+            "nexthop" => array("nexthop")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
       "bcc_maps" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -1679,9 +1972,15 @@ function init_db_schema()
             // Converted JSON may not fit into the old VARCHAR(512)
             $pdo->exec("ALTER TABLE `imapsync` MODIFY COLUMN `custom_params` TEXT");
             $imapsync_allow = \Mailcow\Imapsync\Model\CustomParams::allowlist($GLOBALS['IMAPSYNC_OPTIONS'] ?? [], $GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? []);
-            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
-            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `auth_type`, `active`)
-              VALUES (:name, '', '', 'domain', :host, :port, :enc, :auth, 1)");
+            // A legacy source is found by its connection and its only target domain (none for jobs without domain)
+            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` s WHERE `created_by` = '' AND `host1` = :host AND `port1` = :port
+              AND `enc1` = :enc AND `auth_type` = :auth
+              AND (SELECT GROUP_CONCAT(`domain`) FROM `imapsync_source_domain` d WHERE d.`source_id` = s.`id`) <=> NULLIF(:domain, '')
+              ORDER BY `id` LIMIT 1");
+            $source_name_sel = $pdo->prepare("SELECT COUNT(*) FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
+            // The old runner never checked certificates, legacy sources keep that
+            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `verify_cert`, `auth_type`, `active`)
+              VALUES (:name, 'Migrated sync jobs', '', 'domain', :host, :port, :enc, 0, :auth, 1)");
             $source_domain_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)");
             $job_upd = $pdo->prepare("UPDATE `imapsync` SET `source_id` = :source_id, `custom_params` = :cp, `custom_params_legacy` = :cp_legacy,
               `dry` = GREATEST(`dry`, :dry) WHERE `id` = :id AND `source_id` IS NULL");
@@ -1741,16 +2040,20 @@ function init_db_schema()
               $enc = in_array($row['enc1'], array('TLS', 'SSL', 'PLAIN'), true) ? $row['enc1'] : 'TLS';
               $at = strrchr((string)$row['user2'], '@');
               $domain = ($at === false) ? '' : strtolower(substr($at, 1));
-              // One source per (host, port, enc, auth, domain), name always unique and <= 100 chars
-              $name = 'legacy-' . mb_substr($host, 0, 60) . '-' . $port . '-' . substr(sha1($host . '|' . $port . '|' . $enc . '|' . $auth . '|' . $domain), 0, 8);
-              $source_sel->execute(array(':name' => $name));
+              // One source per (host, port, enc, auth, domain), named "host (domain)" plus " 2", " 3"... when taken (<= 100 chars)
+              $conn = array(':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth);
+              $source_sel->execute($conn + array(':domain' => $domain));
               if (($source_id = $source_sel->fetchColumn()) === false) {
-                $source_ins->execute(array(':name' => $name, ':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth));
-                $source_sel->execute(array(':name' => $name));
-                if (($source_id = $source_sel->fetchColumn()) === false) continue;
-              }
-              if ($domain !== '') {
-                $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
+                $base = mb_substr($host . ($domain !== '' ? ' (' . $domain . ')' : ''), 0, 94);
+                for ($i = 1, $name = $base; $i < 1000; $name = $base . ' ' . ++$i) {
+                  $source_name_sel->execute(array(':name' => $name));
+                  if ($source_name_sel->fetchColumn() == 0) break;
+                }
+                $source_ins->execute($conn + array(':name' => $name));
+                if (($source_id = $pdo->lastInsertId()) == 0) continue;
+                if ($domain !== '') {
+                  $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
+                }
               }
               $job_upd->execute(array(
                 ':source_id' => $source_id,
@@ -1998,6 +2301,7 @@ function init_db_schema()
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.relayhost', \"0\") WHERE JSON_VALUE(`attributes`, '$.relayhost') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_pw_update', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_pw_update') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
+    $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.archive', \"inherit\") WHERE JSON_VALUE(`attributes`, '$.archive') IS NULL;");
     // admin attributes
     $pdo->query("UPDATE `admin` SET `attributes` = '{}' WHERE `attributes` = '' OR `attributes` IS NULL;");
     $pdo->query("UPDATE `admin` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
@@ -2045,6 +2349,7 @@ function init_db_schema()
         "backupmx" => 0,
         "relay_all_recipients" => 0,
         "relay_unknown_only" => 0,
+        "archive" => 0,
         "dkim_selector" => "dkim",
         "key_size" => 2048,
         "max_quota_for_domain" => 10240 * 1048576,
@@ -2062,6 +2367,7 @@ function init_db_schema()
         "rl_value" => "",
         "force_pw_update" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_pw_update']),
         "force_tfa" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_tfa']),
+        "archive" => strval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['archive']),
         "sogo_access" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['sogo_access']),
         "active" => 1,
         "tls_enforce_in" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['tls_enforce_in']),

data/web/inc/lib/mailcow/Archive/ArchiveModule.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive;
+
+use Mailcow\Archive\Controller\ArchiveApiController;
+use Mailcow\Archive\Legacy\LegacyQueue;
+use Mailcow\Archive\Redis\ArchiveState;
+use Mailcow\Archive\Repository\LookupRepository;
+use Mailcow\Archive\Repository\SettingsRepository;
+use Mailcow\Archive\Service\HeaderBuilder;
+use Mailcow\Archive\Service\JournalService;
+use Mailcow\Archive\Service\MailboxListService;
+use Mailcow\Archive\Service\OwnerResolver;
+use Mailcow\Archive\Service\SettingsService;
+use Mailcow\Archive\Service\StatusService;
+use Mailcow\Archive\Smtp\SmtpClient;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use PDO;
+
+/**
+ * Composition root of the archive module (archive copies by SMTP through Postfix).
+ *
+ *   process_get_return(ArchiveModule::fromGlobals()->api->getMailboxes());
+ *   ArchiveModule::create($pdo, Context::system(), $redis)->journal->prepare($request);  // rspamd dynmap
+ *   ArchiveModule::fromGlobals()->journal->submitDelivered($message, $from, $rcpts, 'quarantine');
+ */
+class ArchiveModule
+{
+    /** Request keys whose values never reach a log */
+    private const SECRET_KEYS = ['password', 'signing_secret'];
+
+    public function __construct(
+        public readonly SettingsService $settings,
+        public readonly JournalService $journal,
+        public readonly StatusService $status,
+        public readonly MailboxListService $mailboxes,
+        public readonly ArchiveApiController $api
+    ) {
+    }
+
+    /** @param ?object $redis Redis connection; without it nothing is read from or written to Redis */
+    public static function create(PDO $pdo, Context $context, ?object $redis = null): self
+    {
+        $settingsRepo = new SettingsRepository($pdo);
+        $lookup = new LookupRepository($pdo);
+        $state = new ArchiveState($redis);
+        $smtp = new SmtpClient();
+        $settings = new SettingsService($settingsRepo, $lookup, $state, $smtp, $context);
+        $journal = new JournalService($settingsRepo, new OwnerResolver($lookup), new HeaderBuilder(), $smtp, $state, $context);
+        $status = new StatusService($settingsRepo, $state, new LegacyQueue(), $context);
+        $mailboxes = new MailboxListService($lookup, $state, $context);
+        $api = new ArchiveApiController($settings, $status, $journal, $mailboxes, new ApiResponse(self::SECRET_KEYS));
+        return new self($settings, $journal, $status, $mailboxes, $api);
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo, $redis and $_SESSION; $context overrides the session caller */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        $redis = $GLOBALS['redis'] ?? null;
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), is_object($redis) ? $redis : null);
+    }
+}

data/web/inc/lib/mailcow/Archive/Controller/ArchiveApiController.php

@@ -0,0 +1,103 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Controller;
+
+use Mailcow\Archive\Service\JournalService;
+use Mailcow\Archive\Service\MailboxListService;
+use Mailcow\Archive\Service\SettingsService;
+use Mailcow\Archive\Service\StatusService;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php (routes `archive/*`, `archive`).
+ *
+ * Write methods return what json_api passes to process_*_return() and write their messages via ApiResponse.
+ * Messages are logged as ['archive', action, object, data].
+ */
+class ArchiveApiController
+{
+    public function __construct(private SettingsService $settings, private StatusService $status, private JournalService $journal,
+        private MailboxListService $mailboxes, private ApiResponse $response)
+    {
+    }
+
+    /** GET archive/settings */
+    public function getSettings(): array
+    {
+        return $this->read(fn(): array => $this->settings->get()->toArray());
+    }
+
+    /** GET archive/status */
+    public function getStatus(): array
+    {
+        return $this->read(fn(): array => $this->status->status());
+    }
+
+    /** GET logs/archive[/count] */
+    public function getLog(int $count = 0): array
+    {
+        return $this->read(fn(): array => $this->status->log($count));
+    }
+
+    /** GET archive/mailboxes */
+    public function getMailboxes(): array
+    {
+        return $this->read(fn(): array => $this->mailboxes->archived());
+    }
+
+    /** POST edit/archive: the settings; `action: test` tests the connection, `action: testmail` sends a test copy */
+    public function edit(array $attr): bool
+    {
+        return match ($attr['action'] ?? null) {
+            'test' => $this->handle('edit', 'test', [], fn(array $log): bool => $this->testResult($this->settings->test(), $log)),
+            'testmail' => $this->handle('edit', 'testmail', [],
+                fn(array $log): bool => $this->response->success(['archive_testmail_queued', htmlspecialchars($this->journal->testMail())], $log)),
+            default => $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
+                $this->settings->save($attr);
+                return $this->response->success('archive_settings_saved', $log);
+            }),
+        };
+    }
+
+    /** A missing extension is a warning: the archive works, but may refuse some mails */
+    private function testResult(string|array $msg, array $log): bool
+    {
+        if (is_array($msg)) {
+            $this->response->warning($msg, $log);
+            return true;
+        }
+        return $this->response->success($msg, $log);
+    }
+
+    /** A GET answer, empty when the caller may not read it */
+    private function read(callable $work): array
+    {
+        try {
+            return $work();
+        } catch (UserError) {
+            return [];
+        }
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     */
+    private function handle(string $action, string $object, mixed $data, callable $work): bool
+    {
+        $log = ['archive', $action, $object, $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+}

data/web/inc/lib/mailcow/Archive/Legacy/LegacyQueue.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Legacy;
+
+/**
+ * Bridge to the legacy mailq() (Postfix queue through dockerapi) until it is converted.
+ */
+class LegacyQueue
+{
+    /**
+     * @return array[] queue entries of $sender: queue_id, queue_name, arrival_time, message_size,
+     *                 recipients ("address (delay reason)")
+     */
+    public function entries(string $sender): array
+    {
+        if (!function_exists('mailq')) {
+            return [];
+        }
+        $list = json_decode((string)mailq('get'), true);
+        if (!is_array($list)) {
+            return [];
+        }
+        return array_values(array_filter($list, fn(mixed $entry): bool => is_array($entry) && ($entry['sender'] ?? '') === $sender));
+    }
+}

data/web/inc/lib/mailcow/Archive/Model/Journal.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Model;
+
+/**
+ * What the archive learns about one mail besides its content: owners, SMTP envelope, origin and content hash.
+ */
+class Journal
+{
+    public const DIRECTIONS = ['in', 'out', 'internal'];
+    public const SOURCES = ['smtp', 'quarantine', 'massmail', 'test'];
+
+    /**
+     * @param string[] $owners mailbox usernames the mail belongs to
+     * @param string[] $rcpts SMTP envelope recipients as received (incl. BCC)
+     * @param float $received time mailcow accepted the mail (unix time with milliseconds)
+     * @param string $sha256 hash of the content as sent to the archive (CRLF line ends)
+     */
+    public function __construct(
+        public readonly array $owners,
+        public readonly array $rcpts,
+        public readonly string $from,
+        public readonly string $qid,
+        public readonly string $direction,
+        public readonly string $source,
+        public readonly float $received,
+        public readonly string $sha256
+    ) {
+    }
+}

data/web/inc/lib/mailcow/Archive/Model/Mailbox.php

@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Model;
+
+/**
+ * A mailbox that receives mail (regular kind, active 1 or 2, in an active domain that is no backup MX)
+ * with its archive settings: the mailbox attribute `archive` ('1', '0' or 'inherit') and the domain default.
+ */
+class Mailbox
+{
+    public const INHERIT = 'inherit';
+
+    public function __construct(
+        public readonly string $username,
+        public readonly string $domain,
+        public readonly string $archive,
+        public readonly bool $domainArchive,
+        public readonly string $name = '',
+        public readonly int $active = 1
+    ) {
+    }
+
+    public static function fromRow(array $row): self
+    {
+        return new self(
+            strtolower((string)$row['username']),
+            strtolower((string)$row['domain']),
+            self::setting($row['archive'] ?? null),
+            (int)($row['domain_archive'] ?? 0) === 1,
+            (string)($row['name'] ?? ''),
+            (int)($row['active'] ?? 1)
+        );
+    }
+
+    /** Mailbox setting: '1', '0' (also as number or boolean), anything else (also a missing attribute) is 'inherit' */
+    public static function setting(mixed $value): string
+    {
+        $value = is_bool($value) ? (int)$value : $value;
+        $value = is_scalar($value) ? (string)$value : '';
+        return in_array($value, ['0', '1'], true) ? $value : self::INHERIT;
+    }
+
+    /** Archiving applies by the mailbox setting, 'inherit' follows the domain (without the global switch) */
+    public static function effective(mixed $archive, mixed $domainArchive): bool
+    {
+        $archive = self::setting($archive);
+        return ($archive === self::INHERIT) ? (int)$domainArchive === 1 : $archive === '1';
+    }
+
+    public function archived(): bool
+    {
+        return self::effective($this->archive, $this->domainArchive);
+    }
+
+    /** Why the mailbox is archived: its own setting or the domain default */
+    public function source(): string
+    {
+        return ($this->archive === '1') ? 'mailbox' : 'domain';
+    }
+
+    /** Entry of GET archive/mailboxes */
+    public function toArray(): array
+    {
+        return [
+            'username' => $this->username,
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'active' => $this->active,
+            'source' => $this->source(),
+        ];
+    }
+}

data/web/inc/lib/mailcow/Archive/Model/Settings.php

@@ -0,0 +1,72 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Model;
+
+/**
+ * The archive target (table `archive_settings`): archive copies of accepted mails go by SMTP to `address`,
+ * through the Postfix queue (table `archive_transport`, service archive_smtp in master.cf).
+ */
+class Settings
+{
+    public const TLS_MODES = ['encrypt', 'verify', 'may'];
+    /** Metadata headers of a copy: archivecow (X-Archive-*, signed), envelope (sender/recipients), none */
+    public const PROFILES = ['archivecow', 'envelope', 'none'];
+    /** Envelope sender of copies; Postfix drops every notice to it (archive_discard), nothing reaches a real sender */
+    public const SENDER = 'archive-bounce@localhost';
+
+    /** rspamd hands accepted mails to the archive */
+    public bool $enabled = false;
+    /** Address the archive receives copies at */
+    public string $address = '';
+    /** SMTP server of the archive; empty: MX of the address domain */
+    public string $host = '';
+    public int $port = 25;
+    public string $tls = 'encrypt';
+    /** SMTP AUTH at the archive, optional */
+    public string $username = '';
+    /** Never part of toArray() */
+    public string $password = '';
+    public string $profile = 'archivecow';
+    /** Key of X-Archive-Signature (profile archivecow), never part of toArray() */
+    public string $signingKey = '';
+    /** Header names of the envelope profile */
+    public string $headerFrom = 'X-Envelope-From';
+    public string $headerRcpt = 'X-Envelope-To';
+
+    /** Copies can be sent: an address, and a signing key for archivecow */
+    public function canDeliver(): bool
+    {
+        return $this->address !== '' && ($this->profile !== 'archivecow' || $this->signingKey !== '');
+    }
+
+    /** Next hop for Postfix: [host]:port, else the domain of the address (MX lookup), with the port when not 25 */
+    public function nexthop(): string
+    {
+        if ($this->host !== '') {
+            $host = filter_var($this->host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6:' . $this->host : $this->host;
+            return '[' . $host . ']:' . $this->port;
+        }
+        $domain = substr($this->address, (int)strrpos($this->address, '@') + 1);
+        return ($this->port === 25) ? $domain : $domain . ':' . $this->port;
+    }
+
+    /** API representation (GET archive/settings), without secrets */
+    public function toArray(): array
+    {
+        return [
+            'enabled' => (int)$this->enabled,
+            'address' => $this->address,
+            'host' => $this->host,
+            'port' => $this->port,
+            'tls' => $this->tls,
+            'username' => $this->username,
+            'has_password' => (int)($this->password !== ''),
+            'profile' => $this->profile,
+            'has_signing_secret' => (int)($this->signingKey !== ''),
+            'header_from' => $this->headerFrom,
+            'header_rcpt' => $this->headerRcpt,
+        ];
+    }
+}

data/web/inc/lib/mailcow/Archive/Redis/ArchiveState.php

@@ -0,0 +1,118 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Redis;
+
+use Mailcow\Archive\Model\Settings;
+use RuntimeException;
+
+/**
+ * Redis side of the archive:
+ * - ARCHIVE_ENABLED: the global switch, kept here only; rspamd reads it before it asks for owners.
+ * - ARCHIVE_DROPPED / ARCHIVE_DROPPED_LOG: copies that could not be handed to Postfix (rspamd.local.lua, release
+ *   hooks), counted and the last 100 listed.
+ * - POSTFIX_MAILLOG: the Postfix log, read for the archive log: deliveries of the archive_smtp service and expired
+ *   copies, with the Message-ID of the mail (a copy keeps the Message-ID of the original).
+ *
+ * The connection is a plain object, so the class loads without the redis extension; without a connection
+ * nothing is read or written.
+ */
+class ArchiveState
+{
+    public const ENABLED = 'ARCHIVE_ENABLED';
+    public const DROPPED = 'ARCHIVE_DROPPED';
+    public const DROPPED_LOG = 'ARCHIVE_DROPPED_LOG';
+    private const MAILLOG = 'POSTFIX_MAILLOG';
+
+    public function __construct(private ?object $redis)
+    {
+    }
+
+    public function setEnabled(bool $enabled): void
+    {
+        $this->redis?->set(self::ENABLED, $enabled ? '1' : '0');
+    }
+
+    /** The global switch: on only while ARCHIVE_ENABLED is '1', as rspamd reads it (missing or unreadable is off) */
+    public function enabled(): bool
+    {
+        try {
+            return $this->redis?->get(self::ENABLED) === '1';
+        } catch (RuntimeException) {
+            return false;
+        }
+    }
+
+    /** A mail was not archived: its copy could not be handed to Postfix */
+    public function dropped(string $qid, string $reason, string $messageId = ''): void
+    {
+        if ($this->redis === null) {
+            return;
+        }
+        $this->redis->incr(self::DROPPED);
+        $this->redis->lPush(self::DROPPED_LOG, json_encode(['time' => time(), 'qid' => $qid, 'reason' => $reason, 'message_id' => $messageId]));
+        $this->redis->lTrim(self::DROPPED_LOG, 0, 99);
+    }
+
+    /** @return array{count: int, recent: array[]} count since ever; recent: time, qid, reason, message_id; newest first */
+    public function droppedStats(int $limit = 100): array
+    {
+        if ($this->redis === null) {
+            return ['count' => 0, 'recent' => []];
+        }
+        $recent = [];
+        foreach ((array)$this->redis->lRange(self::DROPPED_LOG, 0, $limit - 1) as $line) {
+            $entry = is_string($line) ? json_decode($line, true) : null;
+            if (is_array($entry)) {
+                $recent[] = ['time' => (int)($entry['time'] ?? 0), 'qid' => (string)($entry['qid'] ?? ''),
+                    'reason' => (string)($entry['reason'] ?? ''), 'message_id' => (string)($entry['message_id'] ?? '')];
+            }
+        }
+        return ['count' => (int)$this->redis->get(self::DROPPED), 'recent' => $recent];
+    }
+
+    /**
+     * Archive log, newest first: deliveries of copies (status sent|deferred|softbounce|bounced, text = answer of the
+     * archive), expired copies (expired) and mails not archived (dropped, text = reason).
+     *
+     * @return array[] time, status, qid (of the copy, of the mail for dropped), message_id, text
+     */
+    public function events(int $limit = 1000): array
+    {
+        if ($this->redis === null) {
+            return [];
+        }
+        $events = [];
+        $messageIds = [];
+        foreach ((array)$this->redis->lRange(self::MAILLOG, 0, -1) as $line) {
+            $entry = is_string($line) ? json_decode($line, true) : null;
+            if (!is_array($entry)) {
+                continue;
+            }
+            $program = (string)($entry['program'] ?? '');
+            $message = (string)($entry['message'] ?? '');
+            $time = (int)($entry['time'] ?? 0);
+            if ($program === 'postfix/cleanup' && preg_match('/^([0-9A-Za-z]+): message-id=(\S*)/', $message, $m)) {
+                $messageIds[$m[1]] = trim($m[2], '<>');
+            }
+            // the archive_discard lines are the dropped notices, not deliveries
+            elseif ($program === 'postfix/archive/smtp' && preg_match('/^([0-9A-Za-z]+): to=<[^>]*>.*\bstatus=(\w+) \((.*)\)$/', $message, $m)) {
+                $events[] = ['time' => $time, 'status' => strtolower($m[2]), 'qid' => $m[1], 'message_id' => '', 'text' => $m[3]];
+            }
+            elseif ($program === 'postfix/qmgr' && preg_match('/^([0-9A-Za-z]+): from=<' . preg_quote(Settings::SENDER, '/') . '>, status=expired/', $message, $m)) {
+                $events[] = ['time' => $time, 'status' => 'expired', 'qid' => $m[1], 'message_id' => '', 'text' => ''];
+            }
+        }
+        foreach ($events as &$event) {
+            $event['message_id'] = $messageIds[$event['qid']] ?? '';
+        }
+        unset($event);
+        foreach ($this->droppedStats()['recent'] as $dropped) {
+            $events[] = ['time' => $dropped['time'], 'status' => 'dropped', 'qid' => $dropped['qid'],
+                'message_id' => $dropped['message_id'], 'text' => $dropped['reason']];
+        }
+        usort($events, fn(array $a, array $b): int => $b['time'] <=> $a['time']);
+        return array_slice($events, 0, $limit);
+    }
+}

data/web/inc/lib/mailcow/Archive/Repository/LookupRepository.php

@@ -0,0 +1,94 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Repository;
+
+use Mailcow\Archive\Model\Mailbox;
+use Mailcow\Core\Repository;
+
+/**
+ * Read-only lookups of the address tables, with the same conditions as the Postfix maps
+ * (data/Dockerfiles/postfix/postfix.sh), and the mailboxes with their archive settings.
+ */
+class LookupRepository extends Repository
+{
+    /** Columns of a mailbox that receives mail; the domain must be active and no backup MX */
+    private const MAILBOX = "SELECT m.`username`, m.`domain`, m.`name`, m.`active`,
+        JSON_VALUE(m.`attributes`, '$.archive') AS `archive`, d.`archive` AS `domain_archive`
+      FROM `mailbox` m
+      JOIN `domain` d ON d.`domain` = m.`domain`
+      WHERE m.`kind` = '' AND (m.`active` = '1' OR m.`active` = '2') AND d.`active` = '1' AND d.`backupmx` = '0'";
+
+    /** recipient_canonical_maps: new destination of an address or '@domain' */
+    public function recipientMap(string $key): ?string
+    {
+        return $this->value("SELECT `new_dest` FROM `recipient_maps` WHERE `old_dest` = :key AND `active` = '1' LIMIT 1", $key);
+    }
+
+    /** Targets of an alias row (aliases, catch-alls '@domain', groups, mailbox self rows) */
+    public function aliasGoto(string $key): ?string
+    {
+        return $this->value("SELECT `goto` FROM `alias` WHERE `address` = :key AND (`active` = '1' OR `active` = '2') LIMIT 1", $key);
+    }
+
+    /** Resources (location, thing, group) are discarded by Postfix */
+    public function isResource(string $username): bool
+    {
+        return $this->value("SELECT `username` FROM `mailbox` WHERE `username` = :key AND `kind` REGEXP 'location|thing|group' LIMIT 1", $username) !== null;
+    }
+
+    public function spamAliasGoto(string $key): ?string
+    {
+        return $this->value("SELECT `goto` FROM `spamalias` WHERE `address` = :key
+          AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0) LIMIT 1", $key);
+    }
+
+    /** Mailbox local@target behind local@domain when domain is an alias domain */
+    public function aliasDomainMailbox(string $local, string $domain): ?string
+    {
+        $username = $this->query("SELECT `mailbox`.`username` FROM `mailbox`, `alias_domain`
+          WHERE `alias_domain`.`alias_domain` = :domain AND `mailbox`.`username` = CONCAT(:local, '@', `alias_domain`.`target_domain`)
+            AND (`mailbox`.`active` = '1' OR `mailbox`.`active` = '2') AND `alias_domain`.`active` = '1' LIMIT 1",
+          ['domain' => $domain, 'local' => $local])->fetchColumn();
+        return is_string($username) ? $username : null;
+    }
+
+    /** BCC destination of an address or '@domain' as rspamd's bcc.php finds it; $type 'rcpt' or 'sender' */
+    public function bccDestination(string $key, string $type): ?string
+    {
+        $dest = $this->query("SELECT `bcc_dest` FROM `bcc_maps` WHERE `type` = :type AND `local_dest` = :key AND `active` = '1' LIMIT 1",
+          ['type' => $type, 'key' => $key])->fetchColumn();
+        return is_string($dest) ? $dest : null;
+    }
+
+    /** A domain or alias domain of this mailcow */
+    public function isMailcowDomain(string $domain): bool
+    {
+        return $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain
+          UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :alias LIMIT 1",
+          ['domain' => $domain, 'alias' => $domain])->fetchColumn() !== false;
+    }
+
+    /** The mailbox when it receives mail */
+    public function mailbox(string $username): ?Mailbox
+    {
+        $row = $this->query(self::MAILBOX . " AND m.`username` = :username", ['username' => $username])->fetch();
+        return is_array($row) ? Mailbox::fromRow($row) : null;
+    }
+
+    /** @return Mailbox[] mailboxes archived by their setting or their domain (without the global switch) */
+    public function archivedMailboxes(): array
+    {
+        $rows = $this->query(self::MAILBOX . " AND (JSON_VALUE(m.`attributes`, '$.archive') = '1'
+            OR (COALESCE(JSON_VALUE(m.`attributes`, '$.archive'), '') NOT IN ('0', '1') AND d.`archive` = '1'))
+          ORDER BY m.`username`")->fetchAll();
+        return array_map(fn(array $row): Mailbox => Mailbox::fromRow($row), $rows);
+    }
+
+    private function value(string $sql, string $key): ?string
+    {
+        $value = $this->query($sql, ['key' => $key])->fetchColumn();
+        return is_string($value) ? $value : null;
+    }
+}

data/web/inc/lib/mailcow/Archive/Repository/SettingsRepository.php

@@ -0,0 +1,71 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Repository;
+
+use Mailcow\Archive\Model\Settings;
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * SQL for the archive target: the settings (table `archive_settings`, name => value) and the row the Postfix maps
+ * read (table `archive_transport`: transport, SASL, TLS policy of the archive_smtp service).
+ */
+class SettingsRepository extends Repository
+{
+    /** Stored settings; keys that are not stored yet keep the defaults of the model */
+    public function load(): Settings
+    {
+        $rows = $this->query("SELECT `name`, `value` FROM `archive_settings`")->fetchAll(PDO::FETCH_KEY_PAIR);
+        $settings = new Settings();
+        $settings->address = (string)($rows['address'] ?? '');
+        $settings->host = (string)($rows['host'] ?? '');
+        $settings->port = (int)($rows['port'] ?? $settings->port);
+        $settings->tls = (string)($rows['tls'] ?? $settings->tls);
+        $settings->username = (string)($rows['username'] ?? '');
+        $settings->password = (string)($rows['password'] ?? '');
+        $settings->profile = (string)($rows['profile'] ?? $settings->profile);
+        $settings->signingKey = (string)($rows['signing_secret'] ?? '');
+        $settings->headerFrom = (string)($rows['header_from'] ?? $settings->headerFrom);
+        $settings->headerRcpt = (string)($rows['header_rcpt'] ?? $settings->headerRcpt);
+        return $settings;
+    }
+
+    /**
+     * Saves the settings and the transport row. The row stays while an address is set, also with archiving off,
+     * so copies still queued in Postfix are delivered.
+     */
+    public function save(Settings $settings): void
+    {
+        $values = [
+            'address' => $settings->address,
+            'host' => $settings->host,
+            'port' => (string)$settings->port,
+            'tls' => $settings->tls,
+            'username' => $settings->username,
+            'password' => $settings->password,
+            'profile' => $settings->profile,
+            'signing_secret' => $settings->signingKey,
+            'header_from' => $settings->headerFrom,
+            'header_rcpt' => $settings->headerRcpt,
+        ];
+        $sql = "INSERT INTO `archive_settings` (`name`, `value`) VALUES (:name, :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)";
+        $this->transaction(function () use ($sql, $values, $settings): void {
+            foreach ($values as $name => $value) {
+                $this->query($sql, ['name' => $name, 'value' => $value]);
+            }
+            $this->query("DELETE FROM `archive_transport`");
+            if ($settings->address !== '') {
+                $this->insertRow('archive_transport', [
+                    'address' => $settings->address,
+                    'nexthop' => $settings->nexthop(),
+                    'username' => $settings->username,
+                    'password' => $settings->password,
+                    'tls_policy' => $settings->tls,
+                    'active' => 1,
+                ]);
+            }
+        });
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/HeaderBuilder.php

@@ -0,0 +1,125 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use DateTime;
+use Mailcow\Archive\Model\Journal;
+use Mailcow\Archive\Model\Settings;
+
+/**
+ * Metadata header block in front of an archive copy, by profile:
+ *
+ * - archivecow: X-Archive-Owner, -Rcpt, -From, -Queue-ID, -Received, -Direction, -Source, -SHA256 and as the last
+ *   line X-Archive-Signature: v=1; a=hmac-sha256; s=<hex>. The signature is HMAC-SHA256 with the signing key over
+ *   the lines above it, each as "<lower case name>:<value without any whitespace>", joined by "\n".
+ *   The block ends with the signature line, everything after it is the mail as accepted (CRLF line ends).
+ * - envelope: SMTP sender and recipients under configurable names (default X-Envelope-From / X-Envelope-To).
+ * - none: no headers.
+ *
+ * Empty values are left out. Lists are folded after commas (CRLF + space).
+ */
+class HeaderBuilder
+{
+    /** Fold a list before a line would get longer */
+    private const FOLD_AT = 78;
+
+    public function build(Settings $settings, Journal $journal): string
+    {
+        return match ($settings->profile) {
+            'archivecow' => $this->archivecow($settings, $journal),
+            'envelope' => $this->envelope($settings, $journal),
+            default => '',
+        };
+    }
+
+    /** One line of the signed text */
+    public static function canonical(string $name, string $value): string
+    {
+        return strtolower($name) . ':' . preg_replace('/\s+/', '', $value);
+    }
+
+    private function archivecow(Settings $settings, Journal $journal): string
+    {
+        $values = [
+            'X-Archive-Owner' => $journal->owners,
+            'X-Archive-Rcpt' => $journal->rcpts,
+            'X-Archive-From' => $journal->from,
+            'X-Archive-Queue-ID' => $journal->qid,
+            'X-Archive-Received' => self::time($journal->received),
+            'X-Archive-Direction' => $journal->direction,
+            'X-Archive-Source' => $journal->source,
+            'X-Archive-SHA256' => $journal->sha256,
+        ];
+        $lines = [];
+        $signed = [];
+        foreach ($values as $name => $value) {
+            $value = is_array($value) ? self::cleanList($value) : self::clean($value);
+            if ($value === '' || $value === []) {
+                continue;
+            }
+            $lines[] = self::line($name, $value);
+            $signed[] = self::canonical($name, is_array($value) ? implode(',', $value) : $value);
+        }
+        $lines[] = 'X-Archive-Signature: v=1; a=hmac-sha256; s=' . hash_hmac('sha256', implode("\n", $signed), $settings->signingKey);
+        return implode("\r\n", $lines) . "\r\n";
+    }
+
+    private function envelope(Settings $settings, Journal $journal): string
+    {
+        $from = self::clean($journal->from);
+        $lines = [self::line($settings->headerFrom, ($from === '') ? '<>' : $from)];
+        $rcpts = self::cleanList($journal->rcpts);
+        if ($rcpts !== []) {
+            $lines[] = self::line($settings->headerRcpt, $rcpts);
+        }
+        return implode("\r\n", $lines) . "\r\n";
+    }
+
+    /** Header line; a list is folded after commas */
+    private static function line(string $name, string|array $value): string
+    {
+        if (is_string($value)) {
+            return $name . ': ' . $value;
+        }
+        $out = $name . ': ' . array_shift($value);
+        $length = strlen($out);
+        foreach ($value as $item) {
+            if ($length + strlen($item) + 1 > self::FOLD_AT) {
+                $out .= ",\r\n " . $item;
+                $length = strlen($item) + 1;
+            } else {
+                $out .= ',' . $item;
+                $length += strlen($item) + 1;
+            }
+        }
+        return $out;
+    }
+
+    /** Header value without control characters (no header injection) */
+    private static function clean(string $value): string
+    {
+        return trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', $value));
+    }
+
+    /** @return string[] */
+    private static function cleanList(array $values): array
+    {
+        $list = [];
+        foreach ($values as $value) {
+            $value = is_string($value) ? str_replace(',', '', self::clean($value)) : '';
+            if ($value !== '') {
+                $list[] = $value;
+            }
+        }
+        return $list;
+    }
+
+    /** RFC 3339 in UTC with milliseconds */
+    private static function time(float $time): string
+    {
+        $date = DateTime::createFromFormat('U.u', sprintf('%.3F', $time));
+        return $date ? $date->format('Y-m-d\TH:i:s.v\Z') : '';
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/JournalService.php

@@ -0,0 +1,144 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use Mailcow\Archive\Model\Journal;
+use Mailcow\Archive\Model\Settings;
+use Mailcow\Archive\Redis\ArchiveState;
+use Mailcow\Archive\Repository\SettingsRepository;
+use Mailcow\Archive\Smtp\SmtpClient;
+use Mailcow\Archive\Smtp\SmtpException;
+use Mailcow\Core\Context;
+use Mailcow\Core\UserError;
+use Throwable;
+
+/**
+ * Archive copies: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through the
+ * Postfix queue. rspamd sends the copies of mails it scanned (rspamd.local.lua) and asks prepare() whether to send
+ * one and for its header block; mails that bypass rspamd (quarantine release, mass mail) are sent here.
+ *
+ * Copies have the envelope sender Settings::SENDER, Postfix drops every notice to it: an archive copy never causes
+ * a bounce or delay warning to a real sender.
+ */
+class JournalService
+{
+    /** Postfix service that takes copies (no milter, mynetworks only) */
+    private const SUBMIT_PORT = 591;
+
+    public function __construct(
+        private SettingsRepository $settings,
+        private OwnerResolver $resolver,
+        private HeaderBuilder $headers,
+        private SmtpClient $smtp,
+        private ArchiveState $state,
+        private Context $context,
+        private string $postfixHost = 'postfix'
+    ) {
+    }
+
+    /**
+     * Archive decision for a mail rspamd accepted (dynmap archive.php), no copy while archiving is off or no
+     * archived mailbox is involved.
+     *
+     * @param array $request from, rcpt[], user, bcc, qid, received, sha256 (of the content rspamd sends)
+     * @return array{archive: bool, rcpt?: string, sender?: string, headers?: string}
+     */
+    public function prepare(array $request): array
+    {
+        $settings = $this->settings->load();
+        if (!$this->state->enabled() || !$settings->canDeliver()) {
+            return ['archive' => false];
+        }
+        $from = self::text($request['from'] ?? '');
+        $rcpts = is_array($request['rcpt'] ?? null) ? array_values(array_filter($request['rcpt'], 'is_string')) : [];
+        $resolved = $this->resolver->resolve($from, $rcpts, self::text($request['user'] ?? ''), ($request['bcc'] ?? false) === true);
+        if ($resolved['owners'] === []) {
+            return ['archive' => false];
+        }
+        $sha256 = self::text($request['sha256'] ?? '');
+        $journal = new Journal($resolved['owners'], $rcpts, $from, mb_strcut(self::text($request['qid'] ?? ''), 0, 64),
+            $resolved['direction'], 'smtp', is_numeric($request['received'] ?? null) ? (float)$request['received'] : microtime(true),
+            preg_match('/^[0-9a-f]{64}$/', $sha256) ? $sha256 : '');
+        return ['archive' => true, 'rcpt' => $settings->address, 'sender' => Settings::SENDER, 'headers' => $this->headers->build($settings, $journal)];
+    }
+
+    /**
+     * Sends the copy of a mail that reached its mailbox without rspamd (quarantine release, mass mail) for its
+     * archived recipients. Errors are logged and counted, never thrown: the mail is delivered already.
+     *
+     * @param string[] $rcpts
+     */
+    public function submitDelivered(string $message, string $from, array $rcpts, string $source): bool
+    {
+        try {
+            $settings = $this->settings->load();
+            if (!$this->state->enabled() || !$settings->canDeliver()) {
+                return false;
+            }
+            $resolved = $this->resolver->resolve($from, $rcpts, '', false);
+            if ($resolved['owners'] === []) {
+                return false;
+            }
+            $this->send($settings, $message, $resolved['owners'], array_values($rcpts), $from, 'in', $source);
+            return true;
+        } catch (Throwable $e) {
+            error_log('ARCHIVE: cannot send the archive copy (' . $source . '): ' . $e->getMessage());
+            $this->state->dropped('', $source . ': ' . $e->getMessage(), self::messageId($message));
+            return false;
+        }
+    }
+
+    /**
+     * Sends a test copy through Postfix to the archive and returns the Postfix queue id.
+     *
+     * @throws UserError
+     */
+    public function testMail(): string
+    {
+        $this->context->requireAdmin();
+        $settings = $this->settings->load();
+        if (!$settings->canDeliver()) {
+            throw new UserError('archive_incomplete');
+        }
+        $message = implode("\r\n", [
+            'From: mailcow <' . Settings::SENDER . '>',
+            'To: <' . $settings->address . '>',
+            'Subject: mailcow archive test',
+            'Date: ' . date(DATE_RFC2822),
+            'Message-ID: <' . bin2hex(random_bytes(12)) . '@archive-test.mailcow>',
+            'MIME-Version: 1.0',
+            'Content-Type: text/plain; charset=utf-8',
+            '',
+            'Test copy from mailcow (System - Configuration - Archive).',
+            '',
+        ]);
+        try {
+            return $this->send($settings, $message, [], [$settings->address], Settings::SENDER, 'in', 'test');
+        } catch (SmtpException $e) {
+            throw new UserError('archive_testmail_failed', [htmlspecialchars($e->describe())]);
+        }
+    }
+
+    /** @throws SmtpException */
+    private function send(Settings $settings, string $message, array $owners, array $rcpts, string $from, string $direction, string $source): string
+    {
+        $message = SmtpClient::crlf($message);
+        $journal = new Journal($owners, $rcpts, $from, '', $direction, $source, microtime(true), hash('sha256', $message));
+        return $this->smtp->submit($this->postfixHost, self::SUBMIT_PORT, Settings::SENDER, $settings->address,
+            $this->headers->build($settings, $journal) . $message);
+    }
+
+    /** Message-ID of a mail, for the archive log */
+    private static function messageId(string $message): string
+    {
+        $headers = explode("\n\n", str_replace("\r\n", "\n", $message), 2)[0];
+        return preg_match('/^Message-ID:\s*<?([^>\s]+)>?/mi', $headers, $m) ? $m[1] : '';
+    }
+
+    private static function text(mixed $value): string
+    {
+        return is_scalar($value) ? trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', (string)$value)) : '';
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/MailboxListService.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use Mailcow\Archive\Model\Mailbox;
+use Mailcow\Archive\Redis\ArchiveState;
+use Mailcow\Archive\Repository\LookupRepository;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+
+/**
+ * The mailboxes archived now (GET archive/mailboxes), the list archivecow keeps its mailboxes in line with.
+ * Same rule as the owners of a mail: the global switch, then the mailbox setting, 'inherit' follows the domain.
+ */
+class MailboxListService
+{
+    public function __construct(private LookupRepository $lookup, private ArchiveState $state, private Context $context)
+    {
+    }
+
+    /**
+     * Admins, read-only API keys included.
+     *
+     * @return array{enabled: int, mailboxes: array[]}
+     * @throws AccessDenied
+     */
+    public function archived(): array
+    {
+        $this->context->requireAdmin(false);
+        $enabled = $this->state->enabled();
+        return [
+            'enabled' => (int)$enabled,
+            'mailboxes' => $enabled ? array_map(fn(Mailbox $mailbox): array => $mailbox->toArray(), $this->lookup->archivedMailboxes()) : [],
+        ];
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/OwnerResolver.php

@@ -0,0 +1,264 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use Mailcow\Archive\Model\Mailbox;
+use Mailcow\Archive\Repository\LookupRepository;
+
+/**
+ * Finds the mailboxes a mail belongs to in the archive, following the way Postfix delivers it.
+ *
+ * Incoming: each envelope recipient goes through recipient_maps (one rewrite) and the virtual alias expansion:
+ * alias rows (incl. catch-alls and groups), resources (discarded), spam aliases and mailboxes of alias domains,
+ * each tried as u+ext@d, u@d and @d, recursively. The local mailboxes at the end receive the mail. BCC map
+ * destinations count as recipients when rspamd sends the BCC copy (it bypasses rspamd).
+ *
+ * Outgoing: the SASL user and the mailbox the user sent as (its address, with +tag or in an alias domain);
+ * Postfix already checked that the login may use that sender. Without a login the sender never counts, so a
+ * forged local sender cannot make a mailbox an owner.
+ *
+ * Only mailboxes with archiving become owners.
+ */
+class OwnerResolver
+{
+    /** virtual_alias_recursion_limit and virtual_alias_expansion_limit of Postfix */
+    private const MAX_DEPTH = 1000;
+    private const MAX_ADDRESSES = 1000;
+
+    public function __construct(private LookupRepository $lookup)
+    {
+    }
+
+    /**
+     * @param string[] $rcpts SMTP envelope recipients
+     * @param bool $bcc rspamd sent the BCC copies of this mail
+     * @return array{owners: string[], direction: string} sorted usernames; direction in, out or internal
+     */
+    public function resolve(string $from, array $rcpts, string $user, bool $bcc): array
+    {
+        $from = self::address($from);
+        $incoming = [];
+        foreach ($rcpts as $rcpt) {
+            $rcpt = self::address($rcpt);
+            if ($rcpt !== '') {
+                $this->deliver($rcpt, $incoming);
+            }
+        }
+        if ($bcc) {
+            foreach ($this->bccDestinations($from, $rcpts) as $destination) {
+                $this->deliver($destination, $incoming);
+            }
+        }
+        $outgoing = $this->senders($from, self::address($user));
+        $owners = [];
+        foreach (array_merge(array_values($incoming), array_values($outgoing)) as $mailbox) {
+            if ($mailbox->archived()) {
+                $owners[$mailbox->username] = true;
+            }
+        }
+        $owners = array_keys($owners);
+        sort($owners);
+        $direction = empty($outgoing) ? 'in' : (empty($incoming) ? 'out' : 'internal');
+        return ['owners' => $owners, 'direction' => $direction];
+    }
+
+    /** Adds the local mailboxes a recipient ends in to $mailboxes (username => Mailbox) */
+    private function deliver(string $address, array &$mailboxes): void
+    {
+        $seen = [];
+        $this->expand($this->canonical($address), 0, $mailboxes, $seen);
+    }
+
+    /** recipient_maps: first matching key form, one rewrite */
+    private function canonical(string $address): string
+    {
+        foreach (self::keys($address) as $key) {
+            $target = self::target($this->lookup->recipientMap($key));
+            if ($target !== '') {
+                return self::rewrite($address, $key, $target);
+            }
+        }
+        return $address;
+    }
+
+    private function expand(string $address, int $depth, array &$mailboxes, array &$seen): void
+    {
+        if ($depth > self::MAX_DEPTH || count($seen) >= self::MAX_ADDRESSES || isset($seen[$address])) {
+            return;
+        }
+        $seen[$address] = true;
+        $targets = $this->virtual($address);
+        if ($targets === null) {
+            $this->final($address, $mailboxes);
+            return;
+        }
+        foreach ($targets as $target) {
+            // an address that expands to itself is delivered as it is
+            if ($target === $address) {
+                $this->final($address, $mailboxes);
+            } else {
+                $this->expand($target, $depth + 1, $mailboxes, $seen);
+            }
+        }
+    }
+
+    /** Targets from the first key form a virtual alias map knows, null if none does */
+    private function virtual(string $address): ?array
+    {
+        if (self::split($address)[1] === 'localhost') {
+            // null@, spam@, ham@localhost: local delivery of Postfix, never a mailbox
+            return [];
+        }
+        foreach (self::keys($address) as $key) {
+            $targets = $this->virtualLookup($key);
+            if ($targets !== null) {
+                return array_map(fn(string $target): string => self::rewrite($address, $key, $target), $targets);
+            }
+        }
+        return null;
+    }
+
+    /** virtual_alias_maps in the order of main.cf: alias, resource, spam alias, alias domain mailbox */
+    private function virtualLookup(string $key): ?array
+    {
+        $domainKey = str_starts_with($key, '@');
+        $targets = self::targets($this->lookup->aliasGoto($key));
+        if ($targets === null && !$domainKey && $this->lookup->isResource($key)) {
+            $targets = ['null@localhost'];
+        }
+        $targets ??= self::targets($this->lookup->spamAliasGoto($key));
+        if ($targets === null && !$domainKey) {
+            [$local, $domain] = self::split($key);
+            $targets = self::targets($this->lookup->aliasDomainMailbox($local, $domain));
+        }
+        return $targets;
+    }
+
+    /** A final address: a mailbox (as u+ext@d or u@d) or anything else, which is not ours */
+    private function final(string $address, array &$mailboxes): void
+    {
+        foreach (array_unique([$address, self::untagged($address)]) as $candidate) {
+            $mailbox = $this->lookup->mailbox($candidate);
+            if ($mailbox !== null) {
+                $mailboxes[$mailbox->username] = $mailbox;
+                return;
+            }
+        }
+    }
+
+    /** BCC destinations of rspamd: rcpt maps for every recipient and its domain, sender maps for MAIL FROM and its domain */
+    private function bccDestinations(string $from, array $rcpts): array
+    {
+        $destinations = [];
+        foreach ($rcpts as $rcpt) {
+            $rcpt = self::address($rcpt);
+            if ($rcpt !== '') {
+                $destinations[] = $this->lookup->bccDestination(self::untagged($rcpt), 'rcpt');
+                $destinations[] = $this->lookup->bccDestination('@' . self::split($rcpt)[1], 'rcpt');
+            }
+        }
+        if ($from !== '') {
+            $destinations[] = $this->lookup->bccDestination(self::untagged($from), 'sender');
+            $destinations[] = $this->lookup->bccDestination('@' . self::split($from)[1], 'sender');
+        }
+        return array_values(array_unique(array_filter(array_map(fn(?string $address): string => self::address($address), $destinations))));
+    }
+
+    /** Mailboxes on the sending side: the SASL user and the mailbox the user sent as */
+    private function senders(string $from, string $user): array
+    {
+        $mailbox = ($user !== '') ? $this->lookup->mailbox($user) : null;
+        if ($mailbox === null) {
+            return [];
+        }
+        $senders = [$mailbox->username => $mailbox];
+        $sentAs = ($from !== '') ? $this->sentAs(self::untagged($from)) : null;
+        if ($sentAs !== null) {
+            $senders[$sentAs->username] = $sentAs;
+        }
+        return $senders;
+    }
+
+    /** Mailbox a sender address belongs to: the mailbox itself or the mailbox behind an alias domain address, not aliases */
+    private function sentAs(string $address): ?Mailbox
+    {
+        $mailbox = $this->lookup->mailbox($address);
+        if ($mailbox === null) {
+            [$local, $domain] = self::split($address);
+            $username = $this->lookup->aliasDomainMailbox($local, $domain);
+            $mailbox = ($username !== null) ? $this->lookup->mailbox(strtolower($username)) : null;
+        }
+        return $mailbox;
+    }
+
+    /** Lookup keys in the order of Postfix: u+ext@d, u@d, @d */
+    private static function keys(string $address): array
+    {
+        $keys = [$address, self::untagged($address), '@' . self::split($address)[1]];
+        return array_values(array_unique($keys));
+    }
+
+    /**
+     * Result of a lookup for $address under $key: '@domain' keeps the local part, a match without the extension
+     * passes it on (propagate_unmatched_extensions).
+     */
+    private static function rewrite(string $address, string $key, string $target): string
+    {
+        [$local] = self::split($address);
+        if (str_starts_with($target, '@')) {
+            return $local . $target;
+        }
+        $extension = self::extension($local);
+        if ($key === $address || $extension === '') {
+            return $target;
+        }
+        [$targetLocal, $targetDomain] = self::split($target);
+        return ($targetDomain === 'localhost' || self::extension($targetLocal) !== '')
+            ? $target
+            : $targetLocal . '+' . $extension . '@' . $targetDomain;
+    }
+
+    /** Comma separated lookup result; null when empty (not found) */
+    private static function targets(?string $value): ?array
+    {
+        $targets = array_values(array_filter(array_map(fn(string $target): string => self::target($target), explode(',', (string)$value))));
+        return empty($targets) ? null : $targets;
+    }
+
+    /** Address or '@domain' from a lookup result, lower case; '' if invalid */
+    private static function target(?string $value): string
+    {
+        $value = strtolower(trim((string)$value));
+        return preg_match('/^[^@\s,<>]*@[^@\s,<>]+$/', $value) ? $value : '';
+    }
+
+    /** Lower case address without angle brackets; '' unless local@domain */
+    private static function address(mixed $value): string
+    {
+        $value = is_string($value) ? strtolower(trim($value, " \t\r\n<>")) : '';
+        return preg_match('/^[^@\s,<>]+@[^@\s,<>]+$/', $value) ? $value : '';
+    }
+
+    private static function untagged(string $address): string
+    {
+        [$local, $domain] = self::split($address);
+        $pos = strpos($local, '+');
+        return ($pos === false || $pos === 0) ? $address : substr($local, 0, $pos) . '@' . $domain;
+    }
+
+    /** Address extension after the recipient delimiter "+" */
+    private static function extension(string $local): string
+    {
+        $pos = strpos($local, '+');
+        return ($pos === false || $pos === 0) ? '' : substr($local, $pos + 1);
+    }
+
+    /** [local part, domain] of an address or '@domain' */
+    private static function split(string $address): array
+    {
+        $at = strrpos($address, '@');
+        return ($at === false) ? [$address, ''] : [substr($address, 0, $at), substr($address, $at + 1)];
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/SettingsService.php

@@ -0,0 +1,164 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use Mailcow\Archive\Model\Settings;
+use Mailcow\Archive\Redis\ArchiveState;
+use Mailcow\Archive\Repository\LookupRepository;
+use Mailcow\Archive\Repository\SettingsRepository;
+use Mailcow\Archive\Smtp\SmtpClient;
+use Mailcow\Archive\Smtp\SmtpException;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+
+/**
+ * Admin use cases of the archive target. Read-only API keys may read the settings (never password or key).
+ */
+class SettingsService
+{
+    private const MAX_SECRET_LENGTH = 255;
+    /** Extensions the archive should announce: Postfix passes 8-bit mails unchanged (archive_smtp in master.cf) */
+    private const WANTED_EXTENSIONS = ['8BITMIME'];
+
+    public function __construct(private SettingsRepository $repo, private LookupRepository $lookup, private ArchiveState $state,
+        private SmtpClient $smtp, private Context $context)
+    {
+    }
+
+    /** @throws AccessDenied */
+    public function get(): Settings
+    {
+        $this->context->requireAdmin(false);
+        $settings = $this->repo->load();
+        $settings->enabled = $this->state->enabled();
+        return $settings;
+    }
+
+    /** Global switch for internal callers (page templates, mailbox details), no permission check */
+    public function enabled(): bool
+    {
+        return $this->state->enabled();
+    }
+
+    /**
+     * Saves the settings: attributes missing in $input keep their value, an empty password or signing key keeps
+     * the stored one, `password_remove` deletes the password. Switching on needs an address (and a signing key for archivecow).
+     *
+     * @throws UserError
+     */
+    public function save(array $input): Settings
+    {
+        $this->context->requireAdmin();
+        $settings = $this->repo->load();
+        $settings->enabled = $this->state->enabled();
+        if (array_key_exists('address', $input)) {
+            $address = strtolower((string)Input::text($input['address']));
+            if (!is_scalar($input['address']) || ($address !== '' && !filter_var($address, FILTER_VALIDATE_EMAIL))) {
+                throw new UserError('archive_address_invalid');
+            }
+            // Postfix would treat an own address as local (aliases, unknown users) before the copy reaches the archive
+            if ($address !== '' && $this->lookup->isMailcowDomain(substr($address, strrpos($address, '@') + 1))) {
+                throw new UserError('archive_address_local');
+            }
+            $settings->address = $address;
+        }
+        if (array_key_exists('host', $input)) {
+            $host = strtolower(trim((string)Input::text($input['host']), " []"));
+            if (!is_scalar($input['host']) || ($host !== '' && !filter_var($host, FILTER_VALIDATE_IP)
+                && !filter_var($host, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME))) {
+                throw new UserError('archive_host_invalid');
+            }
+            $settings->host = $host;
+        }
+        if (array_key_exists('port', $input)) {
+            $settings->port = Input::intInRange($input['port'], 1, 65535) ?? throw new UserError('archive_port_invalid');
+        }
+        if (array_key_exists('tls', $input)) {
+            $settings->tls = in_array($input['tls'], Settings::TLS_MODES, true) ? $input['tls'] : throw new UserError('archive_tls_invalid');
+        }
+        if (array_key_exists('username', $input)) {
+            $username = Input::text($input['username']);
+            if ($username === null || strlen($username) > self::MAX_SECRET_LENGTH || preg_match('/[\s:]/', $username)) {
+                throw new UserError('archive_username_invalid');
+            }
+            $settings->username = $username;
+        }
+        if (array_key_exists('password', $input)) {
+            $settings->password = self::secret($input['password'], $settings->password, 'archive_password_invalid');
+        }
+        if (self::flag($input['password_remove'] ?? '0')) {
+            $settings->password = '';
+        }
+        if (array_key_exists('profile', $input)) {
+            $settings->profile = in_array($input['profile'], Settings::PROFILES, true) ? $input['profile'] : throw new UserError('archive_profile_invalid');
+        }
+        if (array_key_exists('signing_secret', $input)) {
+            $settings->signingKey = self::secret($input['signing_secret'], $settings->signingKey, 'archive_signing_secret_invalid');
+        }
+        foreach (['header_from' => 'headerFrom', 'header_rcpt' => 'headerRcpt'] as $key => $property) {
+            if (array_key_exists($key, $input)) {
+                $name = Input::text($input[$key]);
+                if ($name === null || !preg_match('/^[A-Za-z][A-Za-z0-9-]{0,63}$/', $name)) {
+                    throw new UserError('archive_header_invalid');
+                }
+                $settings->$property = $name;
+            }
+        }
+        if (array_key_exists('enabled', $input)) {
+            $settings->enabled = self::flag($input['enabled']);
+        }
+        if ($settings->enabled && !$settings->canDeliver()) {
+            throw new UserError('archive_incomplete');
+        }
+        $this->repo->save($settings);
+        $this->state->setEnabled($settings->enabled);
+        return $settings;
+    }
+
+    /**
+     * Tests the stored target: connection, TLS, AUTH and whether the archive accepts the address (no mail is sent).
+     * Returns the success message, with the missing extension when the archive lacks 8BITMIME.
+     *
+     * @throws UserError
+     */
+    public function test(): string|array
+    {
+        $this->context->requireAdmin();
+        $settings = $this->repo->load();
+        if ($settings->address === '') {
+            throw new UserError('archive_incomplete');
+        }
+        $host = ($settings->host !== '') ? $settings->host : $this->smtp->mailHost(substr($settings->address, strrpos($settings->address, '@') + 1));
+        try {
+            $extensions = $this->smtp->probe($host, $settings->port, $settings->tls, $settings->username, $settings->password,
+                Settings::SENDER, $settings->address);
+        } catch (SmtpException $e) {
+            throw new UserError('archive_test_failed', [htmlspecialchars($host . ':' . $settings->port . ' ' . $e->describe())]);
+        }
+        $missing = array_values(array_diff(self::WANTED_EXTENSIONS, $extensions));
+        return empty($missing) ? 'archive_test_ok' : ['archive_test_ok_missing', implode(', ', $missing)];
+    }
+
+    /** A new secret, or the stored one when $value is empty */
+    private static function secret(mixed $value, string $current, string $error): string
+    {
+        $secret = Input::text($value);
+        if ($secret === null || strlen($secret) > self::MAX_SECRET_LENGTH || preg_match('/[^\x21-\x7e]/', $secret)) {
+            throw new UserError($error);
+        }
+        return ($secret === '') ? $current : $secret;
+    }
+
+    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
+    private static function flag(mixed $value): bool
+    {
+        if (is_array($value)) {
+            $value = end($value);
+        }
+        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
+    }
+}

data/web/inc/lib/mailcow/Archive/Service/StatusService.php

@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Service;
+
+use Mailcow\Archive\Legacy\LegacyQueue;
+use Mailcow\Archive\Model\Settings;
+use Mailcow\Archive\Redis\ArchiveState;
+use Mailcow\Archive\Repository\SettingsRepository;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+
+/**
+ * State of the archive for the admin: the summary of the archive tab (copies waiting in the Postfix queue with the
+ * reason Postfix gives, last delivery, mails not archived) and the archive log of the dashboard.
+ */
+class StatusService
+{
+    /** Mails not archived count as a problem for a day */
+    private const DROPPED_WINDOW = 86400;
+
+    public function __construct(private SettingsRepository $settings, private ArchiveState $state, private LegacyQueue $queue,
+        private Context $context)
+    {
+    }
+
+    /**
+     * GET archive/status (admins, read-only keys included).
+     *
+     * @throws AccessDenied
+     */
+    public function status(): array
+    {
+        $this->context->requireAdmin(false);
+        $settings = $this->settings->load();
+        $copies = $this->queue->entries(Settings::SENDER);
+        $reasons = [];
+        $oldest = null;
+        $bytes = 0;
+        foreach ($copies as $copy) {
+            $arrival = (int)($copy['arrival_time'] ?? 0);
+            $oldest = ($oldest === null) ? $arrival : min($oldest, $arrival);
+            $bytes += (int)($copy['message_size'] ?? 0);
+            foreach ((array)($copy['recipients'] ?? []) as $rcpt) {
+                if (is_string($rcpt) && preg_match('/^\S+ \((.+)\)$/', $rcpt, $m)) {
+                    $reasons[$m[1]] = true;
+                }
+            }
+        }
+        $dropped = $this->state->droppedStats();
+        $recent = array_values(array_filter($dropped['recent'], fn(array $d): bool => $d['time'] >= time() - self::DROPPED_WINDOW));
+        $lastDelivery = null;
+        foreach ($this->state->events() as $event) {
+            if ($event['status'] === 'sent') {
+                $lastDelivery = $event['time'];
+                break;
+            }
+        }
+        return [
+            'enabled' => (int)$this->state->enabled(),
+            'configured' => (int)$settings->canDeliver(),
+            'queued' => count($copies),
+            'queued_bytes' => $bytes,
+            'oldest_queued' => $oldest,
+            'delay_reasons' => array_slice(array_keys($reasons), 0, 5),
+            'last_delivery' => $lastDelivery,
+            'dropped' => $dropped['count'],
+            'dropped_24h' => count($recent),
+            'last_dropped' => $recent ? ['time' => $recent[0]['time'], 'reason' => $recent[0]['reason']] : null,
+        ];
+    }
+
+    /**
+     * GET logs/archive: deliveries, expired copies and mails not archived, newest first (admins, read-only keys included);
+     * $count limits the entries, 0 keeps the default.
+     *
+     * @throws AccessDenied
+     */
+    public function log(int $count = 0): array
+    {
+        $this->context->requireAdmin(false);
+        return ($count > 0) ? $this->state->events($count) : $this->state->events();
+    }
+}

data/web/inc/lib/mailcow/Archive/Smtp/SmtpClient.php

@@ -0,0 +1,190 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Smtp;
+
+/**
+ * Small SMTP client: hands archive copies to Postfix (port 591: no milter, mynetworks only) and tests the
+ * connection to the archive server (EHLO, STARTTLS, AUTH, MAIL/RCPT, no mail is sent).
+ *
+ * The socket I/O sits in protected methods, so tests can replace it.
+ */
+class SmtpClient
+{
+    private const TIMEOUT = 20;
+
+    /** @var resource|null */
+    private $socket = null;
+
+    /**
+     * Sends one mail and returns the queue id Postfix reports. $data must end with CRLF; it is received as it is.
+     *
+     * @throws SmtpException
+     */
+    public function submit(string $host, int $port, string $from, string $rcpt, string $data): string
+    {
+        $this->open($host, $port);
+        try {
+            $this->expect('connect', 220);
+            $this->ehlo();
+            $this->command('sender', 'MAIL FROM:<' . $from . '>', 250);
+            $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250);
+            $this->command('data', 'DATA', 354);
+            $this->write(self::dotStuff(self::crlf($data)) . ".\r\n");
+            $reply = $this->expect('data', 250);
+            $this->write("QUIT\r\n");
+            return preg_match('/queued as ([0-9A-Za-z]+)/', $reply, $m) ? $m[1] : '';
+        } finally {
+            $this->close();
+        }
+    }
+
+    /**
+     * Connection test against the archive server: STARTTLS (unless $tls is 'may' and it is not offered), AUTH when
+     * a username is set, then MAIL and RCPT for the archive address, then RSET.
+     *
+     * @return string[] extensions the server announced (after STARTTLS)
+     * @throws SmtpException
+     */
+    public function probe(string $host, int $port, string $tls, string $username, string $password, string $from, string $rcpt): array
+    {
+        $this->open($host, $port);
+        try {
+            $this->expect('connect', 220);
+            $extensions = $this->ehlo();
+            if ($tls !== 'may' || in_array('STARTTLS', $extensions, true)) {
+                if (!in_array('STARTTLS', $extensions, true)) {
+                    throw new SmtpException('tls', 'STARTTLS not offered');
+                }
+                $this->command('tls', 'STARTTLS', 220);
+                $this->startTls($host, $tls === 'verify');
+                $extensions = $this->ehlo();
+            }
+            if ($username !== '') {
+                $this->command('auth', 'AUTH PLAIN ' . base64_encode("\0" . $username . "\0" . $password), 235);
+            }
+            $this->command('sender', 'MAIL FROM:<' . $from . '>', 250);
+            $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250);
+            $this->command('recipient', 'RSET', 250);
+            $this->write("QUIT\r\n");
+            return $extensions;
+        } finally {
+            $this->close();
+        }
+    }
+
+    /** Mail server of a domain: the MX with the lowest preference, else the domain itself */
+    public function mailHost(string $domain): string
+    {
+        if (@getmxrr($domain, $hosts, $weights) && !empty($hosts)) {
+            array_multisort($weights, SORT_NUMERIC, $hosts);
+            return rtrim($hosts[0], '.');
+        }
+        return $domain;
+    }
+
+    /** CRLF line ends, ending with CRLF */
+    public static function crlf(string $data): string
+    {
+        $data = (string)preg_replace('/\r\n|\r|\n/', "\r\n", $data);
+        return str_ends_with($data, "\r\n") ? $data : $data . "\r\n";
+    }
+
+    /** RFC 5321 4.5.2: a line starting with "." gets another "." */
+    public static function dotStuff(string $data): string
+    {
+        $data = str_replace("\r\n.", "\r\n..", $data);
+        return str_starts_with($data, '.') ? '.' . $data : $data;
+    }
+
+    /** @throws SmtpException */
+    protected function open(string $host, int $port): void
+    {
+        $target = filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? '[' . $host . ']' : $host;
+        $socket = @stream_socket_client('tcp://' . $target . ':' . $port, $errno, $error, self::TIMEOUT);
+        if ($socket === false) {
+            throw new SmtpException('connect', $error !== '' ? $error : 'connection failed');
+        }
+        stream_set_timeout($socket, self::TIMEOUT);
+        $this->socket = $socket;
+    }
+
+    protected function close(): void
+    {
+        if (is_resource($this->socket)) {
+            fclose($this->socket);
+        }
+        $this->socket = null;
+    }
+
+    /** @throws SmtpException */
+    protected function write(string $data): void
+    {
+        for ($written = 0; $written < strlen($data); $written += $count) {
+            $count = @fwrite($this->socket, substr($data, $written, 1048576));
+            if ($count === false || $count === 0) {
+                throw new SmtpException('data', 'connection lost while sending');
+            }
+        }
+    }
+
+    /** One reply line without CRLF, null when the connection ended */
+    protected function readLine(): ?string
+    {
+        $line = fgets($this->socket, 4096);
+        return ($line === false) ? null : rtrim($line, "\r\n");
+    }
+
+    /** @throws SmtpException */
+    protected function startTls(string $host, bool $verify): void
+    {
+        stream_context_set_option($this->socket, 'ssl', 'verify_peer', $verify);
+        stream_context_set_option($this->socket, 'ssl', 'verify_peer_name', $verify);
+        stream_context_set_option($this->socket, 'ssl', 'allow_self_signed', !$verify);
+        stream_context_set_option($this->socket, 'ssl', 'peer_name', $host);
+        if (!@stream_socket_enable_crypto($this->socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT)) {
+            throw new SmtpException('tls', $verify ? 'TLS handshake failed or certificate not valid' : 'TLS handshake failed');
+        }
+    }
+
+    /** @return string[] extensions in upper case */
+    private function ehlo(): array
+    {
+        $reply = $this->command('connect', 'EHLO ' . (gethostname() ?: 'mailcow'), 250);
+        $extensions = [];
+        foreach (array_slice(explode("\n", $reply), 1) as $line) {
+            $extensions[] = strtoupper((string)strtok(substr($line, 4), ' '));
+        }
+        return $extensions;
+    }
+
+    /** @throws SmtpException */
+    private function command(string $step, string $command, int $code): string
+    {
+        $this->write($command . "\r\n");
+        return $this->expect($step, $code);
+    }
+
+    /**
+     * Reads a (multi-line) reply and returns it, lines joined by "\n".
+     *
+     * @throws SmtpException when the code differs
+     */
+    private function expect(string $step, int $code): string
+    {
+        $lines = [];
+        do {
+            $line = $this->readLine();
+            if ($line === null) {
+                throw new SmtpException($step, 'connection closed');
+            }
+            $lines[] = $line;
+        } while (strlen($line) > 3 && $line[3] === '-');
+        $reply = implode("\n", $lines);
+        if ((int)substr($reply, 0, 3) !== $code && !($code === 250 && (int)substr($reply, 0, 3) === 251)) {
+            throw new SmtpException($step, $reply);
+        }
+        return $reply;
+    }
+}

data/web/inc/lib/mailcow/Archive/Smtp/SmtpException.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Archive\Smtp;
+
+use RuntimeException;
+
+/**
+ * An SMTP step failed: connect, tls, auth, sender, recipient or data, with the reply of the server.
+ */
+class SmtpException extends RuntimeException
+{
+    private const MAX_TEXT_LENGTH = 200;
+
+    public function __construct(public readonly string $step, public readonly string $reply)
+    {
+        parent::__construct($step . ': ' . $reply);
+    }
+
+    /** Short text for messages: "<step>: <reply>", without angle brackets (the UI escapes message arguments twice) */
+    public function describe(): string
+    {
+        $text = trim((string)preg_replace('/\s+/', ' ', $this->step . ': ' . str_replace(['<', '>'], '', $this->reply)));
+        return mb_substr($text, 0, self::MAX_TEXT_LENGTH);
+    }
+}

data/web/inc/lib/mailcow/Backup/BackupModule.php

@@ -0,0 +1,90 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup;
+
+use Mailcow\Backup\Controller\BackupApiController;
+use Mailcow\Backup\Legacy\LegacyDomain;
+use Mailcow\Backup\Legacy\NoTemplates;
+use Mailcow\Backup\Model\Settings;
+use Mailcow\Backup\Repository\ExportRepository;
+use Mailcow\Backup\Repository\JobRepository;
+use Mailcow\Backup\Repository\RestoreRepository;
+use Mailcow\Backup\Repository\SettingsRepository;
+use Mailcow\Backup\Repository\SnapshotRepository;
+use Mailcow\Backup\Service\DomainRestoreService;
+use Mailcow\Backup\Service\ExportService;
+use Mailcow\Backup\Service\GroupMembers;
+use Mailcow\Backup\Service\JobService;
+use Mailcow\Backup\Service\RestoreService;
+use Mailcow\Backup\Service\SettingsService;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\Dovecot\DovecotModule;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+use Mailcow\MailboxGroup\Service\ChangeEffects;
+use Mailcow\MailboxGroup\Service\SharingService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use Mailcow\Sogo\SogoModule;
+use PDO;
+use Redis;
+
+/**
+ * Composition root of the backup with restic: settings, jobs of the runners, snapshots, the export of a domain and the
+ * restore of mailboxes and domains. The runner in the restic container reaches the export and the restore actions
+ * through data/conf/rspamd/dynmaps/backup.php.
+ *
+ *   process_get_return(BackupModule::fromGlobals()->api->getSettings());
+ *   echo json_encode(BackupModule::create($pdo, Context::system(), $config, $redis)->api->export($domain, $page));  // backup.php
+ *
+ * $config: timezone (TZ of mailcow, for the schedule), alias_disable_external_domains ($ALIAS_DISABLE_EXTERNAL_DOMAINS),
+ * demo_mode ($DEMO_MODE: the API changes nothing).
+ * $redis: for DKIM keys, rate limits and the domain map (export and restore).
+ */
+class BackupModule
+{
+    public function __construct(
+        public readonly SettingsService $settings,
+        public readonly JobService $jobs,
+        public readonly ExportService $export,
+        public readonly RestoreService $restore,
+        public readonly DomainRestoreService $domainRestore,
+        public readonly BackupApiController $api
+    ) {
+    }
+
+    public static function create(PDO $pdo, Context $context, array $config = [], ?Redis $redis = null): self
+    {
+        $jobRepo = new JobRepository($pdo);
+        $restoreRepo = new RestoreRepository($pdo);
+        $legacy = new LegacyDomain($redis);
+        $settings = new SettingsService(new SettingsRepository($pdo), $context);
+        $jobs = new JobService($jobRepo, new SnapshotRepository($pdo), $settings, $context, (string)($config['timezone'] ?? 'UTC'));
+        $export = new ExportService(new ExportRepository($pdo), $legacy);
+        $sogo = SogoModule::create()->sogo;
+        $dovecot = DovecotModule::create()->dovecot;
+        // the group module without group templates: a restored mailbox keeps the settings of the backup
+        $groupRepo = new GroupRepository($pdo);
+        $lookup = new LookupRepository($pdo);
+        $templates = new TemplateService($lookup, new NoTemplates());
+        $effects = new ChangeEffects($templates, $sogo, $groupRepo, $dovecot);
+        $groups = new GroupMembers($groupRepo, $lookup, $templates, $effects, new SharingService($groupRepo, $dovecot, $sogo, $effects));
+        $restore = new RestoreService($restoreRepo, $jobRepo, $groups, $sogo, $dovecot, $legacy, (bool)($config['alias_disable_external_domains'] ?? false));
+        $domainRestore = new DomainRestoreService($restoreRepo, $jobRepo, $groups, $legacy);
+        return new self($settings, $jobs, $export, $restore, $domainRestore,
+            new BackupApiController($settings, $jobs, $export, $restore, $domainRestore, new ApiResponse(Settings::SECRETS),
+                (bool)($config['demo_mode'] ?? false)));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo, $redis, $_SESSION, TZ, $ALIAS_DISABLE_EXTERNAL_DOMAINS and $DEMO_MODE */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), [
+            'timezone' => getenv('TZ') ?: 'UTC',
+            'alias_disable_external_domains' => ($GLOBALS['ALIAS_DISABLE_EXTERNAL_DOMAINS'] ?? false) === true,
+            'demo_mode' => ($GLOBALS['DEMO_MODE'] ?? false) === true,
+        ],($GLOBALS['redis'] ?? null) instanceof Redis ? $GLOBALS['redis'] : null);
+    }
+}

data/web/inc/lib/mailcow/Backup/Controller/BackupApiController.php

@@ -0,0 +1,181 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Controller;
+
+use Mailcow\Backup\Model\Job;
+use Mailcow\Backup\Service\DomainRestoreService;
+use Mailcow\Backup\Service\ExportService;
+use Mailcow\Backup\Service\JobService;
+use Mailcow\Backup\Service\RestoreService;
+use Mailcow\Backup\Service\SettingsService;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php (routes backup-settings, backup-status, backup-snapshots, backup-snapshot-content,
+ * backup-jobs, backup-job, backup-restore) and the runner's endpoint data/conf/rspamd/dynmaps/backup.php (domains,
+ * export pages, restore actions). Messages are logged as ['backup', action, object, data].
+ */
+class BackupApiController
+{
+    public function __construct(private SettingsService $settings, private JobService $jobs, private ExportService $export,
+        private RestoreService $restore, private DomainRestoreService $domainRestore, private ApiResponse $response,
+        private bool $demoMode = false)
+    {
+    }
+
+    /** GET backup-settings: secrets as <key>_set; {} for non-admins */
+    public function getSettings(): array
+    {
+        return $this->settings->get()?->toArray() ?? [];
+    }
+
+    /** POST edit/backup-settings */
+    public function editSettings(array $attr): bool
+    {
+        return $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
+            $this->settings->save($attr);
+            return $this->response->success('backup_settings_saved', $log);
+        });
+    }
+
+    /** GET backup-status */
+    public function getStatus(): array
+    {
+        return $this->jobs->status();
+    }
+
+    /** GET backup-snapshots */
+    public function getSnapshots(): array
+    {
+        return $this->jobs->snapshots();
+    }
+
+    /** GET backup-snapshot-content/<snapshot>[/<domain>]: domains of a snapshot, or the mailboxes of one, with their state now */
+    public function getSnapshotContent(mixed $snapshot, mixed $domain = null): array
+    {
+        return ($domain === null || $domain === '') ? $this->jobs->content($snapshot) : $this->jobs->domainContent($snapshot, $domain);
+    }
+
+    /** GET backup-snapshot-search/<snapshot>/<term>: domains and mailboxes of a snapshot matching the term */
+    public function getSnapshotSearch(mixed $snapshot, mixed $term): array
+    {
+        return $this->jobs->search($snapshot, $term);
+    }
+
+    /** GET backup-jobs[/<count>[/<type>]]: newest first, without logs; count 0 = default */
+    public function getJobs(int $limit = 0, mixed $type = null): array
+    {
+        return array_map(fn(Job $job): array => $this->jobs->view($job), $this->jobs->list($limit, $type));
+    }
+
+    /** GET backup-job/<id>: with its log */
+    public function getJob(mixed $id): array
+    {
+        $id = $this->toId($id);
+        $job = ($id === null) ? null : $this->jobs->get($id);
+        return ($job === null) ? [] : $this->jobs->view($job, true);
+    }
+
+    /** POST add/backup-job: {type: backup|check|prune|init} */
+    public function addJob(array $attr): bool
+    {
+        return $this->handle('add', 'job', $attr, function (array $log) use ($attr): bool {
+            $job = $this->jobs->create($attr);
+            return $this->response->success(['backup_job_added', $job->type, $job->id], $log);
+        });
+    }
+
+    /** POST add/backup-restore: {snapshot, scope, mailboxes | domain, components, mail_target} */
+    public function addRestore(array $attr): bool
+    {
+        return $this->handle('add', 'restore', $attr, function (array $log) use ($attr): bool {
+            $job = $this->jobs->restore($attr);
+            return $this->response->success(['backup_restore_added', $job->id, count($job->mailboxes())], $log);
+        });
+    }
+
+    /** POST delete/backup-job: cancels queued and running jobs, one message per id */
+    public function deleteJobs(array $items): bool
+    {
+        return $this->handle('delete', 'job', ['id' => $items], function (array $log) use ($items): bool {
+            foreach ($items as $id) {
+                try {
+                    $job = $this->jobs->cancel($this->requireId($id));
+                    $this->response->success(['backup_job_cancelled', $job->id], $log);
+                } catch (UserError $e) {
+                    $this->response->error($e, $log);
+                }
+            }
+            return true;
+        });
+    }
+
+    /** backup.php ?domains=1 */
+    public function domains(): array
+    {
+        return ['domains' => $this->export->domains()];
+    }
+
+    /** backup.php ?export=<domain>[&after=<username>] @throws UserError */
+    public function export(string $domain, ?string $after = null): array
+    {
+        return $this->export->export($domain, $after);
+    }
+
+    /**
+     * backup.php POST of the runner, {"result": ...} of
+     *   {"action": "restore_domain", "job": id, "data": <domain_data>}: creates a deleted domain
+     *   {"action": "restore_aliases", "job": id, "aliases": [...]}: a part of the domain's aliases (scope domain)
+     *   {"action": "restore", "job": id, "username": u, "data": <mailbox section>}: one mailbox
+     *
+     * @throws UserError
+     */
+    public function post(array $body): array
+    {
+        return ['result' => match ($body['action'] ?? null) {
+            'restore' => $this->restore->merge($body['job'] ?? null, $body['username'] ?? null, $body['data'] ?? null),
+            'restore_aliases' => $this->restore->mergeAliases($body['job'] ?? null, $body['aliases'] ?? null),
+            'restore_domain' => $this->domainRestore->restore($body['job'] ?? null, $body['data'] ?? null),
+            default => throw new UserError('backup_action_invalid'),
+        }];
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     * In demo mode no write call runs.
+     */
+    private function handle(string $action, string $object, mixed $data, callable $work): bool
+    {
+        $log = ['backup', $action, $object, $data];
+        try {
+            if ($this->demoMode) {
+                throw new UserError('demo_mode_enabled');
+            }
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+
+    /** Integer id from request input ("12", 12); null for anything else */
+    private function toId(mixed $id): ?int
+    {
+        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
+    }
+
+    /** @throws AccessDenied for ids that cannot exist */
+    private function requireId(mixed $id): int
+    {
+        return $this->toId($id) ?? throw new AccessDenied();
+    }
+}

data/web/inc/lib/mailcow/Backup/Legacy/LegacyDomain.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Legacy;
+
+use Redis;
+use RuntimeException;
+
+/**
+ * What the legacy code keeps outside the database for domains and mailboxes: DKIM keys, rate limits and the domain
+ * map in Redis (functions.dkim/ratelimit/mailbox.inc.php), and the SOGo restart through dockerapi. Redis and docker()
+ * sit in protected methods so tests can replace them.
+ */
+class LegacyDomain
+{
+    public function __construct(private ?Redis $redis = null)
+    {
+    }
+
+    /** @return ?array{selector: string, pubkey: string, privkey: string} the DKIM key of a domain */
+    public function dkim(string $domain): ?array
+    {
+        $pubkey = $this->redis()->hGet('DKIM_PUB_KEYS', $domain);
+        if (!is_string($pubkey) || $pubkey === '') {
+            return null;
+        }
+        $selector = (string)$this->redis()->hGet('DKIM_SELECTORS', $domain);
+        return ['selector' => $selector, 'pubkey' => $pubkey, 'privkey' => (string)$this->redis()->hGet('DKIM_PRIV_KEYS', $selector . '.' . $domain)];
+    }
+
+    /** Stores a DKIM key like dkim('import') does */
+    public function addDkim(string $domain, string $selector, string $pubkey, string $privkey): void
+    {
+        $this->redis()->hSet('DKIM_PUB_KEYS', $domain, $pubkey);
+        $this->redis()->hSet('DKIM_SELECTORS', $domain, $selector);
+        $this->redis()->hSet('DKIM_PRIV_KEYS', $selector . '.' . $domain, $privkey);
+    }
+
+    /**
+     * @param string[] $objects domains or mailboxes
+     * @return array<string, string> object => rate limit ("10 / 1h") of those with one
+     */
+    public function rateLimits(array $objects): array
+    {
+        if (empty($objects)) {
+            return [];
+        }
+        $values = $this->redis()->hMGet('RL_VALUE', array_values($objects));
+        return array_filter(is_array($values) ? $values : [], fn(mixed $value): bool => is_string($value) && $value !== '');
+    }
+
+    public function setRateLimit(string $object, string $value): void
+    {
+        $this->redis()->hSet('RL_VALUE', $object, $value);
+    }
+
+    /** The domain is local for rspamd (legacy domain and alias domain add) */
+    public function addDomainMap(string $domain): void
+    {
+        $this->redis()->hSet('DOMAIN_MAP', $domain, 1);
+    }
+
+    /**
+     * SOGo reads its domains at start: restart it after a domain was added (like the "restart SOGo" option of the form)
+     *
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function restartSogo(): ?string
+    {
+        $result = json_decode((string)$this->dockerRestart('sogo-mailcow'), true);
+        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
+    }
+
+    protected function redis(): Redis
+    {
+        return $this->redis ?? throw new RuntimeException('Redis is not connected');
+    }
+
+    protected function dockerRestart(string $service): mixed
+    {
+        return docker('post', $service, 'restart');
+    }
+}

data/web/inc/lib/mailcow/Backup/Legacy/NoTemplates.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Legacy;
+
+use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
+
+/**
+ * The group module's bridge to the mailbox code for restores: a membership restored from a backup never applies the
+ * group's mailbox template (the mailbox gets its settings from the backup).
+ */
+class NoTemplates extends LegacyMailbox
+{
+    public function applyTemplate(string $username, string $name, string $template): ?string
+    {
+        return null;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Job.php

@@ -0,0 +1,129 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * A job of the backup runners (table `backup_job`): backup, check, prune and init run in the restic container; a
+ * restore runs there (step `files`, then the merge through backup.php) and, for mails, IMAP ACLs and Sieve
+ * scripts, in the dovecot container (step `mails`). Times are unix timestamps.
+ */
+class Job
+{
+    public const TYPES = ['backup', 'check', 'prune', 'init', 'restore'];
+    /** Types an admin starts with add/backup-job */
+    public const MANUAL_TYPES = ['backup', 'check', 'prune', 'init'];
+    public const ACTIVE_STATES = ['queued', 'running'];
+
+    public ?int $id = null;
+    public string $type = 'backup';
+    public string $state = 'queued';
+    public ?string $step = null;
+    /**
+     * restore: snapshot, snapshot_time, scope, domain, mailboxes, components, mail_target, requested_role, create_domain (0|1),
+     * create_mailboxes (missing when requested), acl and allowed_domains (domain admins)
+     */
+    public array $params = [];
+    public ?array $result = null;
+    /** Only loaded for a single job */
+    public ?string $log = null;
+    public ?string $error = null;
+    /** restore: the domain of its mailboxes, so domain admins see their restores */
+    public string $domain = '';
+    public string $requestedBy = '';
+    public ?int $created = null;
+    public ?int $started = null;
+    public ?int $finished = null;
+
+    public function __construct(string $type = 'backup')
+    {
+        $this->type = $type;
+    }
+
+    /** Row of `backup_job` with the times as unix timestamps */
+    public static function fromRow(array $row): self
+    {
+        $job = new self((string)$row['type']);
+        $job->id = (int)$row['id'];
+        $job->state = (string)$row['state'];
+        $job->step = isset($row['step']) ? (string)$row['step'] : null;
+        $params = is_string($row['params'] ?? null) ? json_decode($row['params'], true) : null;
+        $job->params = is_array($params) ? $params : [];
+        $result = is_string($row['result'] ?? null) ? json_decode($row['result'], true) : null;
+        $job->result = is_array($result) ? $result : null;
+        $job->log = isset($row['log']) ? (string)$row['log'] : null;
+        $job->error = isset($row['error']) ? (string)$row['error'] : null;
+        $job->domain = (string)($row['domain'] ?? '');
+        $job->requestedBy = (string)($row['requested_by'] ?? '');
+        foreach (['created', 'started', 'finished'] as $time) {
+            $job->$time = isset($row[$time]) ? (int)$row[$time] : null;
+        }
+        return $job;
+    }
+
+    /** Column values for a new row */
+    public function toRow(): array
+    {
+        return [
+            'type' => $this->type,
+            'state' => $this->state,
+            'step' => $this->step,
+            'params' => json_encode($this->params ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
+            'domain' => $this->domain,
+            'requested_by' => $this->requestedBy,
+        ];
+    }
+
+    /** @return string[] mailboxes of a restore */
+    public function mailboxes(): array
+    {
+        return array_values(array_filter((array)($this->params['mailboxes'] ?? []), 'is_string'));
+    }
+
+    /** @return string[] components of a restore */
+    public function components(): array
+    {
+        return array_values(array_filter((array)($this->params['components'] ?? []), 'is_string'));
+    }
+
+    /** @return string[] mailboxes of a restore that did not exist when it was requested */
+    public function createMailboxes(): array
+    {
+        return array_values(array_filter((array)($this->params['create_mailboxes'] ?? []), 'is_string'));
+    }
+
+    /** The restore creates its domain (requested by an admin while the domain did not exist) */
+    public function createsDomain(): bool
+    {
+        return (int)($this->params['create_domain'] ?? 0) === 1;
+    }
+
+    public function isActive(): bool
+    {
+        return in_array($this->state, self::ACTIVE_STATES, true);
+    }
+
+    /** Entry of the API; the log only with $withLog (get/backup-job/<id>) */
+    public function toArray(bool $withLog = false): array
+    {
+        $job = [
+            'id' => $this->id,
+            'type' => $this->type,
+            'state' => $this->state,
+            'step' => $this->step,
+            'params' => $this->params ?: new \stdClass(),
+            'result' => $this->result,
+            'error' => $this->error,
+            'domain' => $this->domain,
+            'requested_by' => $this->requestedBy,
+            'created' => $this->created,
+            'started' => $this->started,
+            'finished' => $this->finished,
+        ];
+        if ($withLog) {
+            $job['log'] = (string)$this->log;
+        }
+        return $job;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/RestoreRun.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+use Mailcow\Core\Context;
+
+/**
+ * The merge of one mailbox of a restore job: what the backup holds, who asked for the restore and what the merge
+ * did per component (added, skipped, errors), plus the caches to refresh afterwards.
+ */
+class RestoreRun
+{
+    /** @var array<string, array{added: int, skipped: int, errors: string[]}> */
+    public array $results = [];
+    /** Mailboxes whose SOGo identities (aliases, send-as addresses) changed */
+    public array $sogoUsers = [];
+    /** Mailboxes whose userdb or passdb values changed (Dovecot auth cache) */
+    public array $authUsers = [];
+
+    /**
+     * @param array $mailbox  the mailbox row as it is now
+     * @param array $params   params of the restore job (requested_role, acl of a domain admin, scope, ...)
+     * @param array $data     the mailbox's section of the domain export in the snapshot
+     * @param string[] $domains domains whose aliases and send-as addresses the restore may change, [] = all (admins)
+     * @param string $requestedBy who requested the restore
+     */
+    /** The restore created the mailbox (its own alias gets the forwarding of the backup) */
+    public bool $created = false;
+
+    public function __construct(
+        public readonly string $username,
+        public readonly string $domain,
+        public readonly array $mailbox,
+        public readonly array $params,
+        public readonly array $data,
+        public readonly array $domains = [],
+        public readonly string $requestedBy = ''
+    ) {
+    }
+
+    /**
+     * The caller the group module checks: an admin, or the domain admin who requested the restore with his ACL of then
+     * and his domains now
+     */
+    public function requester(): Context
+    {
+        if ($this->byAdmin()) {
+            return new Context('admin', 'backup', [], true);
+        }
+        return new Context('domainadmin', $this->requestedBy, array_map(fn(mixed $flag): string => (string)(int)$flag, (array)($this->params['acl'] ?? [])));
+    }
+
+    public function start(string $component): void
+    {
+        $this->results[$component] = ['added' => 0, 'skipped' => 0, 'errors' => []];
+    }
+
+    public function added(string $component, int $count = 1): void
+    {
+        $this->results[$component]['added'] += $count;
+    }
+
+    public function skipped(string $component, int $count = 1): void
+    {
+        $this->results[$component]['skipped'] += $count;
+    }
+
+    public function error(string $component, string $error): void
+    {
+        $this->results[$component]['errors'][] = $error;
+    }
+
+    /** The part of a component was rolled back: nothing of it counts */
+    public function failed(string $component, string $error): void
+    {
+        $this->start($component);
+        $this->error($component, $error);
+    }
+
+    /** The restore was requested by an admin (domain admins are limited by their ACL and the job's domain) */
+    public function byAdmin(): bool
+    {
+        return ($this->params['requested_role'] ?? '') === 'admin';
+    }
+
+    /** Admins, or a domain admin with this ACL when he requested the restore */
+    public function may(string $acl): bool
+    {
+        return $this->byAdmin() || (int)($this->params['acl'][$acl] ?? 0) === 1;
+    }
+
+    public function mayChangeDomain(string $domain): bool
+    {
+        return $this->domains === [] || in_array(strtolower($domain), $this->domains, true);
+    }
+
+    /** Rows of a list in the backup (sieve_filters, tfa, ...) */
+    public function rows(string $key): array
+    {
+        return array_values(array_filter((array)($this->data[$key] ?? []), 'is_array'));
+    }
+
+    public function sogo(string ...$usernames): void
+    {
+        array_push($this->sogoUsers, ...$usernames);
+        $this->sogoUsers = array_values(array_unique($this->sogoUsers));
+    }
+
+    public function auth(string ...$usernames): void
+    {
+        array_push($this->authUsers, ...$usernames);
+        $this->authUsers = array_values(array_unique($this->authUsers));
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Settings.php

@@ -0,0 +1,189 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * Backup settings (table `backup_setting`, name/value pairs) over their defaults. Secrets are stored sealed for the
+ * runner (`sealed:v1:<base64>`) and never leave toArray(): the API gets `<key>_set` instead. The state_* values, the
+ * SSH public key and the SFTP host key are written by the runner in the restic container and cannot be set through
+ * the API (PHP writes state_target_changed when the target changes).
+ */
+class Settings
+{
+    public const DEFAULTS = [
+        'enabled' => '0',
+        'target' => 's3',
+        's3_endpoint' => 'https://s3.amazonaws.com',
+        's3_bucket' => '',
+        's3_path' => '',
+        's3_region' => '',
+        's3_access_key' => '',
+        's3_secret_key' => '',
+        's3_path_style' => '0',
+        'sftp_host' => '',
+        'sftp_port' => '22',
+        'sftp_user' => '',
+        'sftp_path' => '',
+        'sftp_public_key' => '',
+        'sftp_host_key' => '',
+        'rest_url' => '',
+        'rest_user' => '',
+        'rest_password' => '',
+        'local_path' => 'mailcow',
+        'password' => '',
+        'password_confirmed' => '0',
+        // JSON list of the scheduled backups: {"days": [weekdays, 0 = Sunday], "time": "HH:MM"}
+        'schedule' => '[{"days":[0,1,2,3,4,5,6],"time":"01:00"}]',
+        'keep_hourly' => '0',
+        'keep_daily' => '7',
+        'keep_weekly' => '4',
+        'keep_monthly' => '12',
+        'keep_yearly' => '0',
+        'prune_weekday' => '0',
+        'max_unused' => '10%',
+        'check_subset' => '30',
+        'include_index' => '1',
+        'include_fts' => '1',
+        'limit_upload' => '0',
+        'notify_email' => '',
+        'state_initialized' => '0',
+        'state_runner_seen' => '',
+        'state_last_backup' => '',
+        'state_last_success' => '',
+        'state_repo_size' => '',
+        'state_repo_stats' => '',
+        'state_schedule_since' => '',
+        'state_schedule' => '',
+        // public key (base64, X25519) the secrets are sealed with; the private key stays in the restic container
+        'state_secrets_pubkey' => '',
+        'state_secrets_error' => '',
+        // JSON {old_target, old_addresses, changed, changed_by, time}: the runner mails the old addresses and clears it
+        'state_target_changed' => '',
+        // RESTIC_BACKUP_MODE of mailcow.conf as the runner sees it
+        'state_backup_mode' => '',
+    ];
+    public const SECRETS = ['password', 's3_secret_key', 'rest_password'];
+    /** Written by the runner only */
+    public const RUNNER = ['sftp_public_key', 'sftp_host_key', 'state_initialized', 'state_runner_seen', 'state_last_backup',
+        'state_last_success', 'state_repo_size', 'state_repo_stats', 'state_schedule_since', 'state_schedule', 'state_secrets_pubkey',
+        'state_secrets_error', 'state_target_changed', 'state_backup_mode'];
+    /** Prefix of a sealed secret (sodium_crypto_box_seal with state_secrets_pubkey) */
+    public const SEALED = 'sealed:v1:';
+    public const TARGETS = ['s3', 'sftp', 'rest', 'local'];
+    /** The connection of each target: a change (or another target or repository password) resets state_initialized */
+    public const CONNECTION = [
+        's3' => ['s3_endpoint', 's3_bucket', 's3_path', 's3_region', 's3_access_key', 's3_secret_key', 's3_path_style'],
+        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
+        'rest' => ['rest_url', 'rest_user', 'rest_password'],
+        'local' => ['local_path'],
+    ];
+    /** Values a target cannot work without (besides the repository password) */
+    public const REQUIRED = [
+        's3' => ['s3_endpoint', 's3_bucket', 's3_access_key', 's3_secret_key'],
+        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
+        'rest' => ['rest_url'],
+        'local' => ['local_path'],
+    ];
+    /** Returned as numbers by the API ('' = not known yet: null; prune_weekday '' = never stays '') */
+    private const NUMBERS = ['enabled', 's3_path_style', 'sftp_port', 'password_confirmed', 'keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly',
+        'keep_yearly', 'prune_weekday', 'check_subset', 'include_index', 'include_fts', 'limit_upload', 'state_initialized',
+        'state_runner_seen', 'state_last_backup', 'state_last_success', 'state_repo_size', 'state_schedule_since'];
+    private const JSON = ['schedule', 'state_repo_stats', 'state_target_changed'];
+
+    /** @param array<string, string> $values every key of DEFAULTS */
+    public function __construct(public array $values = self::DEFAULTS)
+    {
+    }
+
+    /** Stored values (name => value) over the defaults; unknown names are dropped */
+    public static function fromStored(array $stored): self
+    {
+        $values = self::DEFAULTS;
+        foreach ($stored as $name => $value) {
+            if (array_key_exists($name, $values)) {
+                $values[$name] = (string)$value;
+            }
+        }
+        return new self($values);
+    }
+
+    public function get(string $name): string
+    {
+        return $this->values[$name] ?? '';
+    }
+
+    /** The target has its required values and the repository password is set */
+    public function configured(): bool
+    {
+        $required = self::REQUIRED[$this->get('target')] ?? null;
+        if ($required === null || $this->get('password') === '') {
+            return false;
+        }
+        foreach ($required as $name) {
+            if ($this->get($name) === '') {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * The schedule as [weekday (0 = Sunday), hour, minute] slots, sorted and unique
+     *
+     * @return array<int, array{int, int, int}>
+     */
+    public function scheduleSlots(): array
+    {
+        $entries = json_decode($this->get('schedule'), true);
+        $slots = [];
+        foreach (is_array($entries) ? $entries : [] as $entry) {
+            if (!is_array($entry) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', (string)($entry['time'] ?? ''), $time) || !is_array($entry['days'] ?? null)) {
+                continue;
+            }
+            foreach ($entry['days'] as $day) {
+                if (is_int($day) && $day >= 0 && $day <= 6) {
+                    $slots[] = $day * 1440 + (int)$time[1] * 60 + (int)$time[2];
+                }
+            }
+        }
+        $slots = array_unique($slots);
+        sort($slots);
+        return array_map(fn(int $slot): array => [intdiv($slot, 1440), intdiv($slot % 1440, 60), $slot % 60], $slots);
+    }
+
+    /** The longest time between two scheduled backups of a week in seconds, null without a schedule */
+    public function scheduleGap(): ?int
+    {
+        $slots = array_map(fn(array $slot): int => $slot[0] * 86400 + $slot[1] * 3600 + $slot[2] * 60, $this->scheduleSlots());
+        if (empty($slots)) {
+            return null;
+        }
+        $gaps = [$slots[0] + 7 * 86400 - end($slots)];
+        for ($i = 1; $i < count($slots); $i++) {
+            $gaps[] = $slots[$i] - $slots[$i - 1];
+        }
+        return max($gaps);
+    }
+
+    /** Entry of get/backup-settings: every key, secrets as <key>_set, plus `configured` */
+    public function toArray(): array
+    {
+        $settings = [];
+        foreach ($this->values as $name => $value) {
+            if (in_array($name, self::SECRETS, true)) {
+                $settings[$name . '_set'] = (int)($value !== '');
+            } elseif (in_array($name, self::JSON, true)) {
+                $decoded = json_decode($value, true);
+                $settings[$name] = is_array($decoded) ? $decoded : null;
+            } elseif (in_array($name, self::NUMBERS, true)) {
+                $settings[$name] = ($value === '') ? (($name === 'prune_weekday') ? '' : null) : (int)$value;
+            } else {
+                $settings[$name] = $value;
+            }
+        }
+        $settings['configured'] = (int)$this->configured();
+        return $settings;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Snapshot.php

@@ -0,0 +1,142 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * A snapshot of the repository (table `backup_snapshot`, the runner's cache of `restic snapshots --json`).
+ */
+class Snapshot
+{
+    public string $id = '';
+    public string $shortId = '';
+    /** unix timestamp */
+    public int $time = 0;
+    public string $hostname = '';
+    /** @var string[] */
+    public array $tags = [];
+    /** restic's summary of the backup (data_added, files_new, ...) */
+    public ?array $summary = null;
+    /** @var ?string[] domains of its manifest, null for snapshots not made by mailcow */
+    public ?array $domains = null;
+    /** @var array<string, ?int> domain => its creation time (unix) when the snapshot was made, from the snapshot row */
+    public array $created = [];
+    /**
+     * @var ?array<string, array{created: ?int, mailboxes: array<string, string>}> per domain its creation time (unix, null
+     *      when not recorded) and username => name of its mailboxes; null = not known (snapshots this installation did
+     *      not make itself, e.g. of another server in the same repository)
+     */
+    public ?array $content = null;
+
+    /** Row of `backup_snapshot` with the time as unix timestamp */
+    public static function fromRow(array $row): self
+    {
+        $snapshot = new self();
+        $snapshot->id = (string)$row['id'];
+        $snapshot->shortId = (string)$row['short_id'];
+        $snapshot->time = (int)$row['time'];
+        $snapshot->hostname = (string)($row['hostname'] ?? '');
+        $tags = (string)($row['tags'] ?? '');
+        $decoded = json_decode($tags, true);
+        $tags = is_array($decoded) ? $decoded : explode(',', $tags);
+        $snapshot->tags = array_values(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen'));
+        $summary = is_string($row['summary'] ?? null) ? json_decode($row['summary'], true) : null;
+        $snapshot->summary = is_array($summary) ? $summary : null;
+        // {domain: creation time}: lists of snapshots need no content
+        $domains = is_string($row['domains'] ?? null) ? json_decode($row['domains'], true) : null;
+        if (is_array($domains)) {
+            $snapshot->domains = [];
+            foreach ($domains as $domain => $created) {
+                $snapshot->domains[] = (string)$domain;
+                $snapshot->created[strtolower((string)$domain)] = is_numeric($created) ? (int)$created : null;
+            }
+        }
+        if (is_string($row['content'] ?? null) && $row['content'] !== '') {
+            $snapshot->content = self::decodeContent($row['content']);
+        }
+        return $snapshot;
+    }
+
+    /**
+     * `content` (gzip JSON) written by the runner: {"domains": {"<domain>": {"created": <unix>, "mailboxes": {"<username>": "<name>"}}}}.
+     * Keys lower case; null when broken.
+     */
+    public static function decodeContent(string $gzip): ?array
+    {
+        $json = @gzdecode($gzip);
+        $data = is_string($json) ? json_decode($json, true) : null;
+        if (!is_array($data) || !is_array($data['domains'] ?? null)) {
+            return null;
+        }
+        $content = [];
+        foreach ($data['domains'] as $domain => $entry) {
+            $mailboxes = is_array($entry) ? ($entry['mailboxes'] ?? []) : [];
+            $created = is_numeric($entry['created'] ?? null) ? (int)$entry['created'] : null;
+            $names = [];
+            foreach (is_array($mailboxes) ? $mailboxes : [] as $username => $name) {
+                $names[strtolower((string)$username)] = is_scalar($name) ? (string)$name : '';
+            }
+            $content[strtolower((string)$domain)] = ['created' => $created, 'mailboxes' => $names];
+        }
+        return $content;
+    }
+
+    /** Domains of the snapshot: those of its content and of its manifest (an export may have failed) */
+    public function domainList(): array
+    {
+        return array_values(array_unique(array_merge(array_map('strval', array_keys($this->content ?? [])), array_map('strtolower', $this->domains ?? []))));
+    }
+
+    /** @return ?array<string, string> username => name of the domain's mailboxes, null when the content does not list the domain */
+    public function mailboxesOf(string $domain): ?array
+    {
+        return $this->content[strtolower($domain)]['mailboxes'] ?? null;
+    }
+
+    /** Creation time of the domain when the snapshot was made, null when not recorded */
+    public function createdOf(string $domain): ?int
+    {
+        return $this->content[strtolower($domain)]['created'] ?? $this->created[strtolower($domain)] ?? null;
+    }
+
+    public function hasDomain(string $domain): bool
+    {
+        return in_array(strtolower($domain), $this->domainList(), true);
+    }
+
+    /**
+     * RESTIC_BACKUP_MODE of the backup (tag mode:<mode>, none for all): all, data (without the mailcow directory) or
+     * config (without mails)
+     */
+    public function mode(): string
+    {
+        foreach ($this->tags as $tag) {
+            if (str_starts_with($tag, 'mode:')) {
+                return substr($tag, 5);
+            }
+        }
+        return 'all';
+    }
+
+    /** Snapshots not made by mailcow know neither domains nor mailboxes */
+    public function knowsDomains(): bool
+    {
+        return $this->content !== null || $this->domains !== null;
+    }
+
+    /** Entry of the API for admins */
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->id,
+            'short_id' => $this->shortId,
+            'time' => $this->time,
+            'hostname' => $this->hostname,
+            'tags' => $this->tags,
+            'summary' => $this->summary,
+            'domains' => $this->domains,
+            'mode' => $this->mode(),
+        ];
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/ExportRepository.php

@@ -0,0 +1,169 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * Reads what the export of a domain holds: the domain with its settings, its aliases and a page of its mailboxes with
+ * every row that belongs to a mailbox. Rows are read with all their columns (SELECT *).
+ */
+class ExportRepository extends Repository
+{
+    /** @return string[] all domains */
+    public function domains(): array
+    {
+        return $this->query("SELECT `domain` FROM `domain` ORDER BY `domain`")->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    public function dbVersion(): string
+    {
+        return (string)$this->query("SELECT `version` FROM `versions` WHERE `application` = 'db_schema'")->fetchColumn();
+    }
+
+    /** @return array[] rows of a table matching column => value (table and column names from code) */
+    public function rows(string $table, string $column, string $value): array
+    {
+        self::identifier($table);
+        self::identifier($column);
+        return $this->query("SELECT * FROM `$table` WHERE `$column` = :value", ['value' => $value])->fetchAll();
+    }
+
+    /** @return string[] usernames of the domain's mailboxes */
+    public function usernames(string $domain): array
+    {
+        return $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain", ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** @return array[] the domain's mailbox rows after $after in username order (keyset paging), at most $limit */
+    public function mailboxesAfter(string $domain, string $after, int $limit): array
+    {
+        return $this->query("SELECT * FROM `mailbox` WHERE `domain` = :domain AND `username` > :after ORDER BY `username` LIMIT " . max(1, $limit),
+            ['domain' => $domain, 'after' => $after])->fetchAll();
+    }
+
+    /** @return array[] alias rows of the domain and its alias domains */
+    public function aliases(string $domain): array
+    {
+        return $this->query("SELECT * FROM `alias` WHERE `domain` = :domain
+            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target) ORDER BY `id`",
+            ['domain' => $domain, 'target' => $domain])->fetchAll();
+    }
+
+    /** @return array[] alias rows of any domain whose goto may name a mailbox of the domain (the caller checks) */
+    public function aliasesPointingTo(string $domain): array
+    {
+        return $this->query("SELECT * FROM `alias` WHERE `goto` LIKE :pattern ORDER BY `id`",
+            ['pattern' => '%@' . addcslashes($domain, '%_\\') . '%'])->fetchAll();
+    }
+
+    /**
+     * @param string $table  table with rows of mailboxes
+     * @param string $column its column holding the username
+     * @param string[] $usernames
+     * @return array[] rows of these mailboxes
+     */
+    public function rowsOfMailboxes(string $table, string $column, array $usernames): array
+    {
+        self::identifier($table);
+        self::identifier($column);
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT * FROM `$table` WHERE `$column` IN ($in)", $params)->fetchAll();
+    }
+
+    /** @return array[] username, id, name, domain, source of the group memberships of these mailboxes */
+    public function groupMemberships(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT `gm`.`username`, `g`.`id`, `g`.`name`, `g`.`domain`, `gm`.`source` FROM `mailbox_group_member` `gm`
+            JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id` WHERE `gm`.`username` IN ($in) ORDER BY `g`.`id`", $params)->fetchAll();
+    }
+
+    /**
+     * @param string $table  mail_policy or retention_rule
+     * @param string $member its member table
+     * @param string $key    the member table's column with the id
+     * @return array[] username, id, name, domain of the direct assignments of these mailboxes
+     */
+    public function assignments(string $table, string $member, string $key, array $usernames): array
+    {
+        self::identifier($table);
+        self::identifier($member);
+        self::identifier($key);
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT `a`.`username`, `p`.`id`, `p`.`name`, `p`.`domain` FROM `$member` `a`
+            JOIN `$table` `p` ON `p`.`id` = `a`.`$key` WHERE `a`.`username` IN ($in) ORDER BY `p`.`id`", $params)->fetchAll();
+    }
+
+    /** @return array[] sync jobs of these mailboxes (without their log) with host1, port1 and name of the source */
+    public function syncJobs(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        $rows = $this->query("SELECT `i`.*, `s`.`host1` AS `source_host1`, `s`.`port1` AS `source_port1`, `s`.`name` AS `source_name`
+            FROM `imapsync` `i` LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` IN ($in) ORDER BY `i`.`id`", $params)->fetchAll();
+        foreach ($rows as &$row) {
+            unset($row['returned_text']);
+        }
+        return $rows;
+    }
+
+    /** @return array[] BCC maps of the whole domain (local_dest "@domain", older rows "domain") */
+    public function domainBccMaps(string $domain): array
+    {
+        return $this->query("SELECT * FROM `bcc_maps` WHERE `local_dest` IN (:domain, :at)", ['domain' => $domain, 'at' => '@' . $domain])->fetchAll();
+    }
+
+    /**
+     * @param string $table  mail_policy or retention_rule
+     * @param string $member its member table
+     * @param string $key    the member table's column with the id
+     * @return array[] rows of the domain, each with `group_members`: the names of the groups it is assigned to
+     */
+    public function domainRules(string $table, string $member, string $key, string $domain): array
+    {
+        self::identifier($table);
+        self::identifier($member);
+        self::identifier($key);
+        $rows = $this->query("SELECT * FROM `$table` WHERE `domain` = :domain ORDER BY `id`", ['domain' => $domain])->fetchAll();
+        foreach ($rows as &$row) {
+            $row['group_members'] = $this->query("SELECT `g`.`name` FROM `$member` `a` JOIN `mailbox_group` `g` ON `g`.`id` = `a`.`group_id`
+                WHERE `a`.`$key` = :id ORDER BY `g`.`name`", ['id' => $row['id']])->fetchAll(PDO::FETCH_COLUMN);
+        }
+        return $rows;
+    }
+
+    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
+    private static function in(array $values): array
+    {
+        $names = [];
+        $params = [];
+        foreach (array_values($values) as $i => $value) {
+            $names[] = ':in' . $i;
+            $params['in' . $i] = $value;
+        }
+        return [implode(', ', $names), $params];
+    }
+
+    /** Table and column names come from code only; this keeps it that way */
+    private static function identifier(string $name): void
+    {
+        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
+            throw new \InvalidArgumentException('invalid identifier');
+        }
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/JobRepository.php

@@ -0,0 +1,176 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Backup\Model\Job;
+use Mailcow\Core\Context;
+use Mailcow\Core\Repository;
+use Mailcow\Core\UserError;
+use PDO;
+
+/**
+ * Table `backup_job` (the runners claim, log and finish the jobs) and the lookups a new restore job needs.
+ */
+class JobRepository extends Repository
+{
+    private const COLUMNS = "`id`, `type`, `state`, `step`, `params`, `result`, `error`, `domain`, `requested_by`,
+        UNIX_TIMESTAMP(`created`) AS `created`, UNIX_TIMESTAMP(`started`) AS `started`, UNIX_TIMESTAMP(`finished`) AS `finished`";
+    /** Serializes the check for active jobs and the insert of a new one */
+    private const LOCK = 'mailcow_backup_job';
+
+    public function find(int $id, bool $withLog = false): ?Job
+    {
+        $row = $this->query("SELECT " . self::COLUMNS . ($withLog ? ", `log`" : "") . " FROM `backup_job` WHERE `id` = :id", ['id' => $id])->fetch();
+        return is_array($row) ? Job::fromRow($row) : null;
+    }
+
+    /**
+     * @param ?string $type only jobs of this type
+     * @return Job[] newest first: all jobs for admins, the restores of their domains (since the domain was created) for
+     *         domain admins
+     */
+    public function visible(Context $context, int $limit, ?string $type = null): array
+    {
+        if ($context->hasAdminRights()) {
+            [$where, $params] = ['1=1', []];
+        } elseif ($context->isDomainAdmin()) {
+            // restores of their domains, requested since the domain was created (not those of a former domain of that name)
+            [$where, $params] = ["`type` = 'restore' AND `domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :da)
+              AND `created` >= (SELECT `d`.`created` FROM `domain` `d` WHERE `d`.`domain` = `backup_job`.`domain`)", ['da' => $context->username]];
+        } else {
+            return [];
+        }
+        if ($type !== null) {
+            $where .= " AND `type` = :type";
+            $params['type'] = $type;
+        }
+        return array_map(fn(array $row): Job => Job::fromRow($row),
+            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE $where ORDER BY `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll());
+    }
+
+    /** @return Job[] queued and running jobs */
+    public function active(): array
+    {
+        return array_map(fn(array $row): Job => Job::fromRow($row),
+            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `state` IN ('queued', 'running') ORDER BY `id`")->fetchAll());
+    }
+
+    /** @return array<string, Job> type => its newest job */
+    public function latestOfEachType(): array
+    {
+        $jobs = [];
+        foreach ($this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `id` IN (SELECT MAX(`id`) FROM `backup_job` GROUP BY `type`)")->fetchAll() as $row) {
+            $jobs[(string)$row['type']] = Job::fromRow($row);
+        }
+        return $jobs;
+    }
+
+    /** A backup job was created by the schedule since this unix time */
+    public function scheduledSince(int $since): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `backup_job` WHERE `type` = 'backup' AND `requested_by` = 'schedule' AND `created` >= FROM_UNIXTIME(:since) LIMIT 1",
+            ['since' => $since])->fetchColumn();
+    }
+
+    /**
+     * Inserts the job after $check(active jobs) passed; $check throws a UserError on a conflict. A named lock keeps two
+     * requests from queueing the same work at once.
+     *
+     * @param callable(Job[]): void $check
+     * @throws UserError
+     */
+    public function insertExclusive(Job $job, callable $check): int
+    {
+        if ((int)$this->query("SELECT GET_LOCK('" . self::LOCK . "', 10)")->fetchColumn() !== 1) {
+            throw new UserError('backup_job_active', [$job->type]);
+        }
+        try {
+            $check($this->active());
+            $job->id = $this->insertRow('backup_job', $job->toRow());
+            return $job->id;
+        } finally {
+            $this->query("SELECT RELEASE_LOCK('" . self::LOCK . "')");
+        }
+    }
+
+    /**
+     * Cancels a queued or running job; the restic runner stops at its next check. A restore in the dovecot step
+     * (mails, mails_running) cannot be cancelled. False when the job finished or reached that step meanwhile.
+     */
+    public function cancel(int $id): bool
+    {
+        return $this->query("UPDATE `backup_job` SET `state` = 'cancelled', `finished` = NOW() WHERE `id` = :id
+            AND `state` IN ('queued', 'running') AND NOT (`type` = 'restore' AND COALESCE(`step`, '') IN ('mails', 'mails_running'))",
+            ['id' => $id])->rowCount() > 0;
+    }
+
+    public function isDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+    }
+
+    /** Domain of a mailbox, null if there is none */
+    public function mailboxDomain(string $username): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username", ['username' => $username])->fetchColumn();
+        return is_string($domain) ? strtolower($domain) : null;
+    }
+
+    /** Unix time the domain was created, null if it does not exist */
+    public function domainCreated(string $domain): ?int
+    {
+        $created = $this->query("SELECT UNIX_TIMESTAMP(`created`) FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+        return ($created === false || $created === null) ? null : (int)$created;
+    }
+
+    /**
+     * @param string[] $domains
+     * @return array{0: array<string, true>, 1: array<string, array<string, true>>} the domains that exist, and per domain
+     *         the usernames of its mailboxes; one query each per 500 domains
+     */
+    public function domainsAndUsernames(array $domains): array
+    {
+        $existing = [];
+        $usernames = [];
+        foreach (array_chunk(array_values(array_unique($domains)), 500) as $chunk) {
+            $params = [];
+            foreach ($chunk as $i => $domain) {
+                $params['d' . $i] = $domain;
+            }
+            $in = implode(', ', array_map(fn(string $name): string => ':' . $name, array_keys($params)));
+            foreach ($this->query("SELECT `domain` FROM `domain` WHERE `domain` IN ($in)", $params)->fetchAll(PDO::FETCH_COLUMN) as $domain) {
+                $existing[strtolower((string)$domain)] = true;
+            }
+            foreach ($this->query("SELECT `domain`, `username` FROM `mailbox` WHERE `domain` IN ($in)", $params)->fetchAll() as $row) {
+                $usernames[strtolower((string)$row['domain'])][strtolower((string)$row['username'])] = true;
+            }
+        }
+        return [$existing, $usernames];
+    }
+
+    /** @return array<string, string> username => name of the mailboxes of a domain */
+    public function mailboxNames(string $domain): array
+    {
+        $names = [];
+        foreach ($this->query("SELECT `username`, `name` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`", ['domain' => $domain])->fetchAll() as $row) {
+            $names[strtolower((string)$row['username'])] = (string)$row['name'];
+        }
+        return $names;
+    }
+
+    /** @return string[] all mailboxes of a domain */
+    public function mailboxesOf(string $domain): array
+    {
+        return array_map('strtolower', $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`",
+            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /** @return string[] alias domains of a domain */
+    public function aliasDomainsOf(string $domain): array
+    {
+        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
+            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+}