[Web][Restic][Dovecot][Dockerapi] Add restic backups with restore in the UI

- Scheduled, encrypted backups to S3, SFTP, REST or a local folder (new restic-mailcow container)
- Restore of mailboxes and whole domains in the UI, also deleted ones, for admins and domain admins
- Scope set by RESTIC_BACKUP_MODE in mailcow.conf (all, data, config)
- Optional backup before updates (update.sh), helper-scripts/restic.sh for a full restore

_modules/scripts/backup.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# _modules/scripts/backup.sh
+# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
+# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
+
+# Backup with restic before the update when it is set up (target, confirmed password, initialized repository):
+# interactive: asked for; --backup: without asking; --force: none (with --backup: without asking, a failure or a backup
+# that cannot run stops the update; only "not set up" continues)
+backup_failed() {
+  echo -e "\e[31m${1}\e[0m"
+  if [ "$FORCE" ]; then
+    echo "The update was started with --force --backup, it stops here. Nothing was changed."
+    exit 1
+  fi
+  read -r -p "Continue the update without a backup? [y/N] " backup_response
+  if [[ ! "${backup_response}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    echo "OK, exiting."
+    exit 1
+  fi
+}
+
+# Whether to back up (asked with the other questions); the backup itself runs before the first change of the checkout
+backup_decide() {
+  local output rc
+  RUN_BACKUP=
+  if [ "$FORCE" ] && [ ! "$BACKUP" ]; then
+    return 0
+  fi
+  if [[ -z "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
+    output="the backup container does not run"
+    rc=1
+  else
+    # exit code 0: set up, 2: not set up, others: error (e.g. credentials this container cannot read)
+    output=$($COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --configured 2>&1 < /dev/null)
+    rc=$?
+  fi
+  if [[ ${rc} -eq 2 ]]; then
+    [ "$BACKUP" ] && echo -e "\e[33mNo backup before the update: ${output}.\e[0m"
+    return 0
+  elif [[ ${rc} -ne 0 ]]; then
+    if [ "$BACKUP" ]; then
+      backup_failed "No backup before the update possible: ${output}"
+    elif [[ -n "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
+      echo -e "\e[33mNo backup before the update: ${output}\e[0m"
+    fi
+    return 0
+  fi
+  if [ ! "$BACKUP" ]; then
+    read -r -p "Create a backup before the update? [Y/n] " backup_response
+    if [[ "${backup_response}" =~ ^([nN][oO]|[nN])$ ]]; then
+      echo "OK, no backup."
+      return 0
+    fi
+  fi
+  RUN_BACKUP=y
+}
+
+# The backup chosen by backup_decide, once
+backup_run() {
+  if [ ! "$RUN_BACKUP" ]; then
+    return 0
+  fi
+  RUN_BACKUP=
+  if [[ "${RESTIC_BACKUP_MODE}" =~ ^(data|config)$ ]]; then
+    echo -e "\e[33mRESTIC_BACKUP_MODE=${RESTIC_BACKUP_MODE}: the backup holds no complete mailcow directory, a restore does not bring back this mailcow version.\e[0m"
+  fi
+  echo -e "\e[32mCreating a backup before the update...\e[0m"
+  if ! $COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --backup < /dev/null; then
+    backup_failed "The backup before the update failed, see the output above and the backup jobs in the mailcow UI."
+  fi
+}

_modules/scripts/core.sh

@@ -192,6 +192,7 @@ detect_major_update() {
       "2025-02"
       "2025-03"
       "2025-09"
+      "2026-11"
     )
 
     current_version=""

_modules/scripts/new_options.sh

@@ -60,6 +60,9 @@ adapt_new_options() {
   "ACME_DNS_CHALLENGE"
   "ACME_DNS_PROVIDER"
   "ACME_ACCOUNT_EMAIL"
+  "NGINX_MAX_BODY_SIZE"
+  "RESTIC_LOCAL_PATH"
+  "RESTIC_BACKUP_MODE"
   )
 
   sed -i --follow-symlinks '$a\' mailcow.conf
@@ -309,6 +312,24 @@ adapt_new_options() {
             echo '# Account email for ACME DNS-01 challenge registration' >> mailcow.conf
             echo 'ACME_ACCOUNT_EMAIL=me@example.com' >> mailcow.conf
             ;;
+        NGINX_MAX_BODY_SIZE)
+            echo '# Maximum request size for SOGo webmail, CalDAV/CardDAV and ActiveSync, e.g. 128m or 1g - 0 means unlimited' >> mailcow.conf
+            echo '# Keep it above the message size limit of Postfix (100 MB by default). The mailcow UI and API stay limited to 64m.' >> mailcow.conf
+            echo 'NGINX_MAX_BODY_SIZE=128m' >> mailcow.conf
+            ;;
+        RESTIC_LOCAL_PATH)
+            echo '# Folder on the host for backups to the target "local" (backup settings in the mailcow UI), as an absolute path,' >> mailcow.conf
+            echo '# e.g. a NAS mounted from another disk. The repository is created in a subfolder of it.' >> mailcow.conf
+            echo '# Empty: the Docker volume restic-repo-vol-1 of mailcow, on the same disk as the mailcow data.' >> mailcow.conf
+            echo 'RESTIC_LOCAL_PATH=' >> mailcow.conf
+            ;;
+        RESTIC_BACKUP_MODE)
+            echo '# What the backups of the mailcow UI hold:' >> mailcow.conf
+            echo '# - all (default),' >> mailcow.conf
+            echo '# - data (vmail, vmail-index, database, redis, mail crypt keys and mailcow.conf)' >> mailcow.conf
+            echo '# - config (database, redis, mail crypt keys, mailcow.conf and data/conf).' >> mailcow.conf
+            echo 'RESTIC_BACKUP_MODE=all' >> mailcow.conf
+            ;;
         *)
             echo "${option}=" >> mailcow.conf
             ;;

data/Dockerfiles/dockerapi/main.py

@@ -159,6 +159,9 @@ async def post_containers(container_id : str, post_action : str, request: Reques
       api_call_method = getattr(dockerapi, api_call_method_name, lambda container_id: Response(content=json.dumps({'type': 'danger', 'msg':'container_post - unknown api call' }, indent=4), media_type="application/json"))
 
       dockerapi.logger.info("api call: %s, container_id: %s" % (api_call_method_name, container_id))
+      if getattr(api_call_method, 'long_running', False):
+        # runs for minutes (e.g. sogo-tool of a backup restore): in a worker thread, the other api calls go on
+        return await asyncio.to_thread(api_call_method, request_json, container_id=container_id)
       return api_call_method(request_json, container_id=container_id)
     except Exception as e:
       dockerapi.logger.error("error - container_post: %s" % str(e))

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -6,9 +6,15 @@ import time
 import json
 import asyncio
 import platform
+import docker
 from datetime import datetime
 from fastapi import FastAPI, Response, Request
 
+def long_running(method):
+  """Marks an api call that runs for minutes: main.py runs it in a worker thread, the other api calls go on"""
+  method.long_running = True
+  return method
+
 class DockerApi:
   def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
     self.redis_client = redis_client
@@ -318,7 +324,7 @@ class DockerApi:
 
     if 'username' in request_json and 'script_name' in request_json:
       for container in self.sync_docker_client.containers.list(filters=filters):
-        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]
+        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' -- '" + request_json['script_name'].replace("'", "'\\''") + "'"]
         sieve_return = container.exec_run(cmd)
         return self.exec_run_handler('utf8_text_only', sieve_return)
   # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
@@ -445,6 +451,35 @@ class DockerApi:
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
       init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
       return self.exec_run_handler('generic', init_return)
+  # api call: container_post - post_action: exec - cmd: sogo - task: restore
+  # Resets calendars, address books and preferences of a user to the sogo-tool backup file <dir>/<username> in a
+  # restore batch of the backup (/restore/<job>/b<batch>/sogo_backup, only the sogo user can read it); values are
+  # checked and passed as arguments, never through the shell. Runs in a worker thread (main.py) with its own client.
+  @long_running
+  def container_post__exec__sogo__restore(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    directory = str(request_json.get('dir', ''))
+    username = str(request_json.get('username', ''))
+    if (not re.fullmatch(r'/restore/[0-9]+/b[0-9]+/sogo_backup', directory)
+      or not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+', username)):
+      return Response(content=json.dumps({ 'type': 'danger', 'msg': 'dir and username required' }, indent=4), media_type="application/json")
+    # the folder: no symlink, uid 999, mode 0700; the file: plain, uid 999, mode 0600. sogo-tool restores folders and
+    # preferences in two runs, both stopped before the runner gives up waiting (30 minutes)
+    script = ('[ -d "$1" ] && [ ! -L "$1" ] && [ "$(realpath -e -- "$1")" = "$1" ] && [ "$(stat -c "%u %a" -- "$1")" = "999 700" ] '
+      '&& [ -f "$1/$2" ] && [ ! -L "$1/$2" ] && [ "$(stat -c "%u %a" -- "$1/$2")" = "999 600" ] '
+      '|| { echo "invalid directory or file"; exit 1; }; '
+      'timeout -k 30 850 sogo-tool restore -f ALL "$1" "$2" && exec timeout -k 30 850 sogo-tool restore -p "$1" "$2"')
+    client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version='auto', timeout=1800)
+    try:
+      for container in client.containers.list(filters=filters):
+        restore_return = container.exec_run(["/bin/bash", "-c", script, "restore", directory, username], user='sogo')
+        return self.exec_run_handler('generic', restore_return)
+    finally:
+      client.close()
   # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
   # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
   def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
@@ -491,7 +526,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' -- '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -510,7 +545,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' -- '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -534,7 +569,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -568,7 +603,7 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
   # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
@@ -585,7 +620,7 @@ class DockerApi:
       rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
       if not user or id in ('user=', 'group='):
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
-      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
+      action = 'doveadm acl set -u "$ACL_USER" -- "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" -- "$folder" "$ACL_ID"'
       script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
         'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
       acl_return = container.exec_run(["/bin/bash", "-c", script],
@@ -601,7 +636,7 @@ class DockerApi:
 
     for container in self.sync_docker_client.containers.list(filters=filters):
       users = [str(user) for user in request_json.get('users', []) if str(user)]
-      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush", "--"] + users)
       return self.exec_run_handler('generic', flush_return)
 
   # Rights doveadm acl set accepts from the API

data/Dockerfiles/dovecot/Dockerfile

@@ -150,6 +150,8 @@ COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
 COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
+COPY retention_runner.py /usr/local/bin/retention_runner.py
+COPY restore_runner.py /usr/local/bin/restore_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -281,6 +281,8 @@ chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
   /usr/local/bin/imapsync_runner.py \
+  /usr/local/bin/retention_runner.py \
+  /usr/local/bin/restore_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -306,13 +308,20 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
 # Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
-DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+# No error while the versions table is missing (fresh install, set -e)
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
   echo "Waiting for schema update..."
   sleep 5
-  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
 done
-echo "DB schema is ${DBV_NOW}"
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
+else
+  echo "DB schema is ${DBV_NOW}"
+fi
 
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -1,10 +1,12 @@
 #!/usr/bin/python3
 
 import fcntl
+import ipaddress
 import json
 import os
 import re
 import signal
+import socket
 import subprocess
 import sys
 import tempfile
@@ -18,6 +20,15 @@ max_output = 16 * 1024 * 1024
 lock_file = run_dir + '/imapsync_busy.lock'
 # An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
 token_min_valid = 900
+# Custom params never passed, also when stored before (same list as CustomParams::BLOCKED in the web UI)
+blocked_params = {'abort', 'delete2foldersbutnot', 'delete2foldersonly', 'logdir', 'logfile', 'maxlinelengthcmd',
+  'oauthrefreshcmd1', 'oauthrefreshcmd2', 'passfile1', 'passfile2', 'pidfile', 'pipemess', 'regexflag', 'regexmess',
+  'regextrans2', 'showpasswords', 'skipmess', 'tests', 'testsdebug', 'testslive', 'testslive6', 'testsunit', 'tmpdir'}
+# imapsync only gets these variables, not the secrets of the container environment
+job_env_keys = ('PATH', 'LANG', 'LC_ALL', 'TZ')
+# A source must not point to mailcow itself (internal Docker network), host1 is checked before each run
+internal_nets = [ipaddress.ip_network(os.environ.get('IPV4_NETWORK', '172.22.1') + '.0/24', strict=False),
+  ipaddress.ip_network(os.environ.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'), strict=False)]
 # Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
 token_sync_interval = 60
 
@@ -25,7 +36,7 @@ token_sync_interval = 60
 next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
   i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
   i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
-  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
+  s.host1, s.port1, s.enc1, s.verify_cert, s.auth_type, s.oauth_flow
     FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
       WHERE i.active = 1
         AND s.active = 1
@@ -106,6 +117,28 @@ def defer_job(job_id, text, exit_status):
     (text, exit_status, job_id))
   deferred.add(job_id)
 
+# True when host resolves to loopback, link-local or the internal network; unknown hosts are left to imapsync
+def internal_host(host):
+  try:
+    addrs = {a[4][0] for a in socket.getaddrinfo(host, None)}
+  except (socket.gaierror, UnicodeError):
+    return False
+  for addr in addrs:
+    ip = ipaddress.ip_address(addr.split('%')[0])
+    if ip.version == 6 and ip.ipv4_mapped:
+      ip = ip.ipv4_mapped
+    if ip.is_loopback or ip.is_unspecified or ip.is_link_local or any(ip in net for net in internal_nets):
+      return True
+  return False
+
+# SNI takes host names only
+def ip_literal(host):
+  try:
+    ipaddress.ip_address(host)
+    return True
+  except ValueError:
+    return False
+
 def oauth_token(j):
   if j['oauth_flow'] == 'authorization_code':
     return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
@@ -150,6 +183,11 @@ def prepare_job(j, global_bw, files):
       f.write(content)
     return path
 
+  if internal_host(j['host1']):
+    finish_job(j['id'], 'Host1 %s is an address of mailcow itself, the sync job does not connect to it\n' % j['host1'], 0,
+      'EXIT_CONNECTION_FAILURE_HOST1')
+    return None
+
   auth_type = j['auth_type'] or 'PLAIN'
   oauth = None
   if auth_type == 'XOAUTH2':
@@ -174,7 +212,11 @@ def prepare_job(j, global_bw, files):
   except ValueError:
     pairs = []
   for p in pairs if isinstance(pairs, list) else []:
-    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
+    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']) \
+        or p['o'] in blocked_params:
+      continue
+    # The certificate check protects the OAuth token of the source, a job must not turn it off
+    if oauth and p['o'] == 'sslargs1':
       continue
     value = '' if p.get('v') is None else str(p['v'])
     custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
@@ -205,7 +247,13 @@ def prepare_job(j, global_bw, files):
       cmd.append(opt)
   enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
   if enc1:
-    cmd.append(enc1)
+    # Mail::IMAPClient starts TLS without a host name: check the certificate against host1, not its address
+    cmd += [enc1, '--sslargs1', 'SSL_verifycn_scheme=imap', '--sslargs1', 'SSL_verifycn_name=' + j['host1']]
+    if not ip_literal(j['host1']):
+      cmd += ['--sslargs1', 'SSL_hostname=' + j['host1']]
+    # imapsync does not check certificates by default; custom sslargs1 of password jobs can still change it
+    if j['verify_cert'] == 1:
+      cmd += ['--sslargs1', 'SSL_verify_mode=1']
   cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
   cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
   if j['dry'] == 1:
@@ -216,8 +264,10 @@ def prepare_job(j, global_bw, files):
 
 # Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
 def start_job(job):
+  env = {k: os.environ[k] for k in job_env_keys if k in os.environ}
+  env['HOME'] = run_dir
   with open(job['outfile'], 'wb') as out:
-    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
+    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, env=env)
 
 # Store the result of a finished worker and remove its temp files
 def reap_job(job):

data/Dockerfiles/dovecot/restore_runner.py

@@ -0,0 +1,702 @@
+#!/usr/bin/python3
+# Mail part of restores (ofelia, dovecot container, as vmail): the restic container restores a batch of mailboxes
+# into /restore/<job id>/b<n> and sets the job to step 'mails'. Per mailbox this imports the mails folder by folder
+# (into "Restored <snapshot time>" or only the missing ones into the original folders), adds IMAP ACLs and missing
+# Sieve scripts, merges the results into the job and sets step 'mails_done'; the restic runner then removes the staging
+# and goes on with the next batch. Nothing that exists is changed or deleted.
+
+import base64
+import fcntl
+import json
+import os
+import re
+import smtplib
+import stat
+import subprocess
+import sys
+import threading
+import time
+from email.message import EmailMessage
+from email.utils import formatdate, make_msgid
+import MySQLdb
+import MySQLdb.cursors
+
+lock_file = '/tmp/restore_runner.lock'
+restore_dir = '/restore'
+max_log = 1000000
+# import, moves and the removal of the temporary folder ignore the quota (a restore may need more than is free),
+# the quota is recalculated afterwards
+no_quota = ['-o', 'plugin/quota=']
+# uid ranges per import (one argument stays far below the 128 KiB limit), folders per fetch and status
+uid_ranges = 2000
+folder_batch = 200
+# the restic runner waits for the batch while this heartbeat is fresh
+heartbeat_interval = 30
+# seconds to wait for the next batch of a restore
+batch_wait = 120
+rights_by_letter = {'l': 'lookup', 'r': 'read', 'w': 'write', 's': 'write-seen', 't': 'write-deleted', 'i': 'insert',
+  'p': 'post', 'e': 'expunge', 'k': 'create', 'x': 'delete', 'a': 'admin'}
+components_here = {'mails', 'imap_acl', 'filters', 'sieve_activate'}
+
+conn = None
+
+
+def connect():
+  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
+    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
+
+# Reconnect when the server dropped the connection (imports can outlast wait_timeout)
+def query(sql, args=None):
+  global conn
+  try:
+    conn.ping()
+  except (AttributeError, MySQLdb.Error):
+    conn = connect()
+  cur = conn.cursor(MySQLdb.cursors.DictCursor)
+  cur.execute(sql, args)
+  return cur
+
+# state_restore_seen while a batch runs, in its own connection
+class Heartbeat(threading.Thread):
+  def __init__(self):
+    super().__init__(daemon=True)
+    self.done = threading.Event()
+
+  def run(self):
+    db = None
+    while not self.done.is_set():
+      try:
+        if db is None:
+          db = connect()
+        db.cursor().execute("""INSERT INTO backup_setting (name, value) VALUES ('state_restore_seen', %s)
+          ON DUPLICATE KEY UPDATE value = VALUES(value)""", (str(int(time.time())),))
+      except MySQLdb.Error:
+        db = None
+      self.done.wait(heartbeat_interval)
+
+def log(job_id, text):
+  print('restore job %d: %s' % (job_id, text), flush=True)
+  line = '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), text)
+  query("UPDATE backup_job SET log = RIGHT(CONCAT(IFNULL(log, ''), %s), %s) WHERE id = %s", (line, max_log, job_id))
+
+def save(job_id, result):
+  query("UPDATE backup_job SET result = %s WHERE id = %s", (json.dumps(result), job_id))
+
+def doveadm(*args, data=None, check=True, partial=False):
+  result = subprocess.run(['doveadm', *args], input=data, stdin=None if data is not None else subprocess.DEVNULL,
+    capture_output=True, timeout=86400)
+  # mailbox status fails for folders that cannot be opened (implicit parents); the others are listed
+  if check and result.returncode != 0 and not (partial and result.stdout.strip()):
+    # mail_log writes an info line per message, the errors are the other lines
+    lines = [l for l in result.stderr.decode('utf-8', 'replace').splitlines() if ': Info: ' not in l]
+    raise RuntimeError('\n'.join(lines)[-500:] or 'doveadm exit code %d' % result.returncode)
+  return result.stdout.decode('utf-8', 'replace')
+
+def doveadm_json(*args, partial=False):
+  out = doveadm('-f', 'json', *args, partial=partial)
+  return json.loads(out) if out.strip() else []
+
+# free disk space that must stay after the import: max(2 GiB, 5 % of the filesystem), a full disk stops mail
+# (disk_reserve, mib and count_errors as in restic_runner.py of the backup image; tests/python/backup compares them)
+def disk_reserve(total):
+  return max(2 << 30, total // 20)
+
+def import_space_ok(need, free, total):
+  return need * 11 // 10 + disk_reserve(total) <= free
+
+def mib(size):
+  return '%.1f MiB' % (size / 1048576)
+
+def component(errors=None):
+  return {'added': 0, 'skipped': 0, 'errors': errors or []}
+
+# Folder names on disk: modified UTF-7, listescape writes . / \ as \xx
+def mutf7_decode(text):
+  def utf16(m):
+    if not m.group(1):
+      return '&'
+    data = m.group(1).replace(',', '/')
+    return base64.b64decode(data + '=' * (-len(data) % 4)).decode('utf-16-be')
+  return re.sub(r'&([A-Za-z0-9+,]*)-', utf16, text)
+
+def unescape(part):
+  return mutf7_decode(re.sub(r'\\([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1), 16)), part))
+
+# Maildir++: the maildir is INBOX, ".A.B" is A/B
+def folder_name(entry):
+  return '/'.join(unescape(part) for part in entry[1:].split('.')) if entry else 'INBOX'
+
+def is_dir(path):
+  try:
+    return stat.S_ISDIR(os.lstat(path).st_mode)
+  except OSError:
+    return False
+
+def read_file(path):
+  fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+  with os.fdopen(fd, 'rb') as f:
+    return f.read()
+
+# Prefixes of the shared and public namespaces, their folders belong to others
+def shared_prefixes():
+  prefixes, namespace, depth = {'Shared/'}, None, 0
+  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
+  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
+    stripped = line.strip()
+    if stripped.endswith('{'):
+      depth += 1
+      if depth == 1 and stripped.startswith('namespace'):
+        namespace = {}
+    elif stripped == '}':
+      if depth == 1 and namespace is not None:
+        if namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
+          prefixes.add(namespace['prefix'].split('%')[0])
+        namespace = None
+      depth = max(depth - 1, 0)
+    elif depth == 1 and namespace is not None and '=' in stripped:
+      key, value = (part.strip() for part in stripped.split('=', 1))
+      namespace[key] = value
+  return tuple(prefix for prefix in prefixes if prefix)
+
+# name => guid of the user's own folders (implicit parents have none); the names are looked up literally
+def own_folders(user, prefixes):
+  names = [n for n in doveadm('mailbox', 'list', '-u', user).splitlines() if n and not n.startswith(prefixes)]
+  wanted, guids = set(names), {}
+  for i in range(0, len(names), folder_batch):
+    for row in doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', *names[i:i + folder_batch], partial=True):
+      if row.get('mailbox') in wanted and re.fullmatch(r'[0-9a-f]{32}', row.get('guid') or ''):
+        guids[row['mailbox']] = row['guid']
+  return guids
+
+def free_name(base, existing):
+  name, count = base, 1
+  while name in existing or any(folder.startswith(name + '/') for folder in existing):
+    count += 1
+    name = '%s (%d)' % (base, count)
+  return name
+
+# Path of the maildir now (/var/vmail/<domain>/<local>/Maildir), None for unknown or inactive mailboxes
+def mail_location(user):
+  result = subprocess.run(['doveadm', 'user', '-f', 'mail', '--', user], capture_output=True, text=True)
+  m = re.match(r'maildir:([^:]+)', result.stdout.strip()) if result.returncode == 0 else None
+  return m.group(1).rstrip('/') if m else None
+
+# The restored maildir of a mailbox (may hold only ACL files), None if the snapshot has nothing of it
+def staged_maildir(staging, home, location):
+  for path in (staging + location, home + '/' + os.environ.get('MAILDIR_SUB', 'Maildir'), home):
+    if is_dir(path):
+      return path
+  return None
+
+def uid_chunks(uids):
+  ranges, start, prev = [], None, None
+  for uid in sorted(uids):
+    if start is None:
+      start = prev = uid
+    elif uid == prev + 1:
+      prev = uid
+    else:
+      ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
+      start = prev = uid
+  if start is not None:
+    ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
+  return [','.join(ranges[i:i + uid_ranges]) for i in range(0, len(ranges), uid_ranges)]
+
+# Same message: same Message-ID, without one the same Date and size
+def message_key(row):
+  message_id = re.sub(r'\s+', '', row.get('hdr.message-id') or '').strip('<>')
+  if message_id:
+    return 'id:' + message_id
+  return 'date:%s:%s' % (row.get('date.sent'), row.get('size.physical'))
+
+def present_keys(user, guids):
+  keys = set()
+  guids = list(guids)
+  for i in range(0, len(guids), folder_batch):
+    search = []
+    for guid in guids[i:i + folder_batch]:
+      search += (['OR'] if search else []) + ['mailbox-guid', guid]
+    keys.update(message_key(row) for row in doveadm_json('fetch', '-u', user, 'hdr.message-id date.sent size.physical',
+      *search))
+  return keys
+
+def location(path):
+  if ':' in path:
+    raise RuntimeError('unsupported path')
+  return 'maildir:%s:INDEX=MEMORY' % path
+
+# The folders of a restored maildir as standalone maildirs (their messages are the INBOX there): the subfolders are
+# moved out of the maildir, so every folder is imported without the name rules of the source (listescape)
+def split_folders(maildir, home):
+  folders, failed = [('INBOX', maildir)], []
+  out = home + '/_restore_folders'
+  for entry in sorted(os.listdir(maildir)):
+    path = os.path.join(maildir, entry)
+    if not entry.startswith('.') or entry in ('.', '..') or not is_dir(path) or not is_dir(path + '/cur'):
+      continue
+    try:
+      name = folder_name(entry)
+    except (ValueError, UnicodeDecodeError):
+      failed.append(entry)
+      continue
+    if not os.path.isdir(out):
+      os.mkdir(out, 0o700)
+    target = '%s/f%d' % (out, len(folders))
+    os.rename(path, target)
+    folders.append((name, target))
+  return folders, failed
+
+def source_messages(user, root):
+  return doveadm_json('-o', 'mail_location=' + location(root), 'fetch', '-u', user,
+    'uid hdr.message-id date.sent size.physical', 'mailbox', 'INBOX')
+
+def remove_tree(user, name):
+  doveadm(*no_quota, 'mailbox', 'delete', '-u', user, '-r', '-s', '--', name, check=False)
+
+def folder_guid(user, name):
+  rows = doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', name)
+  for row in rows:
+    if row.get('mailbox') == name:
+      return row['guid']
+  raise RuntimeError('folder not found after the import')
+
+def quota_room(user):
+  # free bytes of the storage quota, None without a limit
+  for row in doveadm_json('quota', 'get', '-u', user):
+    if row.get('type') == 'STORAGE' and str(row.get('limit', '-')).isdigit():
+      return (int(row['limit']) - int(row.get('value') or 0)) * 1024
+  return None
+
+# Subscriptions of the backup (subscriptions file: "V\t2", then names with \t between the levels)
+def backup_subscriptions(maildir):
+  try:
+    lines = read_file(maildir + '/subscriptions').decode('utf-8', 'replace').splitlines()
+  except OSError:
+    return []
+  if lines and lines[0].startswith('V\t'):
+    return ['/'.join(unescape(part) for part in line.split('\t')) for line in lines[1:] if line.strip()]
+  # version 1: storage names with the Maildir++ separator
+  return [folder_name('.' + line.strip()) for line in lines if line.strip()]
+
+def restore_subscriptions(user, wanted, prefixes):
+  if not wanted:
+    return 0
+  existing = set(own_folders(user, prefixes)) | {'INBOX'}
+  subscribed = set(doveadm('mailbox', 'list', '-u', user, '-s').splitlines())
+  missing = [name for name in dict.fromkeys(wanted) if name in existing and name not in subscribed]
+  if missing:
+    doveadm('mailbox', 'subscribe', '-u', user, '--', *missing)
+  return len(missing)
+
+# Mails folder by folder: original = only messages missing anywhere in the mailbox, into their original folders;
+# folder = all messages below "Restored <snapshot time>"
+def restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota):
+  result = component()
+  result.update(folders=0, folders_failed=0)
+  if not maildir or not is_dir(maildir + '/cur'):
+    # a key the UI translates
+    result['note'] = 'no_mails'
+    return result
+  tmp = 'Restoring %d' % job_id
+  # a leftover of an interrupted run goes first, it is never counted as present
+  remove_tree(user, tmp)
+  subscriptions = backup_subscriptions(maildir)
+  folders, broken = split_folders(maildir, home)
+  for entry in broken:
+    result['errors'].append('folder %s: name not readable' % entry)
+    result['folders_failed'] += 1
+  live = own_folders(user, prefixes)
+  present = present_keys(user, live.values()) if mode == 'original' else set()
+  plan, need = [], 0
+  for name, root in folders:
+    try:
+      rows = source_messages(user, root)
+    except RuntimeError as e:
+      result['errors'].append('%s: %s' % (name, e))
+      result['folders_failed'] += 1
+      continue
+    wanted = [row for row in rows if mode != 'original' or message_key(row) not in present]
+    result['skipped'] += len(rows) - len(wanted)
+    if wanted:
+      size = sum(int(row.get('size.physical') or 0) for row in wanted)
+      plan.append((name, root, [int(row['uid']) for row in wanted], size))
+      need += size
+  if not plan:
+    if mode == 'original':
+      result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
+    return result
+  vfs = os.statvfs('/var/vmail')
+  free, total = vfs.f_bavail * vfs.f_frsize, vfs.f_blocks * vfs.f_frsize
+  if not import_space_ok(need, free, total):
+    raise RuntimeError('not enough free disk space for the mails: %s needed plus %s reserve, %s free' % (
+      mib(need * 11 // 10), mib(disk_reserve(total)), mib(free)))
+  if check_quota:
+    room = quota_room(user)
+    if room is not None and need > room:
+      raise RuntimeError('the mails do not fit into the quota of the mailbox: %s needed, %s free' % (
+        mib(need), mib(max(room, 0))))
+  names = set(live) | {'INBOX'}
+  dest = None
+  if mode != 'original':
+    dest = free_name('Restored ' + snapshot_time, names)
+    doveadm('mailbox', 'create', '-u', user, '-s', '--', dest)
+    names.add(dest)
+    result['folder'] = dest
+  created = []
+  try:
+    for number, (name, root, uids, size) in enumerate(plan):
+      target = dest + '/' + name if dest else name
+      stage = '%s/f%d' % (tmp, number)
+      try:
+        for chunk in uid_chunks(uids):
+          doveadm(*no_quota, 'import', '-u', user, '-s', location(root), stage, 'mailbox', 'INBOX', 'uid', chunk)
+        guid = folder_guid(user, stage + '/INBOX')
+        if target not in names:
+          doveadm('mailbox', 'create', '-u', user, '-s', '--', target)
+          names.add(target)
+          created.append(target)
+        doveadm(*no_quota, 'move', '-u', user, '--', target, 'mailbox-guid', guid, 'all')
+        result['added'] += len(uids)
+        result['folders'] += 1
+      except RuntimeError as e:
+        result['errors'].append('%s: %s' % (name, e))
+        result['folders_failed'] += 1
+  finally:
+    remove_tree(user, tmp)
+  result['folders_created'] = len(created)
+  if mode == 'original':
+    result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
+  return result
+
+def acl_entries(path):
+  entries = []
+  try:
+    lines = read_file(path).decode('utf-8', 'replace').splitlines()
+  except OSError:
+    return entries
+  for line in lines:
+    parts = line.split()
+    # negative rights would take rights away
+    if len(parts) < 2 or parts[0].startswith('-'):
+      continue
+    rights = [rights_by_letter[c] for c in parts[1] if c in rights_by_letter] + [p for p in parts[2:] if p.startswith(':')]
+    if rights:
+      entries.append((parts[0], rights))
+  return entries
+
+# (folder, entries) of the dovecot-acl files of a restored maildir
+def maildir_acls(maildir):
+  acls = []
+  if not maildir or not is_dir(maildir):
+    return acls
+  for entry in [''] + sorted(e for e in os.listdir(maildir) if e.startswith('.') and e not in ('.', '..')):
+    path = os.path.join(maildir, entry, 'dovecot-acl') if entry else os.path.join(maildir, 'dovecot-acl')
+    if entry and not is_dir(os.path.join(maildir, entry)):
+      continue
+    try:
+      if stat.S_ISREG(os.lstat(path).st_mode):
+        acls.append((folder_name(entry), acl_entries(path)))
+    except (OSError, ValueError, UnicodeDecodeError):
+      continue
+  return acls
+
+# owner => (folder, entries) of every mailbox in the ACL staging (domain admins: only their domains)
+def staged_acls(acl_dir, allowed):
+  acls = {}
+  base = (acl_dir or '') + '/var/vmail'
+  sub = os.environ.get('MAILDIR_SUB', '')
+  if not acl_dir or not is_dir(base):
+    return acls
+  for domain in os.listdir(base):
+    if '.' not in domain or not is_dir(os.path.join(base, domain)) or (allowed is not None and domain.lower() not in allowed):
+      continue
+    for local in os.listdir(os.path.join(base, domain)):
+      home = os.path.join(base, domain, local)
+      maildir = os.path.join(home, sub) if sub and is_dir(os.path.join(home, sub)) else home
+      found = maildir_acls(maildir)
+      if found:
+        acls['%s@%s' % (local, domain)] = found
+  return acls
+
+class AclRestore:
+  def __init__(self, prefixes, allowed):
+    self.prefixes = prefixes
+    self.allowed = allowed
+    self.users, self.folders = {}, {}
+    # anyone/authenticated only when the configuration allows them
+    self.anyone = os.environ.get('ACL_ANYONE', 'disallow') == 'allow'
+
+  def in_scope(self, address):
+    return self.allowed is None or address.rpartition('@')[2].lower() in self.allowed
+
+  def user_exists(self, user):
+    if user not in self.users:
+      self.users[user] = query("SELECT 1 FROM mailbox WHERE username = %s", (user,)).fetchone() is not None
+    return self.users[user]
+
+  def identifier_ok(self, identifier):
+    if identifier.startswith('user='):
+      return self.user_exists(identifier[5:].lower())
+    # group=mcg_<id> belongs to the mailbox groups, the group grants cron keeps them
+    if identifier.startswith('group=mcg_'):
+      return False
+    if identifier in ('anyone', 'authenticated'):
+      return self.anyone
+    return True
+
+  def shown(self, identifier):
+    # domain admins do not see addresses of other domains
+    if identifier.startswith('user=') and not self.in_scope(identifier[5:]):
+      return 'user=(other domain)'
+    return identifier
+
+  def add(self, owner, folder, identifier, rights, result):
+    if not self.in_scope(owner):
+      result['skipped'] += 1
+      return
+    try:
+      if owner not in self.folders:
+        self.folders[owner] = set(own_folders(owner, self.prefixes)) | {'INBOX'}
+      if folder not in self.folders[owner] or not self.identifier_ok(identifier):
+        result['skipped'] += 1
+        return
+      have = set()
+      for row in doveadm_json('acl', 'get', '-u', owner, '--', folder):
+        if row.get('id') == identifier:
+          have.update((row.get('rights') or '').split())
+      missing = [right for right in rights if right not in have]
+      if not missing:
+        result['skipped'] += 1
+        return
+      doveadm('acl', 'add', '-u', owner, '--', folder, identifier, *missing)
+      result['added'] += 1
+    except RuntimeError as e:
+      result['errors'].append('%s %s %s: %s' % (owner, folder, self.shown(identifier), e))
+
+# Rights the mailbox granted on its folders and rights other mailboxes granted to it; added, never removed
+def restore_acl(acl, user, own, granted):
+  result = component()
+  for folder, entries in own:
+    for identifier, rights in entries:
+      acl.add(user, folder, identifier, rights, result)
+  if acl.user_exists(user):
+    for owner, folders in granted.items():
+      if owner == user or not acl.in_scope(owner) or not acl.user_exists(owner):
+        continue
+      for folder, entries in folders:
+        for identifier, rights in entries:
+          if identifier.lower() == 'user=' + user:
+            acl.add(owner, folder, identifier, rights, result)
+  return result
+
+# Own Sieve scripts (incl. the SOGo vacation script): missing names are added; with activate the active script of the
+# backup is activated if none is active now
+def restore_sieve(user, home, active_link, activate):
+  result = component()
+  folder = home + '/sieve'
+  scripts = []
+  if is_dir(folder):
+    scripts = sorted(f[:-6] for f in os.listdir(folder) if f.endswith('.sieve') and stat.S_ISREG(os.lstat(os.path.join(folder, f)).st_mode))
+  current, active = set(), None
+  for line in doveadm('sieve', 'list', '-u', user).splitlines():
+    name, _, flag = line.rpartition(' ')
+    if name:
+      current.add(name)
+      if flag == 'ACTIVE':
+        active = name
+  for name in scripts:
+    if name in current:
+      result['skipped'] += 1
+      continue
+    try:
+      doveadm('sieve', 'put', '-u', user, '--', name, data=read_file(os.path.join(folder, name + '.sieve')))
+      current.add(name)
+      result['added'] += 1
+    except (OSError, RuntimeError) as e:
+      result['errors'].append('%s: %s' % (name, e))
+  if not activate:
+    return result
+  target = os.readlink(active_link) if os.path.islink(active_link) else ''
+  backup_active = os.path.basename(target)[:-6] if target.endswith('.sieve') else None
+  if backup_active and not active and backup_active in current:
+    try:
+      doveadm('sieve', 'activate', '-u', user, '--', backup_active)
+      result['activated'] = backup_active
+    except RuntimeError as e:
+      result['errors'].append('%s: %s' % (backup_active, e))
+  return result
+
+# filters: backup.php reported its part already, the counts are added up
+def merge(results, key, part):
+  old = results.get(key)
+  if not isinstance(old, dict):
+    results[key] = part
+    return
+  for count in ('added', 'skipped'):
+    old[count] = (old.get(count) or 0) + part.get(count, 0)
+  old['errors'] = (old.get('errors') or []) + part.get('errors', [])
+  for name, value in part.items():
+    if name not in ('added', 'skipped', 'errors'):
+      old[name] = value
+
+def count_errors(result):
+  count = sum(len(result.get(key, {}).get('errors') or []) for key in ('domain', 'aliases') if isinstance(result.get(key), dict))
+  for entry in (result.get('mailboxes') or {}).values():
+    count += int(bool(entry.get('error')))
+    count += sum(len(value.get('errors') or []) for value in entry.values() if isinstance(value, dict))
+  return count
+
+def restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, result):
+  local, _, domain = user.rpartition('@')
+  home = '%s/var/vmail/%s/%s' % (staging, domain, local)
+  location = mail_location(user)
+  if not location:
+    for key in components & {'mails', 'imap_acl', 'filters'}:
+      merge(result, key, component(['the mailbox does not exist now or is inactive']))
+    return
+  maildir = staged_maildir(staging, home, location)
+  # the ACL files are read before the folders of the maildir are moved for the import
+  own = maildir_acls(maildir) if 'imap_acl' in components else []
+  if 'mails' in components:
+    try:
+      result['mails'] = restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota)
+    except Exception as e:
+      result['mails'] = component([str(e)[:1000]])
+    try:
+      doveadm('quota', 'recalc', '-u', user)
+    except RuntimeError as e:
+      result['mails']['errors'].append('quota recalc: %s' % e)
+    # new folders
+    acl.folders.pop(user, None)
+  if 'imap_acl' in components:
+    try:
+      result['imap_acl'] = restore_acl(acl, user, own, granted)
+    except Exception as e:
+      result['imap_acl'] = component([str(e)[:1000]])
+  if 'filters' in components:
+    try:
+      part = restore_sieve(user, home, '%s/var/vmail/sieve/%s.sieve' % (staging, user), 'sieve_activate' in components)
+    except Exception as e:
+      part = component([str(e)[:1000]])
+    merge(result, 'filters', part)
+
+def fail(job_id, error):
+  query("""UPDATE backup_job SET state = IF(state = 'cancelled', state, 'failed'), error = %s, finished = NOW()
+    WHERE id = %s""", (error, job_id))
+  log(job_id, 'restore job failed: ' + error)
+  notify(job_id, error)
+
+def notify(job_id, error):
+  hostname = os.environ.get('MAILCOW_HOSTNAME', '')
+  row = query("SELECT value FROM backup_setting WHERE name = 'notify_email'").fetchone()
+  rcpts = [r.strip() for r in (row['value'] if row else '').split(',') if re.fullmatch(r'[^@\s<>,]+@[^@\s<>,]+', r.strip())]
+  if not rcpts:
+    return
+  msg = EmailMessage()
+  msg['From'] = 'backup@' + hostname
+  msg['To'] = ', '.join(rcpts)
+  msg['Subject'] = 'mailcow backup on %s: restore job %d failed' % (hostname, job_id)
+  msg['Date'] = formatdate(localtime=True)
+  msg['Message-ID'] = make_msgid(domain=hostname or None)
+  msg.set_content('The restore job %d failed:\n\n%s\n\nThe job log is in the backup section of the mailcow UI.\n'
+    % (job_id, error))
+  try:
+    with smtplib.SMTP('postfix', 588, hostname or None, timeout=60) as smtp:
+      smtp.send_message(msg)
+  except (OSError, smtplib.SMTPException) as e:
+    print('restore job %d: the notification could not be sent: %s' % (job_id, e), file=sys.stderr)
+
+def staging_dir(job_id, path):
+  # the batch folder of this job, never a symlink
+  if not isinstance(path, str) or not re.fullmatch(r'/restore/%d/[A-Za-z0-9_-]+' % job_id, path) or \
+      os.path.realpath(path) != path or not is_dir(path):
+    return None
+  return path
+
+def run_batch(job):
+  job_id = job['id']
+  params = json.loads(job['params'] or '{}')
+  result = json.loads(job['result'] or '{}')
+  batch = result.get('batch') or {}
+  staging = staging_dir(job_id, batch.get('dir'))
+  acl_dir = staging_dir(job_id, batch.get('acl_dir')) if batch.get('acl_dir') else None
+  mailboxes = [m for m in batch.get('mailboxes') or [] if isinstance(m, str)]
+  components = {c for c in params.get('components') or [] if c in components_here}
+  mode = 'original' if params.get('mail_target') == 'original' else 'folder'
+  allowed = {d.lower() for d in params['allowed_domains']} if isinstance(params.get('allowed_domains'), list) else None
+  check_quota = params.get('requested_role', 'admin') != 'admin'
+  snapshot_time = (result.get('snapshot') or {}).get('time') or time.strftime('%Y-%m-%d %H:%M')
+  results = result.setdefault('mailboxes', {})
+  prefixes = shared_prefixes()
+  acl = AclRestore(prefixes, allowed)
+  granted = staged_acls(acl_dir, allowed) if 'imap_acl' in components else {}
+  log(job_id, 'restoring %s for %d mailboxes%s' % (', '.join(sorted(components)), len(mailboxes),
+    ' (mails: %s)' % mode if 'mails' in components else ''))
+  for user in mailboxes:
+    entry = results.setdefault(user, {})
+    if entry.get('error'):
+      continue
+    started = time.monotonic()
+    if not staging:
+      for key in components & {'mails', 'imap_acl', 'filters'}:
+        merge(entry, key, component(['the restored files are missing']))
+    else:
+      restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, entry)
+    summary = ', '.join('%s +%d' % (key, entry[key].get('added', 0)) for key in sorted(components) if isinstance(entry.get(key), dict))
+    errors = sum(len(entry[key].get('errors') or []) for key in components if isinstance(entry.get(key), dict))
+    log(job_id, '%s: %s%s (%d s)' % (user, summary, ', %d errors' % errors if errors else '', time.monotonic() - started))
+    result['errors'] = count_errors(result)
+    save(job_id, result)
+  result['errors'] = count_errors(result)
+  save(job_id, result)
+  # the restic runner removes the staging and goes on
+  query("UPDATE backup_job SET step = 'mails_done' WHERE id = %s AND state = 'running' AND step = 'mails_running'", (job_id,))
+
+def main():
+  global conn
+  os.umask(0o077)
+  # released by the kernel when the process ends, a crashed run leaves no stale lock
+  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+  try:
+    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  except OSError:
+    sys.exit("restore: can't lock " + lock_file)
+  conn = connect()
+  heartbeat = Heartbeat()
+  heartbeat.start()
+  try:
+    # under the lock: a batch in mails_running was left by a crashed or stopped run, its temporary folders go
+    for row in query("""SELECT id, result FROM backup_job WHERE type = 'restore' AND state = 'running'
+        AND step = 'mails_running'""").fetchall():
+      try:
+        for user in (json.loads(row['result'] or '{}').get('batch') or {}).get('mailboxes') or []:
+          remove_tree(user, 'Restoring %d' % row['id'])
+          doveadm('quota', 'recalc', '-u', user, check=False)
+      except (ValueError, AttributeError, RuntimeError):
+        pass
+      fail(row['id'], 'interrupted: the restore of the mails was stopped')
+    last, waited = None, 0
+    while True:
+      job = query("""SELECT id, params, result FROM backup_job WHERE type = 'restore' AND state = 'running'
+        AND step = 'mails' ORDER BY id LIMIT 1""").fetchone()
+      if not job:
+        # the next batch of a restore follows within seconds: wait for it instead of the next ofelia run
+        row = query("SELECT state FROM backup_job WHERE id = %s", (last,)).fetchone() if last else None
+        if row and row['state'] == 'running' and waited < batch_wait:
+          time.sleep(2)
+          waited += 2
+          continue
+        break
+      last, waited = job['id'], 0
+      if query("UPDATE backup_job SET step = 'mails_running' WHERE id = %s AND state = 'running' AND step = 'mails'",
+          (job['id'],)).rowcount != 1:
+        continue
+      try:
+        run_batch(job)
+      except Exception as e:
+        fail(job['id'], str(e)[:1000])
+  finally:
+    heartbeat.done.set()
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/dovecot/retention_runner.py

@@ -0,0 +1,249 @@
+#!/usr/bin/python3
+# Nightly run of the retention rules (ofelia, dovecot container): gets the plan page by page from the mailcow UI
+# (nginx:8081/retention.php), gives every folder of a mailbox to the first entry of its plan that matches it, counts
+# the mail saved in the folders before the period of the rule with doveadm, deletes it unless the rule only
+# simulates, and reports the results back.
+
+import fcntl
+import hashlib
+import hmac
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+plan_url = 'http://nginx:8081/retention.php'
+lock_file = '/tmp/retention_runner.lock'
+# folder keys of the rules and the special-use flag of their folders in the Dovecot configuration
+special_use = {'trash': '\\Trash', 'junk': '\\Junk', 'sent': '\\Sent', 'drafts': '\\Drafts', 'archive': '\\Archive'}
+own_folder = 'folder:'
+# mailboxes per page of the plan and results per report, a large instance has many mailboxes
+plan_page = 200
+report_batch = 200
+# a request is tried this often, waiting longer each time
+tries = 3
+
+token = hmac.new(os.environ.get('DBPASS', '').encode(), b'retention', hashlib.sha256).hexdigest()
+
+def request(url, body=None):
+  data = None if body is None else json.dumps(body).encode()
+  req = urllib.request.Request(url, data=data, headers={'Content-Type': 'application/json', 'X-Retention-Token': token},
+    method='GET' if body is None else 'POST')
+  with urllib.request.urlopen(req, timeout=300) as response:
+    return json.load(response)
+
+def retried(url, body=None):
+  for attempt in range(1, tries + 1):
+    try:
+      return request(url, body)
+    except Exception as e:
+      if attempt == tries:
+        raise
+      print('retention: request failed, trying again: %s' % e, file=sys.stderr)
+      time.sleep(30 * attempt)
+
+def plan():
+  # one page after the other, so memory stays small; rule changes apply to the pages fetched after them
+  after = ''
+  while after is not None:
+    page = retried(plan_url + '?' + urllib.parse.urlencode({'after': after, 'limit': plan_page}))
+    yield from page['mailboxes']
+    after = page['next']
+
+def post(body):
+  # False if the report could not be sent
+  try:
+    retried(plan_url, body)
+    return True
+  except Exception as e:
+    print('retention: report failed: %s' % e, file=sys.stderr)
+    return False
+
+class Reports:
+  # results go back in parts; once a part cannot be sent, the rest waits for the end of the run
+  def __init__(self):
+    self.pending, self.failed = [], False
+
+  def add(self, result):
+    self.pending.append(result)
+    if len(self.pending) >= report_batch and not self.failed:
+      self.failed = not self.send()
+
+  def send(self, summary=None, run=None):
+    # full parts, at the end all results with the summary and the run in the last part
+    final = summary is not None
+    while len(self.pending) >= report_batch or final:
+      body = {'results': self.pending[:report_batch]}
+      if final and len(self.pending) <= report_batch:
+        body.update(summary=summary, run=run)
+      if not post(body):
+        return False
+      del self.pending[:report_batch]
+      if 'summary' in body:
+        break
+    return True
+
+def doveadm(*args, partial=False):
+  result = subprocess.run(['doveadm', *args], capture_output=True, text=True, timeout=3600)
+  # mailbox status also fails for single folders that cannot be opened; the others are still listed
+  if result.returncode != 0 and not (partial and result.stdout):
+    raise RuntimeError((result.stderr or result.stdout).strip()[:500] or 'doveadm exit %d' % result.returncode)
+  return result.stdout
+
+def configuration():
+  # names of the special folders per key, and the prefixes of the shared and public namespaces (not the user's)
+  names = {key: set() for key in special_use}
+  prefixes = ['Shared/']
+  namespace, mailbox = None, None
+  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
+  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
+    stripped = line.strip()
+    if line.startswith('namespace ') and line.endswith('{'):
+      namespace = {}
+    elif line == '}':
+      if namespace and namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
+        prefixes.append(namespace['prefix'].split('%')[0])
+      namespace = None
+    elif namespace is None:
+      continue
+    elif re.match(r'mailbox .+ \{$', stripped):
+      mailbox = stripped[len('mailbox '):-2].strip().strip('"')
+    elif stripped == '}':
+      mailbox = None
+    elif '=' in stripped:
+      key, value = (part.strip() for part in stripped.split('=', 1))
+      if mailbox is not None and key == 'special_use':
+        for folder, flag in special_use.items():
+          if flag in value.split():
+            names[folder].add(mailbox)
+      elif mailbox is None and key in ('type', 'prefix'):
+        namespace[key] = value
+  if not names['trash']:
+    raise RuntimeError('no special folders in the output of doveconf -n')
+  return names, [prefix for prefix in prefixes if prefix]
+
+def own_folders(user, prefixes):
+  # folder name => guid of the user's own folders; listed first, so folders others share are not opened
+  names = set(name for name in doveadm('-f', 'tab', 'mailbox', 'list', '-u', user).splitlines()[1:]
+    if name and not any((name + '/').startswith(prefix) for prefix in prefixes))
+  folders = {}
+  if not names:
+    return folders
+  for line in doveadm('-f', 'tab', 'mailbox', 'status', '-u', user, 'guid', *sorted(names), partial=True).splitlines()[1:]:
+    name, _, guid = line.rpartition('\t')
+    if name in names and re.fullmatch(r'[0-9a-f]{32}', guid):
+      folders[name] = guid
+  return folders
+
+def canonical(name):
+  # Dovecot writes INBOX in capitals, also as the parent of folders
+  return 'INBOX' + name[5:] if name[:5].upper() == 'INBOX' and name[5:6] in ('', '/') else name
+
+def matches(action, name, names):
+  # match: the folder itself, an own folder name with *, the subfolders of a special folder or the inbox, all folders
+  key, match = action['folder'], action['match']
+  if match == 'all':
+    return True
+  if key.startswith(own_folder):
+    # * is the only wildcard of own folder names
+    pattern = '.*'.join(re.escape(part) for part in canonical(key[len(own_folder):]).split('*'))
+    return re.fullmatch(pattern, name) is not None
+  folders = ['INBOX'] if key == 'inbox' else names.get(key, ())
+  if match == 'subfolders':
+    return any(name.startswith(folder + '/') for folder in folders)
+  return name in folders
+
+def decide(actions, folders, names):
+  # rule id => (its action, the folders it decides about): every folder goes to the first action that matches it
+  decided = {}
+  for name in sorted(folders):
+    for action in actions:
+      if matches(action, canonical(name), names):
+        decided.setdefault(action['rule_id'], (action, []))[1].append(name)
+        break
+  return decided
+
+def found(user, query):
+  return len(doveadm('search', '-u', user, *query).splitlines())
+
+def clean(user, action, selected, folders, result):
+  # name and guid together: doveadm then only opens these folders
+  query = []
+  for name in selected:
+    query += (['OR'] if query else []) + ['(', 'mailbox', name, 'mailbox-guid', folders[name], ')']
+  query = ['('] + query + [')', 'savedbefore', '%dd' % int(action['days'])]
+  if action['keep_flagged']:
+    query += ['NOT', 'FLAGGED']
+  result['matched'] = found(user, query)
+  if not result['matched'] or action['mode'] != 'delete':
+    return
+  try:
+    doveadm('expunge', '-u', user, *query)
+    result['deleted'] = result['matched']
+  except Exception as e:
+    # part of the mail may be gone: what is left is counted
+    result['error'] = str(e)[:1000]
+    try:
+      result['deleted'] = max(0, result['matched'] - found(user, query))
+    except Exception:
+      pass
+
+def main():
+  # released by the kernel when the process ends, a crashed run leaves no stale lock
+  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
+  try:
+    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+  except OSError:
+    sys.exit("retention: can't lock " + lock_file)
+  reports = Reports()
+  run = {'started': int(time.time()), 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0, 'error': ''}
+  summary = {}
+  try:
+    names, prefixes = configuration()
+    for mailbox in plan():
+      user = mailbox['username']
+      run['mailboxes'] += 1
+      try:
+        folders, error = own_folders(user, prefixes), ''
+        decided = decide(mailbox['actions'], folders, names)
+      except Exception as e:
+        # the folders are unknown: an error for every rule of the mailbox
+        folders, error, decided = {}, str(e)[:1000], {}
+        for action in mailbox['actions']:
+          decided.setdefault(action['rule_id'], (action, []))
+      for rule_id, (action, selected) in decided.items():
+        rule = summary.setdefault(rule_id, {'rule_id': rule_id, 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0,
+          'simulated': int(action['mode'] != 'delete')})
+        result = {'username': user, 'rule_id': rule_id, 'folder': '\n'.join(selected), 'days': action['days'], 'matched': 0,
+          'deleted': 0, 'simulated': rule['simulated'], 'error': error}
+        if not error:
+          try:
+            clean(user, action, selected, folders, result)
+          except Exception as e:
+            result['error'] = str(e)[:1000]
+        for counter in (rule, run):
+          counter['matched'] += result['matched']
+          counter['deleted'] += result['deleted']
+          counter['errors'] += int(result['error'] != '')
+        rule['mailboxes'] += 1
+        # only mailboxes with found mail or an error are logged, every rule gets a summary
+        if result['matched'] or result['error']:
+          reports.add(result)
+  except Exception as e:
+    run['error'] = str(e)[:1000]
+  reported = reports.send(list(summary.values()), run)
+  print('retention: %d mailboxes, %d mails deleted, %d found by simulating rules, %d errors' % (run['mailboxes'],
+    run['deleted'], sum(rule['matched'] for rule in summary.values() if rule['simulated']), run['errors']))
+  if run['error']:
+    print('retention: %s' % run['error'], file=sys.stderr)
+  if not reported:
+    print('retention: the results could not be reported', file=sys.stderr)
+  if run['error'] or not reported:
+    sys.exit(1)
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -7,9 +7,6 @@ class BanManager:
   and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
   (chains, SNAT), because the backends must not run in parallel."""
 
-  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
-  EXCUSE_TIME = 60
-
   def __init__(self, r, tables, options, logger, lock, clock=time.time):
     self.r = r
     self.tables = tables
@@ -21,16 +18,6 @@ class BanManager:
     self.banned = set()
     self.denied = {}
     self.allowlist = []
-    self.excused = {}
-
-  def excuse(self, ip):
-    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
-    with self.lock:
-      now = self.clock()
-      if self.excused.get(ip, 0) <= now:
-        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
-          ip, self.EXCUSE_TIME))
-      self.excused[ip] = now + self.EXCUSE_TIME
 
   def attempt(self, ip):
     """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
@@ -43,9 +30,6 @@ class BanManager:
       if allowed is not None:
         self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
         return
-      if self.excused.get(ip, 0) > now:
-        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
-        return
       if net in self.banned:
         # log lines of connections from before the ban
         return
@@ -208,8 +192,6 @@ class BanManager:
       for net in [net for net, counter in self.counters.items()
                   if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
         del self.counters[net]
-      for ip in [ip for ip, until in self.excused.items() if until <= now]:
-        del self.excused[ip]
 
   def apply(self, nets):
     for net in self.tables.ban(nets):

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -109,12 +109,6 @@ class Netfilter:
   def handle_line(self, line):
     # a broken line must not end the watcher
     try:
-      excused = Regexes.temporary_failure(line)
-      if excused is not None:
-        ip = Regexes.parse_ip(excused)
-        if self.external(ip):
-          self.bans.excuse(ip)
-        return
       result = self.regexes.match(line)
       if result is None:
         return

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -6,20 +6,43 @@ import re
 class Regexes:
   """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
 
+  # Anchored to the line start: text clients control in other lines (login names, SMTP commands) must not match.
+  # Failed IMAP, POP3, Sieve and SMTP logins are reported by mailcowauth.php (rule 3), not by Dovecot or Postfix.
+  # Only the master passdb is checked by Dovecot itself (rule 8); the last address is the one Dovecot added.
   DEFAULTS = {
-    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
+    1: r'^mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)$',
-    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
+    2: r'^Rspamd UI: Invalid password by ([0-9a-f\.:]+)$',
-    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
+    3: r'^MAILCOWAUTH: Login failed for user .+ with service \w* from IP ([0-9a-f\.:]+)$',
-    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
+    4: r'^warning: non-SMTP command from [^\[\]\s]*\[([0-9a-f\.:]+)\]:.+',
-    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    5: r'^NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
-    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
+    6: r'^(?:\[\d+\]: )?SOGo\w* Login from \'(?:[0-9a-f\.:]+, )*([0-9a-f\.:]+)\' for user .+ might not have worked',
-    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
+    7: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
-    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
+    8: r'^auth: passwd-file\(.*,([0-9a-f\.:]+),master(?:,<[^<>]*>)?\): (?:Password mismatch|unknown user)',
-    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
   }
-  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
+  # Former default rules (with any rule id) => rule id of their replacement, None: the rule is removed.
-  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
+  # Rules the admin changed are kept.
-  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+  FORMER_DEFAULTS = {
+    r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)': 1,
+    r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)': 2,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed': 3,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+': 3,
+    r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+': 4,
+    r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+': 5,
+    r"SOGo.+ Login from '([0-9a-f\.:]+)' for user .+ might not have worked": 6,
+    r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked': 6,
+    r'-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'auth: static\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'auth: \w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)': None,
+  }
+  # Lines of the web UI are not sanitized like syslog lines: a newline in a login name must not hide the address
+  CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f]')
 
   def __init__(self, logger):
     self.logger = logger
@@ -42,6 +65,10 @@ class Regexes:
     if not isinstance(stored, dict):
       self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
       return
+    migrated = self.migrate(stored)
+    if migrated != stored:
+      self.store_migrated(r)
+      stored = migrated
 
     rules = []
     for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
@@ -54,6 +81,7 @@ class Regexes:
 
   def match(self, line):
     """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    line = self.CONTROL_CHARS.sub('_', line)
     for rule_id, regex in self.rules:
       result = regex.search(line)
       if result and result.lastindex and result.group(1):
@@ -61,10 +89,35 @@ class Regexes:
     return None
 
   @classmethod
-  def temporary_failure(cls, line):
+  def migrate(cls, stored):
-    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
+    """The stored rules with former default rules replaced by the current ones (or removed)"""
-    result = cls.TEMPORARY_FAILURE.search(line)
+    migrated = {}
-    return result.group(1) if result else None
+    for rule_id, regex in stored.items():
+      if isinstance(regex, str) and regex in cls.FORMER_DEFAULTS:
+        replacement = cls.FORMER_DEFAULTS[regex]
+        if replacement is None:
+          continue
+        regex = cls.DEFAULTS[replacement]
+      migrated[rule_id] = regex
+    return migrated
+
+  def store_migrated(self, r):
+    # WATCH keeps a save of the UI between reading and writing
+    def replace(pipe):
+      current = json.loads(pipe.get('F2B_REGEX') or '{}')
+      if not isinstance(current, dict):
+        return
+      migrated = self.migrate(current)
+      if migrated == current:
+        return
+      pipe.multi()
+      pipe.set('F2B_REGEX', json.dumps(migrated, ensure_ascii=False))
+
+    try:
+      r.transaction(replace, 'F2B_REGEX')
+      self.logger.logInfo('Replaced former default rules in F2B_REGEX')
+    except Exception as ex:
+      self.logger.logWarn('Could not replace former default rules in F2B_REGEX: %s' % ex)
 
   @staticmethod
   def parse_ip(text):

data/Dockerfiles/nginx/bootstrap.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 from jinja2 import Environment, FileSystemLoader
 
@@ -44,6 +45,11 @@ def prepare_template_vars():
   ipv4_network = os.getenv("IPV4_NETWORK", "172.22.1")
   additional_server_names = os.getenv("ADDITIONAL_SERVER_NAMES", "")
   trusted_proxies = os.getenv("TRUSTED_PROXIES", "")
+  # nginx size syntax, invalid values fall back to the default
+  max_body_size = os.getenv("NGINX_MAX_BODY_SIZE", "128m").strip()
+  if not re.fullmatch(r'[0-9]+[kKmMgG]?', max_body_size):
+    print(f"Invalid NGINX_MAX_BODY_SIZE '{max_body_size}', using 128m")
+    max_body_size = "128m"
 
   template_vars = {
     'IPV4_NETWORK': ipv4_network,
@@ -60,6 +66,7 @@ def prepare_template_vars():
     'PHPFPMHOST': os.getenv("PHPFPMHOST", "php-fpm-mailcow"),
     'ENABLE_IPV6': os.getenv("ENABLE_IPV6", "true").lower() != "false",
     'HTTP_REDIRECT': os.getenv("HTTP_REDIRECT", "n").lower() in ("y", "yes"),
+    'NGINX_MAX_BODY_SIZE': max_body_size,
   }
 
   ssl_dir = '/etc/ssl/mail/'

data/Dockerfiles/postfix/Dockerfile

@@ -26,6 +26,7 @@ RUN groupadd -g 102 postfix \
 	postfix \
 	postfix-mysql \
 	postfix-pcre \
+	python3 \
 	redis-tools \
 	sasl2-bin \
 	sudo \
@@ -46,6 +47,7 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
+COPY mailcow-policy.py /usr/local/bin/mailcow-policy.py
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
@@ -53,6 +55,7 @@ RUN chmod +x /opt/postfix.sh \
   /usr/local/bin/rspamd-pipe-ham \
   /usr/local/bin/rspamd-pipe-spam \
   /usr/local/bin/whitelist_forwardinghosts.sh \
+  /usr/local/bin/mailcow-policy.py \
   /usr/local/sbin/stop-supervisor.sh
 RUN rm -rf /tmp/* /var/tmp/*
 

data/Dockerfiles/postfix/mailcow-policy.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+# Policy service of the mail policies (mailcow-policy in master.cf, spawned by Postfix): reads the policy requests
+# of smtpd and asks the mailcow UI for the action (nginx:8081/mailpolicy.php). Without an answer the recipient is
+# deferred; smtpd only asks while policies or internal aliases exist (mysql_mail_policy_maps.cf).
+
+import http.client
+import re
+import socket
+import sys
+from urllib.parse import urlencode
+
+# one line: DUNNO or a rejection with code and text
+valid_action = re.compile(rb'DUNNO|[45]\d\d [45]\.\d{1,3}\.\d{1,3} [\x20-\x7e]{1,200}')
+unavailable = b'DEFER_IF_PERMIT Mail policy check temporarily unavailable'
+# address of nginx, resolved once per process (a lookup costs more than the check)
+nginx = None
+
+def check(request):
+  global nginx
+  body = urlencode(sorted(request.items())).encode()
+  try:
+    if nginx is None:
+      nginx = socket.gethostbyname('nginx')
+    # http.client sets TCP_NODELAY, so the body is not held back until the headers are acknowledged
+    # below the timeout of Postfix (main.cf), so an answer always arrives
+    conn = http.client.HTTPConnection(nginx, 8081, timeout=8)
+    try:
+      conn.request('POST', '/mailpolicy.php', body, {'Host': 'nginx', 'Content-Type': 'application/x-www-form-urlencoded'})
+      response = conn.getresponse()
+      action = response.read()
+    finally:
+      conn.close()
+  except (OSError, http.client.HTTPException):
+    # nginx may have a new address after a restart
+    nginx = None
+    return unavailable
+  return action if response.status == 200 and valid_action.fullmatch(action) else unavailable
+
+def main():
+  request = {}
+  # attributes as bytes: addresses may contain any 8-bit characters
+  for line in sys.stdin.buffer:
+    line = line.rstrip(b'\r\n')
+    if line:
+      name, separator, value = line.partition(b'=')
+      if separator:
+        request[name] = value
+      continue
+    sys.stdout.buffer.write(b'action=' + check(request) + b'\n\n')
+    sys.stdout.buffer.flush()
+    request = {}
+
+if __name__ == '__main__':
+  main()

data/Dockerfiles/postfix/postfix.sh

@@ -15,6 +15,21 @@ until dig +short mailcow.email > /dev/null; do
   sleep 1
 done
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+done
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
+else
+  echo "DB schema is ${DBV_NOW}"
+fi
+
 cat <<EOF > /etc/aliases
 # Autogenerated by mailcow
 null: /dev/null
@@ -224,6 +239,19 @@ query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports
   LIMIT 1;
 EOF
 
+# Mail policies: while policies or internal aliases exist, smtpd asks the policy service (mailcow_mail_policy in main.cf)
+cat <<EOF > /opt/postfix/conf/sql/mysql_mail_policy_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT 'mailcow_mail_policy' FROM DUAL WHERE '%s' != ''
+  AND (EXISTS (SELECT 1 FROM mail_policy_member JOIN mail_policy ON mail_policy.id = mail_policy_member.policy_id
+      WHERE mail_policy.active = '1')
+    OR EXISTS (SELECT 1 FROM alias WHERE internal = '1' AND active IN ('1', '2')));
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}

data/Dockerfiles/restic/Dockerfile

@@ -0,0 +1,28 @@
+FROM alpine:3.24
+
+LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
+
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, the runner connects through the socket
+ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
+
+RUN apk add --no-cache --update \
+  ca-certificates \
+  git \
+  mariadb-client \
+  openssh-client \
+  py3-mysqlclient \
+  py3-pynacl \
+  python3 \
+  restic \
+  tini \
+  tzdata \
+  # dump, export, manifest and the mailcow directory are below /mailcow (paths of the snapshot), only root may read them
+  && mkdir -m 700 /mailcow
+
+COPY restic_runner.py /usr/local/bin/restic_runner.py
+COPY sogo_restore.py /usr/local/bin/sogo_restore.py
+
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD ["/usr/bin/python3", "-u", "/usr/local/bin/restic_runner.py"]

data/Dockerfiles/restic/sogo_restore.py

@@ -0,0 +1,118 @@
+"""SOGo part of a mailbox restore (component sogo, overwrites), called by the restic runner for one mailbox.
+
+Runs `sogo-tool restore -f ALL <dir> <user>` and `sogo-tool restore -p <dir> <user>` in the sogo container through
+dockerapi on the user's file of the nightly sogo-tool backup: missing calendars, address books and entries come back,
+the names and shares of existing folders and all SOGo preferences are replaced by the backup's, entries changed since
+stay. Then the user's entries in SOGo's cache (memcached) are removed, so the change shows at once.
+"""
+
+import json
+import os
+import re
+import socket
+import ssl
+import urllib.request
+
+dockerapi_url = 'https://dockerapi:443'
+memcached_addr = ('memcached-mailcow', 11211)
+# dockerapi stops each of the two sogo-tool runs after 850 s
+restore_timeout = 1800
+# accepted by the dockerapi handler as well
+user_re = re.compile(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+')
+
+
+def _short(text, limit=300):
+  # sogo-tool may print database URLs: no credentials, one line, the end has the error
+  text = re.sub(r'([A-Za-z][A-Za-z0-9+.-]*://)[^\s/@]*@', r'\1***@', str(text))
+  text = ' '.join(text.split())
+  return text if len(text) <= limit else '...' + text[-limit:]
+
+
+def _api(path, body=None, timeout=60):
+  # dockerapi uses the mail certificate, its name does not match
+  ctx = ssl.create_default_context()
+  ctx.check_hostname = False
+  ctx.verify_mode = ssl.CERT_NONE
+  data = None if body is None else json.dumps(body).encode()
+  req = urllib.request.Request(dockerapi_url + path, data=data, headers={'Content-Type': 'application/json'},
+    method='GET' if body is None else 'POST')
+  with urllib.request.urlopen(req, timeout=timeout, context=ctx) as response:
+    return json.load(response)
+
+
+def _sogo_container():
+  project = os.environ.get('COMPOSE_PROJECT_NAME', '').lower()
+  by_name = None
+  for cid, info in _api('/containers/json').items():
+    if not cid.isalnum() or not isinstance(info, dict):
+      continue
+    labels = (info.get('Config') or {}).get('Labels') or {}
+    if labels.get('com.docker.compose.service') == 'sogo-mailcow':
+      if not project or labels.get('com.docker.compose.project', '').lower() == project:
+        return cid
+    elif by_name is None and 'sogo-mailcow' in str(info.get('Name', '')):
+      by_name = cid
+  if by_name:
+    return by_name
+  raise RuntimeError('sogo container not found')
+
+
+def _memcached_delete(keys):
+  keys = [k for k in keys if 0 < len(k.encode()) <= 250 and not re.search(r'[\s\x00-\x1f\x7f]', k)]
+  if not keys:
+    return
+  with socket.create_connection(memcached_addr, timeout=10) as s:
+    for key in keys:
+      s.sendall(b'delete ' + key.encode() + b'\r\n')
+      reply = b''
+      while not reply.endswith(b'\r\n'):
+        chunk = s.recv(64)
+        if not chunk:
+          raise RuntimeError('memcached closed the connection')
+        reply += chunk
+      if reply.strip() not in (b'DELETED', b'NOT_FOUND'):
+        raise RuntimeError('memcached: %s' % _short(reply.decode('ascii', 'replace')))
+
+
+def _cache_keys(db, username):
+  # preferences and the ACLs of the user's folders (<uid>/Calendar/personal+acl); never read the c_*location
+  # columns of sogo_folder_info (database credentials)
+  keys = ['%s+defaults' % username, '%s+settings' % username]
+  cur = db.cursor()
+  try:
+    cur.execute('SELECT c_path FROM sogo_folder_info WHERE c_path2 = %s', (username,))
+    keys += ['%s+acl' % row[0][len('/Users/'):] for row in cur.fetchall() if row[0].startswith('/Users/')]
+  finally:
+    cur.close()
+  return keys
+
+
+def restore(db, backup_dir, username, log):
+  """Restores the SOGo data of one user from <backup_dir>/<username> (staged readable for the sogo user only).
+  db: open MySQLdb connection; log: callable for short progress lines. Returns {"added": 0|1, "errors": [...]}."""
+  result = {'added': 0, 'errors': []}
+  if not user_re.fullmatch(username):
+    result['errors'].append('username not supported by the SOGo restore')
+    return result
+  path = os.path.join(backup_dir, username)
+  if not os.path.isfile(path) or os.path.islink(path):
+    # sogo-tool backup ALL skips users without SOGo folders
+    # a key the UI translates
+    result['note'] = 'no_sogo_data'
+    log('sogo: no SOGo data in the snapshot')
+    return result
+  try:
+    res = _api('/containers/%s/exec' % _sogo_container(), {'cmd': 'sogo', 'task': 'restore', 'dir': backup_dir,
+      'username': username}, timeout=restore_timeout)
+    if not isinstance(res, dict) or res.get('type') != 'success':
+      raise RuntimeError('sogo-tool restore failed: %s' % _short(res.get('msg', '') if isinstance(res, dict) else res))
+    result['added'] = 1
+    log('sogo: calendars, address books and preferences reset to the backup')
+  except Exception as e:
+    result['errors'].append(_short(e))
+  try:
+    # SOGo caches ACLs and preferences for 15 minutes
+    _memcached_delete(_cache_keys(db, username))
+  except Exception as e:
+    result['errors'].append('SOGo cache not cleared, changes may show up after 15 minutes: %s' % _short(e))
+  return result

data/Dockerfiles/sogo/Dockerfile

@@ -1,6 +1,6 @@
 # SOGo built from source to enable security patch application
 # Repository: https://github.com/Alinto/sogo
-# Version: SOGo-5.12.12
+# Version: SOGo-5.12.13
 #
 # Applied security patches:
 # -
@@ -12,8 +12,8 @@ FROM debian:bookworm
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SOGO_VERSION=SOGo-5.12.12
+ARG SOGO_VERSION=SOGo-5.12.13
-ARG SOPE_VERSION=SOPE-5.12.12
+ARG SOPE_VERSION=SOPE-5.12.13
 # Security patches to apply (space-separated commit hashes)
 ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$

data/Dockerfiles/watchdog/watchdog.sh

@@ -43,6 +43,19 @@ while ! mariadb-admin status --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u$
   sleep 2
 done
 
+# Wait for the schema update (DB_VERSION) at startup, do not restart them meanwhile.
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
+done
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31m$(date) - DB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting checks anyway, check the php-fpm log (init_db)\e[0m"
+fi
+
 # Do not attempt to write to slave
 if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"

data/conf/dovecot/auth/mailcowauth.php

@@ -68,13 +68,17 @@ require_once 'functions.acl.inc.php';
 
 
 $isSOGoRequest = $post['real_rip'] == getenv('IPV4_NETWORK') . '.248';
+// real_rip is the address that connected to Dovecot (for proxied connections the proxy), client_ip the client (for logs)
+$client_ip = filter_var($post['client_ip'] ?? '', FILTER_VALIDATE_IP) ? $post['client_ip'] : $post['real_rip'];
 $result = false;
+// set when the password is right, but the account does not allow this login
+$denied = false;
 if ($isSOGoRequest) {
   // This is a SOGo Auth request. First check for SSO password.
   $sogo_sso_pass = file_get_contents("/etc/sogo-sso/sogo-sso.pass");
   if ($sogo_sso_pass === $post['password']){
     error_log('MAILCOWAUTH: SOGo SSO auth for user ' . $post['username']);
-    set_sasl_log($post['username'], $post['real_rip'], "SOGO");
+    set_sasl_log($post['username'], $client_ip, "SOGO");
     $result = true;
   }
 }
@@ -91,10 +95,10 @@ if ($result === false){
     'service' => $post['service'],
     'is_internal' => true,
     'remote_addr' => $post['real_rip']
-  ));
+  ), $denied);
   if ($result) {
-    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $post['real_rip']);
+    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
-    set_sasl_log($post['username'], $post['real_rip'], $service);
+    set_sasl_log($post['username'], $client_ip, $service);
   }
 }
 if ($result === false){
@@ -102,26 +106,37 @@ if ($result === false){
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
   try {
-    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']), $denied);
   } catch (\Mailcow\Auth\IdpUnavailable $e) {
     // temporary failure for Dovecot, no failed login
-    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip . ": " . $e->getMessage());
     http_response_code(503); // Service Unavailable
     echo json_encode($return);
     session_destroy();
     exit;
   }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip);
-    set_sasl_log($post['username'], $post['real_rip'], $post['service']);
+    set_sasl_log($post['username'], $client_ip, $post['service']);
   }
 }
 
 if ($result) {
   http_response_code(200); // OK
   $return['success'] = true;
+} elseif ($denied) {
+  // Logged, but no failed login for netfilter: e.g. a client of a shared mailbox or without IMAP access
+  error_log("MAILCOWAUTH: Login denied for user " . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+  http_response_code(401); // Unauthorized
 } else {
-  error_log("MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
+  // Same line for the log and netfilter, which counts it for IMAP, POP3, Sieve and SMTP (Postfix asks Dovecot);
+  // SOGo logins have their own rule
+  $failed = "MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " .
+    preg_replace('/[^a-z0-9]/', '', strtolower((string)$service)) . " from IP " . $client_ip;
+  error_log($failed);
+  if (!$isSOGoRequest) {
+    $redis->publish("F2B_CHANNEL", $failed);
+  }
   http_response_code(401); // Unauthorized
 }
 

data/conf/dovecot/auth/passwd-verify.lua

@@ -1,8 +1,4 @@
 function auth_password_verify(request, password)
-  if request.domain == nil then
-    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
-  end
-
   local json = require "cjson"
   local ltn12 = require "ltn12"
   local https = require "ssl.https"
@@ -11,7 +7,10 @@ function auth_password_verify(request, password)
   local req = {
     username = request.user,
     password = password,
+    -- address that connected to Dovecot, for proxied connections the proxy
     real_rip = request.real_rip,
+    -- address of the client, also for proxied connections
+    client_ip = request.rip,
     service = request.service
   }
   local req_json = json.encode(req)

data/conf/dovecot/dovecot.conf

@@ -51,13 +51,6 @@ recipient_delimiter = +
 auth_master_user_separator = *
 mail_shared_explicit_inbox = yes
 mail_prefetch_count = 30
-passdb {
-  driver = lua
-  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
-  result_success = return-ok
-  result_failure = continue
-  result_internalfail = continue
-}
 # try a master passwd
 passdb {
   driver = passwd-file
@@ -67,8 +60,9 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
+# each check asks mailcowauth.php, which reports a failed login to netfilter once
 # cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
-# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
+# of a passdb lookup (master logins) and reject the login as unknown user
 passdb {
   driver = lua
   args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
@@ -205,6 +199,8 @@ plugin {
   sieve_plugins = sieve_imapsieve sieve_extprograms
   sieve_vacation_send_from_recipient = yes
   sieve_redirect_envelope_from = recipient
+  # Notifications are sent with the address of the mailbox (not :from or postmaster), so mail policies apply
+  sieve_notify_mailto_envelope_from = recipient
   # From elsewhere to Spam folder
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY

data/conf/nginx/templates/nginx.conf.j2

@@ -47,6 +47,13 @@ http {
         default "";
     }
 
+    # Client address for PHP: on the internal sogo-auth listener nginx itself is the client,
+    # the real address comes from the X-Real-IP header set by /sogo-auth-verify
+    map $sogo_auth_internal $mailcow_client_ip {
+        "1"     $http_x_real_ip;
+        default $remote_addr;
+    }
+
     {% if HTTP_REDIRECT %}
     # HTTP to HTTPS redirect
     server {
@@ -143,6 +150,15 @@ http {
         access_log /var/log/nginx/access.log;
         root /dynmaps;
 
+        # backup endpoint of the restic runner: large restore requests, long merges of mailboxes and domains
+        location = /backup.php {
+            client_max_body_size 32m;
+            fastcgi_read_timeout 600s;
+            fastcgi_pass {{ PHPFPMHOST }}:9001;
+            include fastcgi_params;
+            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+        }
+
         location ~ \.php$ {
             try_files $uri =404;
             fastcgi_split_path_info ^(.+\.php)(/.+)$;

data/conf/nginx/templates/sites-default.conf.j2

@@ -22,7 +22,8 @@ add_header Referrer-Policy strict-origin;
 
 index index.php index.html;
 
-client_max_body_size 0;
+# PHP accepts no larger requests (post_max_size), SOGo webmail, CalDAV/CardDAV and ActiveSync use NGINX_MAX_BODY_SIZE
+client_max_body_size 64m;
 
 gzip on;
 gzip_disable "msie6";
@@ -116,7 +117,8 @@ location ~ \.php$ {
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
-    fastcgi_param HTTP_X_REAL_IP $remote_addr;
+    fastcgi_param REMOTE_ADDR $mailcow_client_ip;
+    fastcgi_param HTTP_X_REAL_IP $mailcow_client_ip;
     # trusted internal-auth marker; empty for external clients (see nginx.conf map)
     fastcgi_param SOGO_AUTH_INTERNAL $sogo_auth_internal;
     fastcgi_read_timeout 3600;
@@ -181,6 +183,7 @@ location /sogo-auth-verify {
     proxy_set_header  Content-Length "";
     proxy_pass        http://127.0.0.1:65510/sogo-auth;
     proxy_pass_request_body off;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
 }
 
 location ^~ /Microsoft-Server-ActiveSync {
@@ -204,7 +207,7 @@ location ^~ /Microsoft-Server-ActiveSync {
     proxy_busy_buffers_size 512k;
     proxy_set_header Host $http_host;
     client_body_buffer_size 512k;
-    client_max_body_size 0;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
 }
 
 location ^~ /SOGo {
@@ -255,7 +258,7 @@ location ^~ /SOGo {
     proxy_send_timeout 3600;
     proxy_read_timeout 3600;
     client_body_buffer_size 128k;
-    client_max_body_size 0;
+    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
     break;
 }
 

data/conf/postfix/main.cf

@@ -50,6 +50,7 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   $sender_dependent_default_transport_maps,
@@ -90,6 +91,7 @@ smtpd_hard_error_limit = ${stress?1}${stress:5}
 smtpd_helo_required = yes
 smtpd_proxy_timeout = 600s
 smtpd_recipient_restrictions = check_recipient_mx_access proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
+  check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   permit_sasl_authenticated,
   permit_mynetworks,
   check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
@@ -170,5 +172,11 @@ parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks
 # This Option is added to correctly set the X-Original-To Header when mails are send to lmtp (dovecot)
 lmtp_destination_recipient_limit=1
 
+# Mail policies: while policies or internal aliases exist, mysql_mail_policy_maps.cf returns this class and smtpd
+# asks the policy service for every recipient (mailcow-policy in master.cf); no answer defers the recipient
+smtpd_restriction_classes = mailcow_mail_policy
+mailcow_mail_policy = check_policy_service { unix:private/mailcow-policy, timeout=10s, try_limit=1, default_action=DEFER_IF_PERMIT }
+mailcow-policy_time_limit = 3600s
+
 # DO NOT EDIT ANYTHING BELOW #
 # Overrides #

data/conf/postfix/master.cf

@@ -117,6 +117,9 @@ smtp_sender_cleanup unix n - y - 0 cleanup
 127.0.0.1:10027 inet n n n - 0 spawn user=nobody argv=/usr/local/bin/whitelist_forwardinghosts.sh
 # end whitelist_fwd
 
+# mail policies: policy service of smtpd, asks the mailcow UI (mailcow_mail_policy in main.cf)
+mailcow-policy unix - n n - 0 spawn user=nobody argv=/usr/local/bin/mailcow-policy.py
+
 # start watchdog-specific
 # logs to local7 (hidden)
 589 inet n      -       n       -       -       smtpd

data/conf/rspamd/dynmaps/backup.php

@@ -0,0 +1,97 @@
+<?php
+// Backup with restic: endpoint of the runner in the restic container (Mailcow\Backup)
+// GET ?domains=1: {"domains": [...]}, all domains
+// GET ?export=<domain>[&after=<username>]: a page of the export of a domain (the first with domain_data and aliases,
+//   every page with a part of the mailboxes and "next_after", the last username to ask for the next page with, null
+//   after the last), incl. password hashes, DKIM keys and secrets; the runner merges the pages into
+//   /mailcow/export/<domain>.json of the snapshot.
+// POST {"action": "restore_domain", "job": <id>, "data": <domain_data>}: recreates a deleted domain
+// POST {"action": "restore_aliases", "job": <id>, "aliases": [...]}: a part (at most 500) of the aliases of a domain restore
+// POST {"action": "restore", "job": <id>, "username": <mailbox>, "data": <its section of the export>}: one mailbox
+//   (created first when it was deleted); all answer {"result": {<component>: {"added", "skipped", "errors"}}}
+// All need the header X-Backup-Token: HMAC-SHA256 of "backup" with the database password as key
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'backup', $database_pass), (string)($_SERVER['HTTP_X_BACKUP_TOKEN'] ?? ''))) {
+  http_response_code(403);
+  exit;
+}
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("BACKUP: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+try {
+  // DKIM keys, rate limits and the domain map are kept in Redis
+  $redis = new Redis();
+  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
+    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
+  }
+  else {
+    $redis->connect('redis-mailcow', 6379);
+  }
+  $redis->auth(getenv("REDISPASS"));
+  $config = array(
+    'timezone' => getenv('TZ') ?: 'UTC',
+    'alias_disable_external_domains' => ($ALIAS_DISABLE_EXTERNAL_DOMAINS ?? false) === true,
+  );
+  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
+    $body = json_decode((string)file_get_contents('php://input'), true);
+    if (!is_array($body)) {
+      http_response_code(400);
+      exit;
+    }
+    // The restore has the side effects of the UI: SOGo view and restart, Dovecot auth cache, group shares (legacy code)
+    require_once __DIR__ . '/../web/inc/functions.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
+    require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
+    // The legacy mailbox functions run as admin, without a session
+    $_SESSION = array(
+      'mailcow_cc_username' => 'admin',
+      'mailcow_cc_role' => 'admin',
+      'access_all_exception' => '0',
+      'acl' => array_fill_keys(array('tls_policy', 'quarantine_notification', 'quarantine_category', 'ratelimit', 'sogo_access',
+        'protocol_access', 'mailbox_relayhost', 'unlimited_quota'), '1'),
+    );
+    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
+    echo json_encode($backup->api->post($body), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+  }
+  else {
+    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
+    if (isset($_GET['export'])) {
+      echo json_encode($backup->api->export((string)$_GET['export'], (isset($_GET['after']) && is_string($_GET['after'])) ? $_GET['after'] : null),
+        JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
+    }
+    elseif (isset($_GET['domains'])) {
+      echo json_encode($backup->api->domains());
+    }
+    else {
+      http_response_code(400);
+    }
+  }
+}
+catch (\Mailcow\Core\UserError $e) {
+  http_response_code(($e->key === 'domain_not_found') ? 404 : 400);
+  echo json_encode(array('error' => $e->key, 'args' => $e->args));
+}
+catch (Throwable $e) {
+  error_log("BACKUP: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/mailpolicy.php

@@ -0,0 +1,39 @@
+<?php
+// Mail policy check for one recipient, asked by the Postfix policy service (mailcow-policy in master.cf)
+// Request: POST with the attributes of the Postfix policy request; answer: the action (DUNNO or the rejection)
+header('Content-Type: text/plain');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("MAILPOLICY: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+if (!isset($_POST['request']) || $_POST['request'] !== 'smtpd_access_policy') {
+  http_response_code(400);
+  exit;
+}
+$value = fn(string $key): string => is_string($_POST[$key] ?? null) ? $_POST[$key] : '';
+
+try {
+  $policy = \Mailcow\MailPolicy\MailPolicyModule::create($pdo, \Mailcow\Core\Context::system());
+  echo $policy->api->check($value('sender'), $value('recipient'), $value('sasl_username'), (int)$value('server_port'), $value('helo_name'));
+}
+catch (Throwable $e) {
+  // the policy service answers with a temporary error
+  error_log("MAILPOLICY: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/retention.php

@@ -0,0 +1,52 @@
+<?php
+// Retention rules for the nightly run in the dovecot container (retention_runner.py)
+// GET ?after=<username>&limit=<n>: a page of the plan, per mailbox the folders to clean up (JSON)
+// POST (JSON {"results": [...], "summary": [...], "run": {...}}): the results
+// Both need the header X-Retention-Token: HMAC-SHA256 of "retention" with the database password as key
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'retention', $database_pass), (string)($_SERVER['HTTP_X_RETENTION_TOKEN'] ?? ''))) {
+  http_response_code(403);
+  exit;
+}
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("RETENTION: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+
+try {
+  $retention = \Mailcow\Retention\RetentionModule::create($pdo, \Mailcow\Core\Context::system());
+  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
+    $body = json_decode((string)file_get_contents('php://input'), true);
+    if (!is_array($body) || !is_array($body['results'] ?? null)) {
+      http_response_code(400);
+      exit;
+    }
+    echo json_encode(['stored' => $retention->api->report($body['results'], is_array($body['summary'] ?? null) ? $body['summary'] : [],
+      is_array($body['run'] ?? null) ? $body['run'] : null)]);
+  }
+  else {
+    echo json_encode($retention->api->plan((string)($_GET['after'] ?? ''), (int)($_GET['limit'] ?? 0)));
+  }
+}
+catch (\Mailcow\Core\UserError $e) {
+  http_response_code(400);
+}
+catch (Throwable $e) {
+  error_log("RETENTION: " . $e->getMessage() . PHP_EOL);
+  http_response_code(500);
+}

data/conf/rspamd/dynmaps/settings.php

@@ -469,35 +469,5 @@ while ($row = array_shift($rows)) {
 }
 ?>
 
-<?php
-// Start internal aliases
-
-$stmt = $pdo->query("SELECT `id`, `address`, `domain` FROM `alias` WHERE `active` = '1' AND `internal` = '1'");
-$aliases = $stmt->fetchAll(PDO::FETCH_ASSOC);
-while ($alias = array_shift($aliases)) {
-  // build allowed_domains regex and add target domain and alias domains
-  $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `active` = '1' AND `target_domain` = :target_domain");
-  $stmt->execute(array(':target_domain' => $alias['domain']));
-  $allowed_domains = $stmt->fetchAll(PDO::FETCH_ASSOC);
-  $allowed_domains = array_map(function($item) {
-    return str_replace('.', '\.', $item['alias_domain']);
-  }, $allowed_domains);
-  $allowed_domains[] = str_replace('.', '\.', $alias['domain']);
-  $allowed_domains = implode('|', $allowed_domains);
-?>
-  internal_alias_<?=$alias['id'];?> {
-    priority = 10;
-    rcpt = "<?=$alias['address'];?>";
-    from = "/^((?!.*@(<?=$allowed_domains;?>)).)*$/";
-    apply "default" {
-      MAILCOW_INTERNAL_ALIAS = 9999.0;
-    }
-    symbols [
-      "MAILCOW_INTERNAL_ALIAS"
-    ]
-  }
-<?php
-}
-?>
 
 }

data/conf/rspamd/lua/rspamd.local.lua

@@ -150,41 +150,41 @@ rspamd_config:register_symbol({
     local function ipv6_to_segments(ip_str)
       -- Remove zone identifier if present (e.g., %eth0)
       ip_str = ip_str:gsub("%%.*$", "")
-      
+
       local segments = {}
-      
+
       -- Handle :: compression
       if ip_str:find('::') then
         local before, after = ip_str:match('^(.*)::(.*)$')
         before = before or ''
         after = after or ''
-        
+
         local before_parts = {}
         local after_parts = {}
-        
+
         if before ~= '' then
           for seg in before:gmatch('[^:]+') do
             table.insert(before_parts, tonumber(seg, 16) or 0)
           end
         end
-        
+
         if after ~= '' then
           for seg in after:gmatch('[^:]+') do
             table.insert(after_parts, tonumber(seg, 16) or 0)
           end
         end
-        
+
         -- Add before segments
         for _, seg in ipairs(before_parts) do
           table.insert(segments, seg)
         end
-        
+
         -- Add compressed zeros
         local zeros_needed = 8 - #before_parts - #after_parts
         for i = 1, zeros_needed do
           table.insert(segments, 0)
         end
-        
+
         -- Add after segments
         for _, seg in ipairs(after_parts) do
           table.insert(segments, seg)
@@ -195,12 +195,12 @@ rspamd_config:register_symbol({
           table.insert(segments, tonumber(seg, 16) or 0)
         end
       end
-      
+
       -- Ensure we have exactly 8 segments
       while #segments < 8 do
         table.insert(segments, 0)
       end
-      
+
       return segments
     end
 
@@ -208,46 +208,46 @@ rspamd_config:register_symbol({
     local function get_ipv6_variants(ip_str)
       local variants = {}
       local seen = {}
-      
+
       local function add_variant(v)
         if v and not seen[v] then
           table.insert(variants, v)
           seen[v] = true
         end
       end
-      
+
       -- For IPv4, just return the original
       if not ip_str:find(':') then
         add_variant(ip_str)
         return variants
       end
-      
+
       local segments = ipv6_to_segments(ip_str)
-      
+
       -- 1. Fully expanded form (all zeros shown as 0000)
       local expanded_parts = {}
       for _, seg in ipairs(segments) do
         table.insert(expanded_parts, string.format('%04x', seg))
       end
       add_variant(table.concat(expanded_parts, ':'))
-      
+
       -- 2. Standard form (no leading zeros, but all segments present)
       local standard_parts = {}
       for _, seg in ipairs(segments) do
         table.insert(standard_parts, string.format('%x', seg))
       end
       add_variant(table.concat(standard_parts, ':'))
-      
+
       -- 3. Find all possible :: compressions
       -- RFC 5952: compress the longest run of consecutive zeros
       -- But we need to check all possibilities since Redis might have any form
-      
+
       -- Find all zero runs
       local zero_runs = {}
       local in_run = false
       local run_start = 0
       local run_length = 0
-      
+
       for i = 1, 8 do
         if segments[i] == 0 then
           if not in_run then
@@ -266,21 +266,21 @@ rspamd_config:register_symbol({
           end
         end
       end
-      
+
       -- Don't forget the last run
       if in_run and run_length >= 1 then
         table.insert(zero_runs, {start = run_start, length = run_length})
       end
-      
+
       -- Generate variant for each zero run compression
       for _, run in ipairs(zero_runs) do
         local parts = {}
-        
+
         -- Before compression
         for i = 1, run.start - 1 do
           table.insert(parts, string.format('%x', segments[i]))
         end
-        
+
         -- The compression
         if run.start == 1 then
           table.insert(parts, '')
@@ -291,22 +291,22 @@ rspamd_config:register_symbol({
         else
           table.insert(parts, '')
         end
-        
+
         -- After compression
         for i = run.start + run.length, 8 do
           table.insert(parts, string.format('%x', segments[i]))
         end
-        
+
         local compressed = table.concat(parts, ':'):gsub('::+', '::')
         add_variant(compressed)
       end
-      
+
       return variants
     end
 
     local from_ip_string = tostring(ip)
     local ip_check_table = {}
-    
+
     -- Add all variants of the exact IP
     for _, variant in ipairs(get_ipv6_variants(from_ip_string)) do
       table.insert(ip_check_table, variant)
@@ -318,18 +318,18 @@ rspamd_config:register_symbol({
         maxbits = 32
         minbits = 8
     end
-    
+
     -- Add all CIDR notations with variants
     for i=maxbits,minbits,-1 do
       local masked_ip = ip:apply_mask(i)
       local cidr_base = masked_ip:to_string()
-      
+
       for _, variant in ipairs(get_ipv6_variants(cidr_base)) do
         local cidr = variant .. "/" .. i
         table.insert(ip_check_table, cidr)
       end
     end
-    
+
     local function keep_spam_cb(err, data)
       if err then
         rspamd_logger.infox(rspamd_config, "keep_spam query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
@@ -345,7 +345,7 @@ rspamd_config:register_symbol({
         end
       end
     end
-    
+
     table.insert(ip_check_table, 1, 'KEEP_SPAM')
     local redis_ret_user = rspamd_redis_make_request(task,
       redis_params, -- connect params
@@ -441,7 +441,7 @@ rspamd_config:register_symbol({
 
     -- Check if we have a pre-result (e.g., from KEEP_SPAM or POSTMASTER_HANDLER)
     local allow_processing = false
-    
+
     if task.has_pre_result then
       local has_pre, pre_action = task:has_pre_result()
       if has_pre then
@@ -765,13 +765,22 @@ rspamd_config:register_symbol({
   end
 })
 
+-- Expand a domain footer: replace only "{= key =}" placeholders with their values.
+-- Unknown placeholders render empty, matching the previous engine behaviour.
+local function expand_footer(template, replacements)
+  return (template:gsub('{=%s*([%w_]+)%s*=}', function(key)
+    local value = replacements[key]
+    if value == nil then return '' end
+    return tostring(value)
+  end))
+end
+
 rspamd_config:register_symbol({
   name = 'MOO_FOOTER',
   type = 'prefilter',
   callback = function(task)
     local cjson = require "cjson"
     local lua_mime = require "lua_mime"
-    local lua_util = require "lua_util"
     local rspamd_logger = require "rspamd_logger"
     local rspamd_http = require "rspamd_http"
     local envfrom = task:get_from(1)
@@ -844,10 +853,10 @@ rspamd_config:register_symbol({
               end
             end
             if footer.html and footer.html ~= "" then
-              footer.html = lua_util.jinja_template(footer.html, replacements, true)
+              footer.html = expand_footer(footer.html, replacements)
             end
             if footer.plain and footer.plain ~= "" then
-              footer.plain = lua_util.jinja_template(footer.plain, replacements, true)
+              footer.plain = expand_footer(footer.plain, replacements)
             end
 
             -- add footer

data/conf/rspamd/meta_exporter/pipe.php

@@ -216,8 +216,9 @@ foreach ($rcpts as $rcpt) {
 foreach ($rcpt_final_mailboxes as $rcpt_final) {
   error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt_final . PHP_EOL);
   try {
-    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`)
+    // qhash is the secret of the release/delete links in notifications, so it must not be derivable
-      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes)");
+    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`, `qhash`)
+      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes, :qhash)");
     $stmt->execute(array(
       ':qid' => $qid,
       ':subject' => $subject,
@@ -229,11 +230,9 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
       ':ip' => $ip,
       ':msg' => $raw_data,
       ':action' => $action,
-      ':fuzzy_hashes' => $fuzzy
+      ':fuzzy_hashes' => $fuzzy,
+      ':qhash' => bin2hex(random_bytes(32))
     ));
-    $lastId = $pdo->lastInsertId();
-    $stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
-    $stmt_update->execute(array(':id' => $lastId));
     $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
       SELECT `id`
       FROM (

data/web/admin/mailbox.php

@@ -11,6 +11,8 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
+$js_minifier->add('/web/js/site/mail-policy.js');
+$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -46,6 +48,8 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
+  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
+  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/restore.php

@@ -0,0 +1,8 @@
+<?php
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
+
+protect_route(['admin']);
+
+$role = "admin";
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/admin/system.php

@@ -13,6 +13,7 @@ $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 $js_minifier->add('/web/js/site/identity.js');
+$js_minifier->add('/web/js/site/backup.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -121,6 +122,8 @@ $template_data = [
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
   'iam_settings' => $iam_settings,
   'mbox_templates' => $mbox_templates,
+  'backup_timezone' => getenv('TZ') ?: 'UTC',
+  'backup_hostname' => getenv('MAILCOW_HOSTNAME') ?: '',
   'lang_admin' => json_encode($lang['admin']),
   'lang_datatables' => json_encode($lang['datatables'])
 ];

data/web/autodiscover.php

@@ -144,41 +144,15 @@ try {
 }
 
 $username = trim((string)$email);
-try {
+if (!filter_var($username, FILTER_VALIDATE_EMAIL)) {
-  $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox` 
-    INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
-    WHERE `mailbox`.`username` = :username 
-    AND `mailbox`.`active` = '1'
-    AND `domain`.`active` = '1'");
-  $stmt->execute(array(':username' => $username));
-  $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
-}
-catch(PDOException $e) {
-  // Database error - return error response with complete XML
-  list($usec, $sec) = explode(' ', microtime());
-?>
-  <Response>
-    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
-      <ErrorCode>500</ErrorCode>
-      <Message>Database Error</Message>
-      <DebugData />
-    </Error>
-  </Response>
-</Autodiscover>
-<?php
-  exit(0);
-}
-
-// Mailbox not found or not active - return generic error to prevent user enumeration
-if (empty($MailboxData)) {
   try {
     $json = json_encode(
       array(
         "time" => time(),
         "ua" => $_SERVER['HTTP_USER_AGENT'],
-        "user" => $email,
+        "user" => "none",
         "ip" => $_SERVER['REMOTE_ADDR'],
-        "service" => "Error: mailbox not found or inactive"
+        "service" => "Error: invalid email address in request"
       )
     );
     $redis->lPush('AUTODISCOVER_LOG', $json);
@@ -202,13 +176,33 @@ if (empty($MailboxData)) {
 <?php
   exit(0);
 }
+try {
+  $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox`
+    INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
+    WHERE `mailbox`.`username` = :username
+    AND `mailbox`.`active` = '1'
+    AND `domain`.`active` = '1'");
+  $stmt->execute(array(':username' => $username));
+  $MailboxData = $stmt->fetch(PDO::FETCH_ASSOC);
+}
+catch(PDOException $e) {
+  // Database error - return error response with complete XML
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>500</ErrorCode>
+      <Message>Database Error</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
 
-if (!empty($MailboxData['name'])) {
+$email = $username;
-  $displayname = $MailboxData['name'];
+$displayname = $email;
-}
-else {
-  $displayname = $email;
-}
 try {
   $json = json_encode(
     array(
@@ -216,11 +210,16 @@ try {
       "ua" => $_SERVER['HTTP_USER_AGENT'],
       "user" => $email,
       "ip" => $_SERVER['REMOTE_ADDR'],
-      "service" => $autodiscover_config['autodiscoverType']
+      // The answer is the same, only the log tells unknown or inactive mailboxes apart
+      "service" => (empty($MailboxData)) ? "Error: mailbox not found or inactive" : $autodiscover_config['autodiscoverType']
     )
   );
   $redis->lPush('AUTODISCOVER_LOG', $json);
   $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
+  if (empty($MailboxData)) {
+    $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+    error_log("Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
+  }
 }
 catch (RedisException $e) {
   $_SESSION['return'][] = array(
@@ -243,7 +242,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['imap']['server'];?></Server>
         <Port><?=$autodiscover_config['imap']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -253,7 +252,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['smtp']['server'];?></Server>
         <Port><?=$autodiscover_config['smtp']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -265,15 +264,15 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     ?>
       <Protocol>
         <Type>CalDAV</Type>
-        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
+        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
       </Protocol>
       <Protocol>
         <Type>CardDAV</Type>
-        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
+        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=$email;?></LoginName>
+        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
       </Protocol>
     <?php
     }
@@ -288,7 +287,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     <Culture>en:en</Culture>
     <User>
       <DisplayName><?=htmlspecialchars($displayname, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></DisplayName>
-      <EMailAddress><?=$email;?></EMailAddress>
+      <EMailAddress><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></EMailAddress>
     </User>
     <Action>
       <Settings>

data/web/css/site/admin.css

@@ -1,83 +1,133 @@
-.pagination a {
+.pagination a {
-  text-decoration: none !important;
+  text-decoration: none !important;
-}
+}
-.panel.panel-default {
+.panel.panel-default {
-  overflow: visible !important;
+  overflow: visible !important;
-}
+}
-.table-responsive {
+.table-responsive {
-  overflow: visible !important;
+  overflow: visible !important;
-}
+}
-.table-responsive {
+.table-responsive {
-  overflow-x: scroll !important;
+  overflow-x: scroll !important;
-}
+}
-body {
+body {
-  overflow-y:scroll;
+  overflow-y:scroll;
-}
+}
-/* Fix modal moving content left */
+/* Fix modal moving content left */
-body.modal-open {
+body.modal-open {
-  overflow-y:scroll;
+  overflow-y:scroll;
-  padding-right: inherit !important;
+  padding-right: inherit !important;
-}
+}
-@media (min-width: 992px) {
+@media (min-width: 992px) {
-  .container {
+  .container {
-      width: 80%;
+      width: 80%;
-  }
+  }
-}
+}
-.mass-actions-admin {
+.mass-actions-admin {
-  user-select: none;
+  user-select: none;
-}
+}
-.inputMissingAttr {
+.inputMissingAttr {
-  border-color: #FF4136;
+  border-color: #FF4136;
-}
+}
-.rotate {
+.rotate {
-  -moz-transition: all 0.3s linear;
+  -moz-transition: all 0.3s linear;
-  -webkit-transition: all 0.3s linear;
+  -webkit-transition: all 0.3s linear;
-  transition: all 0.3s linear;
+  transition: all 0.3s linear;
-}
+}
-.rotate.animation {
+.rotate.animation {
-  -ms-transform:rotateX(180deg);
+  -ms-transform:rotateX(180deg);
-  -moz-transform:rotateX(180deg);
+  -moz-transform:rotateX(180deg);
-  -webkit-transform:rotateX(180deg);
+  -webkit-transform:rotateX(180deg);
-  transform:rotateX(180deg);
+  transform:rotateX(180deg);
-}
+}
-.anchor {
+.anchor {
-  display: block;
+  display: block;
-  height: 65px;
+  height: 65px;
-  margin-top: -65px;
+  margin-top: -65px;
-  visibility: hidden;
+  visibility: hidden;
-}
+}
-.thumbnail img {
+.thumbnail img {
-  min-height:100px;
+  min-height:100px;
-  height:100px;
+  height:100px;
-}
+}
-.nav-tabs>li>a {
+.nav-tabs>li>a {
-  z-index: 1;
+  z-index: 1;
-}
+}
-.table-condensed .input-sm {
+.table-condensed .input-sm {
-  width: 100%!important;
+  width: 100%!important;
-}
+}
-.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
+.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
-  padding: 3px;
+  padding: 3px;
-}
+}
-table tbody tr td input[type="checkbox"] {
+table tbody tr td input[type="checkbox"] {
-  cursor: pointer;
+  cursor: pointer;
-}
+}
-#quarantine_template {
+#quarantine_template {
-  margin:20px;
+  margin:20px;
-}
+}
-.regex-input {
+.regex-input {
-  font-family: Consolas,monaco,monospace;
+  font-family: Consolas,monaco,monospace;
-  font-size: 1rem;
+  font-size: 1rem;
-}
+}
-.label-keys {
+.label-keys {
-  font-size:100%;
+  font-size:100%;
-  margin: 0px !important;
+  margin: 0px !important;
-  white-space: normal !important;
+  white-space: normal !important;
-}
+}
-.key-action {
+.key-action {
-  font-weight:bold;
+  font-weight:bold;
-  color:white !important;
+  color:white !important;
-}
+}
-.dkim-label {
+.dkim-label {
-  margin: 0 0 8px !important;
+  margin: 0 0 8px !important;
-}
+}
+/* backup schedule: the weekday toggles and the remove buttons of the entries react to the mouse, in every theme */
+.backup-schedule-days .btn:hover {
+  background-image: linear-gradient(rgba(128, 128, 128, .3), rgba(128, 128, 128, .3));
+}
+.backup-schedule-remove {
+  opacity: .6;
+}
+.backup-schedule-remove:hover, .backup-schedule-remove:focus-visible {
+  opacity: 1;
+  color: var(--bs-danger) !important;
+}
+/* result and log of a backup job: the border of the theme in light and dark mode (Bootstrap's .border stays light) */
+.backup-pre {
+  border: 1px solid var(--mc-border);
+}
+/* result of a restore job: an entry to open per mailbox, numbers in columns */
+.backup-result {
+  border: 1px solid var(--mc-border);
+}
+.backup-result > details + details {
+  border-top: 1px solid var(--mc-border);
+}
+.backup-result summary {
+  list-style: none;
+  cursor: pointer;
+}
+.backup-result summary::-webkit-details-marker {
+  display: none;
+}
+.backup-result summary:hover {
+  background-image: linear-gradient(rgba(128, 128, 128, .12), rgba(128, 128, 128, .12));
+}
+.backup-result summary .bi-chevron-right {
+  transition: transform .15s;
+}
+.backup-result details[open] > summary .bi-chevron-right {
+  transform: rotate(90deg);
+}
+.backup-result-body {
+  padding-left: 2.6rem !important;
+}
+.backup-result-body .table {
+  --bs-table-bg: transparent;
+}
+.backup-result-body .num {
+  width: 8.5rem;
+  text-align: right;
+  font-variant-numeric: tabular-nums;
+  white-space: nowrap;
+}

data/web/css/themes/mailcow-darkmode.css

@@ -495,4 +495,16 @@ input::placeholder {
 }
 .btn-light, .btn-light:hover {
     background-image: none;
-}
+}
+/* native time picker icon and popup in dark mode */
+input[type="time"] {
+    color-scheme: dark;
+}
+/* switches while off: a light knob and a visible border (Bootstrap's dark knob is for light pages, the dark mode
+   does not set data-bs-theme) */
+.form-switch .form-check-input:not(:checked):not(:focus) {
+    --bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.6%29'/%3e%3c/svg%3e");
+}
+.form-switch .form-check-input:not(:checked) {
+    border-color: var(--mc-btn-outline-border);
+}

data/web/domainadmin/mailbox.php

@@ -11,6 +11,8 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
+$js_minifier->add('/web/js/site/mail-policy.js');
+$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -45,6 +47,8 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
+  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
+  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/domainadmin/restore.php

@@ -0,0 +1,8 @@
+<?php
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.domainadmin.inc.php';
+
+protect_route(['domainadmin']);
+
+$role = "domainadmin";
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/edit.php

@@ -115,6 +115,28 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
         ];
     }
+    elseif (isset($_GET['mail-policy']) &&
+      is_numeric($_GET['mail-policy'])) {
+        $mail_policy = \Mailcow\MailPolicy\MailPolicyModule::fromGlobals();
+        $result = $mail_policy->api->getPolicy($_GET['mail-policy']);
+        $template = 'edit/mail-policy.twig';
+        // Choices: mailboxes and groups of the domain of the policy
+        $template_data = [
+          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
+          'mail_policy_groups' => !empty($result) ? $mail_policy->policies->groupChoices([$result['domain']]) : [],
+        ];
+    }
+    elseif (isset($_GET['retention-rule']) &&
+      is_numeric($_GET['retention-rule'])) {
+        $retention = \Mailcow\Retention\RetentionModule::fromGlobals();
+        $result = $retention->api->getRule($_GET['retention-rule']);
+        $template = 'edit/retention-rule.twig';
+        // Choices: mailboxes and groups of the domain of the rule
+        $template_data = [
+          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
+          'retention_groups' => !empty($result) ? $retention->rules->groupChoices([$result['domain']]) : [],
+        ];
+    }
     elseif (isset($_GET['api-key']) &&
       is_numeric($_GET['api-key'])) {
         $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
@@ -227,7 +249,8 @@ if (isset($_SESSION['mailcow_cc_role'])) {
     if (isset($_GET['syncjob']) &&
       is_numeric($_GET['syncjob'])) {
         $id = $_GET["syncjob"];
-        $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id);
+        // Without the syncjobs ACL a job cannot be saved, so do not show the form
+        $result = (($_SESSION['acl']['syncjobs'] ?? '0') == '1') ? \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id) : [];
         $template = 'edit/syncjob.twig';
       }
     elseif (isset($_GET['syncjob_source']) &&

data/web/inc/footer.inc.php

@@ -74,6 +74,9 @@ $globalVariables = [
   'lang_danger' => json_encode($lang['danger']),
   'lang_syncjobs' => json_encode($lang['syncjobs']),
   'lang_groups' => json_encode($lang['groups']),
+  'lang_mail_policy' => json_encode($lang['mail_policy'] ?? []),
+  'lang_retention' => json_encode($lang['retention'] ?? []),
+  'lang_backup' => json_encode($lang['backup'] ?? []),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.admin.inc.php

@@ -74,6 +74,15 @@ function admin($_action, $_data = null) {
         $usernames = $_data['username'];
       }
       foreach ($usernames as $username) {
+        // A pending password update only allows changing the own password
+        if (!empty($_SESSION['pending_pw_update']) && $username != $_SESSION['mailcow_cc_username']) {
+          $_SESSION['return'][] = array(
+            'type' => 'danger',
+            'log' => array(__FUNCTION__, $_action, $_data_log),
+            'msg' => 'access_denied'
+          );
+          continue;
+        }
         $is_now = admin('details', $username);
         if (!empty($is_now)) {
           $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];

data/web/inc/functions.app_passwd.inc.php

@@ -5,6 +5,15 @@ function app_passwd($_action, $_data = null) {
   $_data_log = $_data;
   !isset($_data_log['app_passwd']) ?: $_data_log['app_passwd'] = '*';
   !isset($_data_log['app_passwd2']) ?: $_data_log['app_passwd2'] = '*';
+  if (in_array($_action, array('add', 'edit', 'delete')) &&
+    (!isset($_SESSION['acl']['app_passwds']) || $_SESSION['acl']['app_passwds'] != "1")) {
+    $_SESSION['return'][] = array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $_action, $_data_log),
+      'msg' => 'access_denied'
+    );
+    return false;
+  }
   if (isset($_data['username']) && filter_var($_data['username'], FILTER_VALIDATE_EMAIL)) {
     if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data['username'])) {
       $_SESSION['return'][] = array(

data/web/inc/functions.auth.inc.php

@@ -6,6 +6,8 @@ function check_login($user, $pass, $extra = null) {
   $is_internal = $extra['is_internal'];
   $role = $extra['role'];
   $extra['service'] = !isset($extra['service']) ? 'NONE' : $extra['service'];
+  // set when the password is right, but the account does not allow this login
+  $denied = false;
 
   // Try validate admin
   if (!isset($role) || $role == "admin") {
@@ -26,7 +28,7 @@ function check_login($user, $pass, $extra = null) {
 
   // Try validate app password
   if (!isset($role) || $role == "app") {
-    $result = apppass_login($user, $pass, $extra);
+    $result = apppass_login($user, $pass, $extra, $denied);
     if ($result !== false) {
       $real_rip = ($_SERVER['HTTP_X_REAL_IP'] ?? $_SERVER['REMOTE_ADDR']);
       set_sasl_log($user, $real_rip, $extra['service'], $pass);
@@ -37,7 +39,7 @@ function check_login($user, $pass, $extra = null) {
   // Try validate user
   if (!isset($role) || $role == "user") {
     try {
-      $result = user_login($user, $pass, $extra);
+      $result = user_login($user, $pass, $extra, $denied);
     } catch (\Mailcow\Auth\IdpUnavailable $e) {
       // no failed login, netfilter must not count it
       error_log("mailcow UI: Identity provider unavailable for " . $user . " by " . $_SERVER['REMOTE_ADDR'] . ": " . $e->getMessage());
@@ -54,19 +56,20 @@ function check_login($user, $pass, $extra = null) {
       return $result;
     }
   }
-
   // skip log and only return false if it's an internal request
   if ($is_internal == true) return false;
 
+  // A denied login is logged, but not counted by netfilter (e.g. EAS/DAV clients of accounts without access)
+  $msg = ($denied ? "mailcow UI: Login denied for " : "mailcow UI: Invalid password for ") . $user . " by " . $_SERVER['REMOTE_ADDR'];
   if (!isset($_SESSION['ldelay'])) {
     $_SESSION['ldelay'] = "0";
-    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
-    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    error_log($msg);
   }
   elseif (!isset($_SESSION['mailcow_cc_username'])) {
     $_SESSION['ldelay'] = $_SESSION['ldelay']+0.5;
-    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
-    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
+    error_log($msg);
   }
   $_SESSION['return'][] =  array(
     'type' => 'danger',
@@ -207,7 +210,7 @@ function domainadmin_login($user, $pass){
 
   return false;
 }
-function user_login($user, $pass, $extra = null){
+function user_login($user, $pass, $extra = null, &$denied = false){
   global $pdo;
   global $iam_provider;
   global $iam_settings;
@@ -261,11 +264,14 @@ function user_login($user, $pass, $extra = null){
         if ($extra['service'] != 'NONE') {
           $key = strtolower($extra['service']) . "_access";
           if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
+            $denied = true;
             return false;
           }
         }
         return true;
       }
+      // the IdP accepted the password, the mailbox is inactive
+      $denied = true;
     }
     clear_session();
     return false;
@@ -273,18 +279,16 @@ function user_login($user, $pass, $extra = null){
 
   // check if user has access to service (imap, smtp, pop3, sieve) if service is set
   $row['attributes'] = json_decode($row['attributes'], true);
-  // Shared mailboxes have no login of their own, members open them through group shares
+  $key = strtolower($extra['service']) . "_access";
-  if (($row['attributes']['shared'] ?? '0') == '1') {
+  // Shared mailboxes have no login of their own, members open them through group shares.
-    return false;
+  // Checked before asking an IdP (no template side effects).
-  }
+  if (($row['attributes']['shared'] ?? '0') == '1'
-  if ($extra['service'] != 'NONE') {
+    || ($extra['service'] != 'NONE' && isset($row['attributes'][$key]) && $row['attributes'][$key] != '1')
-    $key = strtolower($extra['service']) . "_access";
+    || $row['active'] != '1' || $row['d_active'] != '1') {
-    if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
+    // Denied, not failed, unless the password is wrong; IdP passwords are not checked for that
-      return false;
+    if ($row['authsource'] != 'mailcow' || verify_hash($row['password'], $pass) !== false) {
+      $denied = true;
     }
-  }
-  // IdP accounts: refuse inactive mailboxes and domains before asking the IdP (no template side effects)
-  if ($row['authsource'] != 'mailcow' && ($row['active'] != '1' || $row['d_active'] != '1')) {
     return false;
   }
   switch ($row['authsource']) {
@@ -303,6 +307,7 @@ function user_login($user, $pass, $extra = null){
           $stmt->execute(array(':user' => $user));
           $row = $stmt->fetch(PDO::FETCH_ASSOC);
           if (empty($row)) {
+            $denied = true;
             return false;
           }
 
@@ -361,6 +366,7 @@ function user_login($user, $pass, $extra = null){
         $stmt->execute(array(':user' => $user));
         $row = $stmt->fetch(PDO::FETCH_ASSOC);
         if (empty($row)) {
+          $denied = true;
           return false;
         }
 
@@ -403,9 +409,6 @@ function user_login($user, $pass, $extra = null){
       return $result;
     break;
     case 'mailcow':
-      if ($row['active'] != 1 || $row['d_active'] != 1) {
-        return false;
-      }
       // verify password
       if (verify_hash($row['password'], $pass) !== false) {
 
@@ -446,13 +449,15 @@ function user_login($user, $pass, $extra = null){
           }
           return "user";
         }
+        // TFA accounts use app passwords in mail clients
+        $denied = true;
       }
     break;
   }
 
   return false;
 }
-function apppass_login($user, $pass, $extra = null){
+function apppass_login($user, $pass, $extra = null, &$denied = false){
   global $pdo;
 
   $is_internal = $extra['is_internal'];
@@ -473,15 +478,12 @@ function apppass_login($user, $pass, $extra = null){
     return false;
   }
 
-  // fetch app password data
+  // fetch app password data, inactive ones too: their right password is denied, not failed
-  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id` FROM `app_passwd`
+  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id`,
+    `mailbox`.`active` as `mailbox_active`, `domain`.`active` as `domain_active`, `mailbox`.`attributes` as `mailbox_attributes` FROM `app_passwd`
     INNER JOIN `mailbox` ON `mailbox`.`username` = `app_passwd`.`mailbox`
     INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
     WHERE `mailbox`.`kind` NOT REGEXP 'location|thing|group'
-      AND `mailbox`.`active` = '1'
-      AND `domain`.`active` = '1'
-      AND `app_passwd`.`active` = '1'
-      AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
       AND `app_passwd`.`mailbox` = :user"
   );
   // fetch password data
@@ -491,16 +493,25 @@ function apppass_login($user, $pass, $extra = null){
   $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
 
   foreach ($rows as $row) {
-    if ($extra['service'] != 'NONE' && $row[strtolower($extra['service']) . '_access'] != '1'){
+    // verify password
+    if (verify_hash($row['password'], $pass) === false) {
       continue;
     }
 
-    // verify password
+    $key = strtolower($extra['service']) . '_access';
-    if (verify_hash($row['password'], $pass) !== false) {
+    $mailbox_attributes = json_decode($row['mailbox_attributes'], true);
-      $_SESSION['app_passwd_id'] = $row['app_passwd_id'];
+    // the protocols of the mailbox only limit direct user access, app passwords have their own
-      unset($_SESSION['ldelay']);
+    if ($row['active'] != '1' || $row['mailbox_active'] != '1' || $row['domain_active'] != '1'
-      return "user";
+      || ($mailbox_attributes['shared'] ?? '0') == '1'
+      || ($extra['service'] != 'NONE' && $row[$key] != '1')) {
+      // right password, but the app password does not allow this login
+      $denied = true;
+      continue;
     }
+
+    $_SESSION['app_passwd_id'] = $row['app_passwd_id'];
+    unset($_SESSION['ldelay']);
+    return "user";
   }
 
   return false;
@@ -538,7 +549,8 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
 
   // get the mailcow_password attribute from keycloak user
   $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users";
-  $queryParams = array('email' => $user, 'exact' => true);
+  // as string, http_build_query() would send exact=1 and Keycloak would search for substrings
+  $queryParams = array('email' => $user, 'exact' => 'true');
   $queryString = http_build_query($queryParams);
   $curl = curl_init();
   curl_setopt($curl, CURLOPT_TIMEOUT, 7);
@@ -559,7 +571,10 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   if ($code != 200) {
     throw new \Mailcow\Auth\IdpUnavailable('Keycloak returned HTTP ' . $code);
   }
-  // exact=true can still return several users with that email
+  // exact=true can still return several users with that email, only the exact address counts
+  if (is_array($users)) {
+    $users = array_values(array_filter($users, fn($u) => is_array($u) && strcasecmp((string)($u['email'] ?? ''), $user) === 0));
+  }
   if (!is_array($users) || count($users) != 1) {
     return false;
   }
@@ -710,12 +725,15 @@ function ldap_mbox_login($user, $pass, $extra = null){
       ->select(array_values(array_filter(array_merge([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn'],
         \Mailcow\Identity\Directory\LdapDirectory::idAttributes($iam_settings['id_attribute'] ?? '')))));
 
-    $user_res = $ldap_query->firstOrFail();
+    $entries = $ldap_query->get();
-  } catch (\LdapRecord\Query\ObjectNotFoundException $e) {
-    return false;
   } catch (Exception $e) {
     throw new \Mailcow\Auth\IdpUnavailable('LDAP search failed: ' . $e->getMessage(), 0, $e);
   }
+  // the login name must identify exactly one entry, several matches are ambiguous
+  if (!is_array($entries) || count($entries) != 1) {
+    return false;
+  }
+  $user_res = reset($entries);
   $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
   if (!$idp_login->verifyLdapPassword($iam_provider, (string)$user_res['dn'], (string)$pass)) {
     return false;

data/web/inc/functions.fail2ban.inc.php

@@ -145,6 +145,28 @@ function fail2ban($_action, $_data = null, $_extra = null) {
           );
           return true;
         }
+        // Queue all automatic bans for unbanning, the denylist stays
+        elseif ($_data['action'] == "unban-all") {
+          try {
+            foreach ($redis->hKeys('F2B_ACTIVE_BANS') as $network) {
+              $redis->hSet('F2B_QUEUE_UNBAN', $network, 1);
+            }
+          }
+          catch (RedisException $e) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_data_log),
+              'msg' => array('redis_error', $e)
+            );
+            return false;
+          }
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_data_log),
+            'msg' => 'f2b_modified'
+          );
+          return true;
+        }
 
         // Start actions in dependency of network
         if (!empty($_data['network'])) {

data/web/inc/functions.inc.php

@@ -1857,6 +1857,40 @@ function get_tfa($username = null, $id = null) {
     }
 }
 function verify_tfa_login($username, $_data) {
+  global $redis;
+  global $TFA_MAX_FAILS;
+  global $TFA_FAIL_WINDOW;
+
+  if (empty($username)) {
+    return false;
+  }
+  // Count failed second factors per account, so codes cannot be guessed from many addresses
+  $fails_key = 'TFA_FAILS_' . $username;
+  if (intval($redis->get($fails_key)) >= $TFA_MAX_FAILS) {
+    $_SESSION['return'][] =  array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $username, '*'),
+      'msg' => 'tfa_login_locked'
+    );
+    $verified = false;
+  }
+  else {
+    $verified = verify_tfa_token($username, $_data);
+  }
+  if ($verified !== true) {
+    // The window starts with the first failure and is not extended by further attempts
+    if ($redis->incr($fails_key) == 1) {
+      $redis->expire($fails_key, $TFA_FAIL_WINDOW);
+    }
+    // Same message format as failed passwords, so the default fail2ban rule counts it
+    error_log("mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
+    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
+    return false;
+  }
+  $redis->del($fails_key);
+  return true;
+}
+function verify_tfa_token($username, $_data) {
   global $pdo;
   global $yubi;
   global $u2f;
@@ -3013,6 +3047,18 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
+      // A refresh keeps the logged-in mailbox: a changed IdP email or another IdP user ends the session
+      if (strtolower($info['email']) !== strtolower($_SESSION['mailcow_cc_username'] ?? '') ||
+        !\Mailcow\Auth\AuthModule::fromGlobals()->idp->verifyLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
+        clear_session();
+        $_SESSION['return'][] =  array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $info['email'], 'The IdP user does not match the logged-in mailbox'),
+          'msg' => 'refresh_login_failed'
+        );
+        return false;
+      }
+
       // The mailbox may have been deleted or deactivated since the login
       $stmt = $pdo->prepare("SELECT `mailbox`.`authsource` FROM `mailbox`
         INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`

data/web/inc/functions.mailbox.inc.php

@@ -2432,6 +2432,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               if ($address != $is_now['address']) {
                 // Send-as grants of groups follow the address
                 \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
+                \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
               }
             }
             $_SESSION['return'][] = array(
@@ -3295,7 +3296,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'mailbox_group_imap_grant' => ['owner'],
               'mailbox_group_sogo_grant' => ['owner'],
               'mailbox_group_sogo_acl' => ['c_uid'],
-              'mailbox_group_subscription' => ['username', 'owner']
+              'mailbox_group_subscription' => ['username', 'owner'],
+              'mail_policy_member' => ['username'],
+              'retention_rule_member' => ['username']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3311,6 +3314,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
             // Group memberships and send-as grants follow, group aliases are rebuilt
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
+            // Address entries of mail policies follow
+            \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($old_username, $new_username);
 
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
@@ -5348,6 +5353,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ':domain' => $domain,
             ));
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
+            \Mailcow\Retention\RetentionModule::fromGlobals()->api->domainDeleted($domain);
             $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,

data/web/inc/functions.quarantine.inc.php

@@ -767,8 +767,9 @@ function quarantine($_action, $_data = null) {
       else {
         $domains = array_merge(mailbox('get', 'domains'), mailbox('get', 'alias_domains'));
         foreach ($domains as $domain) {
-          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE `rcpt` REGEXP :domain');
+          // Compare the domain part exactly, a regex would treat dots in the domain as wildcards
-          $stmt->execute(array(':domain' => '@' . $domain . '$'));
+          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE SUBSTRING_INDEX(`rcpt`, "@", -1) = :domain');
+          $stmt->execute(array(':domain' => $domain));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while($row = array_shift($rows)) {
             $q_meta[] = $row;

data/web/inc/init_db.inc.php

@@ -491,6 +491,257 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
+      // Mail policies: whom the assigned mailboxes and groups may send to and receive from (Mailcow\MailPolicy)
+      "mail_policy" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "name" => "VARCHAR(100) NOT NULL",
+          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          // all, internal or list; the entries are JSON lists of addresses, domains and group:<id>
+          "send_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
+          "send_allow" => "MEDIUMTEXT NOT NULL",
+          "send_block" => "MEDIUMTEXT NOT NULL",
+          "receive_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
+          "receive_allow" => "MEDIUMTEXT NOT NULL",
+          "receive_block" => "MEDIUMTEXT NOT NULL",
+          "reject_text" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "domain_name" => array("domain", "name")
+          ),
+          "fkey" => array(
+            "fk_mail_policy_domain" => array(
+              "col" => "domain",
+              "ref" => "domain.domain",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // a mailbox or a group (its members and its address) a policy is assigned to
+      "mail_policy_member" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "policy_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NULL DEFAULT NULL",
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "policy_mailbox" => array("policy_id", "username"),
+            "policy_group" => array("policy_id", "group_id")
+          ),
+          "key" => array(
+            "username" => array("username"),
+            "group_id" => array("group_id")
+          ),
+          "fkey" => array(
+            "fk_mail_policy_member_policy" => array(
+              "col" => "policy_id",
+              "ref" => "mail_policy.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mail_policy_member_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_mail_policy_member_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "retention_rule" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "name" => "VARCHAR(100) NOT NULL",
+          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          // JSON list: trash, junk, sent, drafts, archive, inbox, all, folder:<name>
+          "folders" => "TEXT NOT NULL",
+          "days" => "INT NOT NULL DEFAULT '30'",
+          "keep_flagged" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          // simulate (count only) or delete
+          "mode" => "VARCHAR(16) NOT NULL DEFAULT 'simulate'",
+          "all_mailboxes" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "domain_name" => array("domain", "name")
+          ),
+          "fkey" => array(
+            "fk_retention_rule_domain" => array(
+              "col" => "domain",
+              "ref" => "domain.domain",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // a mailbox or a group (its members of the rule's domain) a retention rule is assigned to
+      "retention_rule_member" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "rule_id" => "INT NOT NULL",
+          "username" => "VARCHAR(255) NULL DEFAULT NULL",
+          "group_id" => "INT NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "unique" => array(
+            "rule_mailbox" => array("rule_id", "username"),
+            "rule_group" => array("rule_id", "group_id")
+          ),
+          "key" => array(
+            "username" => array("username"),
+            "group_id" => array("group_id")
+          ),
+          "fkey" => array(
+            "fk_retention_rule_member_rule" => array(
+              "col" => "rule_id",
+              "ref" => "retention_rule.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_retention_rule_member_mailbox" => array(
+              "col" => "username",
+              "ref" => "mailbox.username",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            ),
+            "fk_retention_rule_member_group" => array(
+              "col" => "group_id",
+              "ref" => "mailbox_group.id",
+              "delete" => "CASCADE",
+              "update" => "NO ACTION"
+            )
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // results of the nightly run: per mailbox and rule with found mail or an error (folder: the folder names), per
+      // rule a summary (username '') and the whole run (username '', rule_id NULL); kept 90 days, of simulating
+      // rules only the last run
+      "retention_log" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "domain" => "VARCHAR(255) NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "rule_id" => "INT NULL DEFAULT NULL",
+          "rule_name" => "VARCHAR(100) NOT NULL DEFAULT ''",
+          "folder" => "TEXT NULL DEFAULT NULL",
+          "days" => "INT NOT NULL DEFAULT '0'",
+          "mailboxes" => "INT NOT NULL DEFAULT '0'",
+          "matched" => "INT NOT NULL DEFAULT '0'",
+          "deleted" => "INT NOT NULL DEFAULT '0'",
+          "errors" => "INT NOT NULL DEFAULT '0'",
+          "simulated" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "error" => "TEXT NULL DEFAULT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "domain_created" => array("domain", "created"),
+            "rule_created" => array("rule_id", "username", "created"),
+            "created" => array("created")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_setting" => array(
+        "cols" => array(
+          "name" => "VARCHAR(64) NOT NULL",
+          "value" => "TEXT NOT NULL",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("name")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_job" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "type" => "ENUM('backup', 'check', 'prune', 'init', 'restore') NOT NULL",
+          "state" => "ENUM('queued', 'running', 'done', 'failed', 'cancelled') NOT NULL DEFAULT 'queued'",
+          "step" => "VARCHAR(32) NULL DEFAULT NULL",
+          "params" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "result" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "log" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "error" => "TEXT NULL DEFAULT NULL",
+          "domain" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "started" => "DATETIME NULL DEFAULT NULL",
+          "finished" => "DATETIME NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "state_type" => array("state", "type"),
+            "domain_created" => array("domain", "created")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      "backup_snapshot" => array(
+        "cols" => array(
+          "id" => "VARCHAR(64) NOT NULL",
+          "short_id" => "VARCHAR(16) NOT NULL",
+          "time" => "DATETIME NOT NULL",
+          "hostname" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "tags" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "summary" => "TEXT NULL DEFAULT NULL",
+          "domains" => "MEDIUMTEXT NULL DEFAULT NULL",
+          "content" => "MEDIUMBLOB NULL DEFAULT NULL"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "time" => array("time")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
       "mailbox_group_imap_grant" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -703,7 +954,8 @@ function init_db_schema()
             "group_id" => array("group_id")
           ),
           "key" => array(
-            "domain" => array("domain")
+            "domain" => array("domain"),
+            "internal" => array("internal", "active")
           ),
           "fkey" => array(
             "fk_alias_mailbox_group" => array(
@@ -1053,7 +1305,9 @@ function init_db_schema()
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "mail_policies" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "retention" => "TINYINT(1) NOT NULL DEFAULT '0'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1098,6 +1352,7 @@ function init_db_schema()
           "host1" => "VARCHAR(255) NOT NULL",
           "port1" => "SMALLINT UNSIGNED NOT NULL",
           "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'",
+          "verify_cert" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "auth_type" => "ENUM('PLAIN','LOGIN','CRAM-MD5','XOAUTH2') DEFAULT 'PLAIN'",
           "oauth_flow" => "ENUM('client_credentials','authorization_code') NOT NULL DEFAULT 'client_credentials'",
           "oauth_token_endpoint" => "VARCHAR(500) DEFAULT NULL",
@@ -1679,9 +1934,15 @@ function init_db_schema()
             // Converted JSON may not fit into the old VARCHAR(512)
             $pdo->exec("ALTER TABLE `imapsync` MODIFY COLUMN `custom_params` TEXT");
             $imapsync_allow = \Mailcow\Imapsync\Model\CustomParams::allowlist($GLOBALS['IMAPSYNC_OPTIONS'] ?? [], $GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? []);
-            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
+            // A legacy source is found by its connection and its only target domain (none for jobs without domain)
-            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `auth_type`, `active`)
+            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` s WHERE `created_by` = '' AND `host1` = :host AND `port1` = :port
-              VALUES (:name, '', '', 'domain', :host, :port, :enc, :auth, 1)");
+              AND `enc1` = :enc AND `auth_type` = :auth
+              AND (SELECT GROUP_CONCAT(`domain`) FROM `imapsync_source_domain` d WHERE d.`source_id` = s.`id`) <=> NULLIF(:domain, '')
+              ORDER BY `id` LIMIT 1");
+            $source_name_sel = $pdo->prepare("SELECT COUNT(*) FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
+            // The old runner never checked certificates, legacy sources keep that
+            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `verify_cert`, `auth_type`, `active`)
+              VALUES (:name, 'Migrated sync jobs', '', 'domain', :host, :port, :enc, 0, :auth, 1)");
             $source_domain_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)");
             $job_upd = $pdo->prepare("UPDATE `imapsync` SET `source_id` = :source_id, `custom_params` = :cp, `custom_params_legacy` = :cp_legacy,
               `dry` = GREATEST(`dry`, :dry) WHERE `id` = :id AND `source_id` IS NULL");
@@ -1741,16 +2002,20 @@ function init_db_schema()
               $enc = in_array($row['enc1'], array('TLS', 'SSL', 'PLAIN'), true) ? $row['enc1'] : 'TLS';
               $at = strrchr((string)$row['user2'], '@');
               $domain = ($at === false) ? '' : strtolower(substr($at, 1));
-              // One source per (host, port, enc, auth, domain), name always unique and <= 100 chars
+              // One source per (host, port, enc, auth, domain), named "host (domain)" plus " 2", " 3"... when taken (<= 100 chars)
-              $name = 'legacy-' . mb_substr($host, 0, 60) . '-' . $port . '-' . substr(sha1($host . '|' . $port . '|' . $enc . '|' . $auth . '|' . $domain), 0, 8);
+              $conn = array(':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth);
-              $source_sel->execute(array(':name' => $name));
+              $source_sel->execute($conn + array(':domain' => $domain));
               if (($source_id = $source_sel->fetchColumn()) === false) {
-                $source_ins->execute(array(':name' => $name, ':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth));
+                $base = mb_substr($host . ($domain !== '' ? ' (' . $domain . ')' : ''), 0, 94);
-                $source_sel->execute(array(':name' => $name));
+                for ($i = 1, $name = $base; $i < 1000; $name = $base . ' ' . ++$i) {
-                if (($source_id = $source_sel->fetchColumn()) === false) continue;
+                  $source_name_sel->execute(array(':name' => $name));
-              }
+                  if ($source_name_sel->fetchColumn() == 0) break;
-              if ($domain !== '') {
+                }
-                $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
+                $source_ins->execute($conn + array(':name' => $name));
+                if (($source_id = $pdo->lastInsertId()) == 0) continue;
+                if ($domain !== '') {
+                  $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
+                }
               }
               $job_upd->execute(array(
                 ':source_id' => $source_id,

data/web/inc/lib/mailcow/Backup/BackupModule.php

@@ -0,0 +1,87 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup;
+
+use Mailcow\Backup\Controller\BackupApiController;
+use Mailcow\Backup\Legacy\LegacyDomain;
+use Mailcow\Backup\Legacy\NoTemplates;
+use Mailcow\Backup\Model\Settings;
+use Mailcow\Backup\Repository\ExportRepository;
+use Mailcow\Backup\Repository\JobRepository;
+use Mailcow\Backup\Repository\RestoreRepository;
+use Mailcow\Backup\Repository\SettingsRepository;
+use Mailcow\Backup\Repository\SnapshotRepository;
+use Mailcow\Backup\Service\DomainRestoreService;
+use Mailcow\Backup\Service\ExportService;
+use Mailcow\Backup\Service\GroupMembers;
+use Mailcow\Backup\Service\JobService;
+use Mailcow\Backup\Service\RestoreService;
+use Mailcow\Backup\Service\SettingsService;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\Dovecot\DovecotModule;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+use Mailcow\MailboxGroup\Service\ChangeEffects;
+use Mailcow\MailboxGroup\Service\SharingService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+use Mailcow\Sogo\SogoModule;
+use PDO;
+use Redis;
+
+/**
+ * Composition root of the backup with restic: settings, jobs of the runners, snapshots, the export of a domain and the
+ * restore of mailboxes and domains. The runner in the restic container reaches the export and the restore actions
+ * through data/conf/rspamd/dynmaps/backup.php.
+ *
+ *   process_get_return(BackupModule::fromGlobals()->api->getSettings());
+ *   echo json_encode(BackupModule::create($pdo, Context::system(), $config, $redis)->api->export($domain, $page));  // backup.php
+ *
+ * $config: timezone (TZ of mailcow, for the schedule), alias_disable_external_domains ($ALIAS_DISABLE_EXTERNAL_DOMAINS).
+ * $redis: for DKIM keys, rate limits and the domain map (export and restore).
+ */
+class BackupModule
+{
+    public function __construct(
+        public readonly SettingsService $settings,
+        public readonly JobService $jobs,
+        public readonly ExportService $export,
+        public readonly RestoreService $restore,
+        public readonly DomainRestoreService $domainRestore,
+        public readonly BackupApiController $api
+    ) {
+    }
+
+    public static function create(PDO $pdo, Context $context, array $config = [], ?Redis $redis = null): self
+    {
+        $jobRepo = new JobRepository($pdo);
+        $restoreRepo = new RestoreRepository($pdo);
+        $legacy = new LegacyDomain($redis);
+        $settings = new SettingsService(new SettingsRepository($pdo), $context);
+        $jobs = new JobService($jobRepo, new SnapshotRepository($pdo), $settings, $context, (string)($config['timezone'] ?? 'UTC'));
+        $export = new ExportService(new ExportRepository($pdo), $legacy);
+        $sogo = SogoModule::create()->sogo;
+        $dovecot = DovecotModule::create()->dovecot;
+        // the group module without group templates: a restored mailbox keeps the settings of the backup
+        $groupRepo = new GroupRepository($pdo);
+        $lookup = new LookupRepository($pdo);
+        $templates = new TemplateService($lookup, new NoTemplates());
+        $effects = new ChangeEffects($templates, $sogo, $groupRepo, $dovecot);
+        $groups = new GroupMembers($groupRepo, $lookup, $templates, $effects, new SharingService($groupRepo, $dovecot, $sogo, $effects));
+        $restore = new RestoreService($restoreRepo, $jobRepo, $groups, $sogo, $dovecot, $legacy, (bool)($config['alias_disable_external_domains'] ?? false));
+        $domainRestore = new DomainRestoreService($restoreRepo, $jobRepo, $groups, $legacy);
+        return new self($settings, $jobs, $export, $restore, $domainRestore,
+            new BackupApiController($settings, $jobs, $export, $restore, $domainRestore, new ApiResponse(Settings::SECRETS)));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo, $redis, $_SESSION, TZ and $ALIAS_DISABLE_EXTERNAL_DOMAINS */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), [
+            'timezone' => getenv('TZ') ?: 'UTC',
+            'alias_disable_external_domains' => ($GLOBALS['ALIAS_DISABLE_EXTERNAL_DOMAINS'] ?? false) === true,
+        ], ($GLOBALS['redis'] ?? null) instanceof Redis ? $GLOBALS['redis'] : null);
+    }
+}

data/web/inc/lib/mailcow/Backup/Controller/BackupApiController.php

@@ -0,0 +1,176 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Controller;
+
+use Mailcow\Backup\Model\Job;
+use Mailcow\Backup\Service\DomainRestoreService;
+use Mailcow\Backup\Service\ExportService;
+use Mailcow\Backup\Service\JobService;
+use Mailcow\Backup\Service\RestoreService;
+use Mailcow\Backup\Service\SettingsService;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php (routes backup-settings, backup-status, backup-snapshots, backup-snapshot-content,
+ * backup-jobs, backup-job, backup-restore) and the runner's endpoint data/conf/rspamd/dynmaps/backup.php (domains,
+ * export pages, restore actions). Messages are logged as ['backup', action, object, data].
+ */
+class BackupApiController
+{
+    public function __construct(private SettingsService $settings, private JobService $jobs, private ExportService $export,
+        private RestoreService $restore, private DomainRestoreService $domainRestore, private ApiResponse $response)
+    {
+    }
+
+    /** GET backup-settings: secrets as <key>_set; {} for non-admins */
+    public function getSettings(): array
+    {
+        return $this->settings->get()?->toArray() ?? [];
+    }
+
+    /** POST edit/backup-settings */
+    public function editSettings(array $attr): bool
+    {
+        return $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
+            $this->settings->save($attr);
+            return $this->response->success('backup_settings_saved', $log);
+        });
+    }
+
+    /** GET backup-status */
+    public function getStatus(): array
+    {
+        return $this->jobs->status();
+    }
+
+    /** GET backup-snapshots */
+    public function getSnapshots(): array
+    {
+        return $this->jobs->snapshots();
+    }
+
+    /** GET backup-snapshot-content/<snapshot>[/<domain>]: domains of a snapshot, or the mailboxes of one, with their state now */
+    public function getSnapshotContent(mixed $snapshot, mixed $domain = null): array
+    {
+        return ($domain === null || $domain === '') ? $this->jobs->content($snapshot) : $this->jobs->domainContent($snapshot, $domain);
+    }
+
+    /** GET backup-snapshot-search/<snapshot>/<term>: domains and mailboxes of a snapshot matching the term */
+    public function getSnapshotSearch(mixed $snapshot, mixed $term): array
+    {
+        return $this->jobs->search($snapshot, $term);
+    }
+
+    /** GET backup-jobs[/<count>[/<type>]]: newest first, without logs; count 0 = default */
+    public function getJobs(int $limit = 0, mixed $type = null): array
+    {
+        return array_map(fn(Job $job): array => $this->jobs->view($job), $this->jobs->list($limit, $type));
+    }
+
+    /** GET backup-job/<id>: with its log */
+    public function getJob(mixed $id): array
+    {
+        $id = $this->toId($id);
+        $job = ($id === null) ? null : $this->jobs->get($id);
+        return ($job === null) ? [] : $this->jobs->view($job, true);
+    }
+
+    /** POST add/backup-job: {type: backup|check|prune|init} */
+    public function addJob(array $attr): bool
+    {
+        return $this->handle('add', 'job', $attr, function (array $log) use ($attr): bool {
+            $job = $this->jobs->create($attr);
+            return $this->response->success(['backup_job_added', $job->type, $job->id], $log);
+        });
+    }
+
+    /** POST add/backup-restore: {snapshot, scope, mailboxes | domain, components, mail_target} */
+    public function addRestore(array $attr): bool
+    {
+        return $this->handle('add', 'restore', $attr, function (array $log) use ($attr): bool {
+            $job = $this->jobs->restore($attr);
+            return $this->response->success(['backup_restore_added', $job->id, count($job->mailboxes())], $log);
+        });
+    }
+
+    /** POST delete/backup-job: cancels queued and running jobs, one message per id */
+    public function deleteJobs(array $items): bool
+    {
+        return $this->handle('delete', 'job', ['id' => $items], function (array $log) use ($items): bool {
+            foreach ($items as $id) {
+                try {
+                    $job = $this->jobs->cancel($this->requireId($id));
+                    $this->response->success(['backup_job_cancelled', $job->id], $log);
+                } catch (UserError $e) {
+                    $this->response->error($e, $log);
+                }
+            }
+            return true;
+        });
+    }
+
+    /** backup.php ?domains=1 */
+    public function domains(): array
+    {
+        return ['domains' => $this->export->domains()];
+    }
+
+    /** backup.php ?export=<domain>[&after=<username>] @throws UserError */
+    public function export(string $domain, ?string $after = null): array
+    {
+        return $this->export->export($domain, $after);
+    }
+
+    /**
+     * backup.php POST of the runner, {"result": ...} of
+     *   {"action": "restore_domain", "job": id, "data": <domain_data>}: creates a deleted domain
+     *   {"action": "restore_aliases", "job": id, "aliases": [...]}: a part of the domain's aliases (scope domain)
+     *   {"action": "restore", "job": id, "username": u, "data": <mailbox section>}: one mailbox
+     *
+     * @throws UserError
+     */
+    public function post(array $body): array
+    {
+        return ['result' => match ($body['action'] ?? null) {
+            'restore' => $this->restore->merge($body['job'] ?? null, $body['username'] ?? null, $body['data'] ?? null),
+            'restore_aliases' => $this->restore->mergeAliases($body['job'] ?? null, $body['aliases'] ?? null),
+            'restore_domain' => $this->domainRestore->restore($body['job'] ?? null, $body['data'] ?? null),
+            default => throw new UserError('backup_action_invalid'),
+        }];
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     */
+    private function handle(string $action, string $object, mixed $data, callable $work): bool
+    {
+        $log = ['backup', $action, $object, $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+
+    /** Integer id from request input ("12", 12); null for anything else */
+    private function toId(mixed $id): ?int
+    {
+        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
+    }
+
+    /** @throws AccessDenied for ids that cannot exist */
+    private function requireId(mixed $id): int
+    {
+        return $this->toId($id) ?? throw new AccessDenied();
+    }
+}

data/web/inc/lib/mailcow/Backup/Legacy/LegacyDomain.php

@@ -0,0 +1,84 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Legacy;
+
+use Redis;
+use RuntimeException;
+
+/**
+ * What the legacy code keeps outside the database for domains and mailboxes: DKIM keys, rate limits and the domain
+ * map in Redis (functions.dkim/ratelimit/mailbox.inc.php), and the SOGo restart through dockerapi. Redis and docker()
+ * sit in protected methods so tests can replace them.
+ */
+class LegacyDomain
+{
+    public function __construct(private ?Redis $redis = null)
+    {
+    }
+
+    /** @return ?array{selector: string, pubkey: string, privkey: string} the DKIM key of a domain */
+    public function dkim(string $domain): ?array
+    {
+        $pubkey = $this->redis()->hGet('DKIM_PUB_KEYS', $domain);
+        if (!is_string($pubkey) || $pubkey === '') {
+            return null;
+        }
+        $selector = (string)$this->redis()->hGet('DKIM_SELECTORS', $domain);
+        return ['selector' => $selector, 'pubkey' => $pubkey, 'privkey' => (string)$this->redis()->hGet('DKIM_PRIV_KEYS', $selector . '.' . $domain)];
+    }
+
+    /** Stores a DKIM key like dkim('import') does */
+    public function addDkim(string $domain, string $selector, string $pubkey, string $privkey): void
+    {
+        $this->redis()->hSet('DKIM_PUB_KEYS', $domain, $pubkey);
+        $this->redis()->hSet('DKIM_SELECTORS', $domain, $selector);
+        $this->redis()->hSet('DKIM_PRIV_KEYS', $selector . '.' . $domain, $privkey);
+    }
+
+    /**
+     * @param string[] $objects domains or mailboxes
+     * @return array<string, string> object => rate limit ("10 / 1h") of those with one
+     */
+    public function rateLimits(array $objects): array
+    {
+        if (empty($objects)) {
+            return [];
+        }
+        $values = $this->redis()->hMGet('RL_VALUE', array_values($objects));
+        return array_filter(is_array($values) ? $values : [], fn(mixed $value): bool => is_string($value) && $value !== '');
+    }
+
+    public function setRateLimit(string $object, string $value): void
+    {
+        $this->redis()->hSet('RL_VALUE', $object, $value);
+    }
+
+    /** The domain is local for rspamd (legacy domain and alias domain add) */
+    public function addDomainMap(string $domain): void
+    {
+        $this->redis()->hSet('DOMAIN_MAP', $domain, 1);
+    }
+
+    /**
+     * SOGo reads its domains at start: restart it after a domain was added (like the "restart SOGo" option of the form)
+     *
+     * @return ?string null on success, else the error of dockerapi
+     */
+    public function restartSogo(): ?string
+    {
+        $result = json_decode((string)$this->dockerRestart('sogo-mailcow'), true);
+        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
+    }
+
+    protected function redis(): Redis
+    {
+        return $this->redis ?? throw new RuntimeException('Redis is not connected');
+    }
+
+    protected function dockerRestart(string $service): mixed
+    {
+        return docker('post', $service, 'restart');
+    }
+}

data/web/inc/lib/mailcow/Backup/Legacy/NoTemplates.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Legacy;
+
+use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
+
+/**
+ * The group module's bridge to the mailbox code for restores: a membership restored from a backup never applies the
+ * group's mailbox template (the mailbox gets its settings from the backup).
+ */
+class NoTemplates extends LegacyMailbox
+{
+    public function applyTemplate(string $username, string $name, string $template): ?string
+    {
+        return null;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Job.php

@@ -0,0 +1,129 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * A job of the backup runners (table `backup_job`): backup, check, prune and init run in the restic container; a
+ * restore runs there (step `files`, then the merge through backup.php) and, for mails, IMAP ACLs and Sieve
+ * scripts, in the dovecot container (step `mails`). Times are unix timestamps.
+ */
+class Job
+{
+    public const TYPES = ['backup', 'check', 'prune', 'init', 'restore'];
+    /** Types an admin starts with add/backup-job */
+    public const MANUAL_TYPES = ['backup', 'check', 'prune', 'init'];
+    public const ACTIVE_STATES = ['queued', 'running'];
+
+    public ?int $id = null;
+    public string $type = 'backup';
+    public string $state = 'queued';
+    public ?string $step = null;
+    /**
+     * restore: snapshot, snapshot_time, scope, domain, mailboxes, components, mail_target, requested_role, create_domain (0|1),
+     * create_mailboxes (missing when requested), acl and allowed_domains (domain admins)
+     */
+    public array $params = [];
+    public ?array $result = null;
+    /** Only loaded for a single job */
+    public ?string $log = null;
+    public ?string $error = null;
+    /** restore: the domain of its mailboxes, so domain admins see their restores */
+    public string $domain = '';
+    public string $requestedBy = '';
+    public ?int $created = null;
+    public ?int $started = null;
+    public ?int $finished = null;
+
+    public function __construct(string $type = 'backup')
+    {
+        $this->type = $type;
+    }
+
+    /** Row of `backup_job` with the times as unix timestamps */
+    public static function fromRow(array $row): self
+    {
+        $job = new self((string)$row['type']);
+        $job->id = (int)$row['id'];
+        $job->state = (string)$row['state'];
+        $job->step = isset($row['step']) ? (string)$row['step'] : null;
+        $params = is_string($row['params'] ?? null) ? json_decode($row['params'], true) : null;
+        $job->params = is_array($params) ? $params : [];
+        $result = is_string($row['result'] ?? null) ? json_decode($row['result'], true) : null;
+        $job->result = is_array($result) ? $result : null;
+        $job->log = isset($row['log']) ? (string)$row['log'] : null;
+        $job->error = isset($row['error']) ? (string)$row['error'] : null;
+        $job->domain = (string)($row['domain'] ?? '');
+        $job->requestedBy = (string)($row['requested_by'] ?? '');
+        foreach (['created', 'started', 'finished'] as $time) {
+            $job->$time = isset($row[$time]) ? (int)$row[$time] : null;
+        }
+        return $job;
+    }
+
+    /** Column values for a new row */
+    public function toRow(): array
+    {
+        return [
+            'type' => $this->type,
+            'state' => $this->state,
+            'step' => $this->step,
+            'params' => json_encode($this->params ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
+            'domain' => $this->domain,
+            'requested_by' => $this->requestedBy,
+        ];
+    }
+
+    /** @return string[] mailboxes of a restore */
+    public function mailboxes(): array
+    {
+        return array_values(array_filter((array)($this->params['mailboxes'] ?? []), 'is_string'));
+    }
+
+    /** @return string[] components of a restore */
+    public function components(): array
+    {
+        return array_values(array_filter((array)($this->params['components'] ?? []), 'is_string'));
+    }
+
+    /** @return string[] mailboxes of a restore that did not exist when it was requested */
+    public function createMailboxes(): array
+    {
+        return array_values(array_filter((array)($this->params['create_mailboxes'] ?? []), 'is_string'));
+    }
+
+    /** The restore creates its domain (requested by an admin while the domain did not exist) */
+    public function createsDomain(): bool
+    {
+        return (int)($this->params['create_domain'] ?? 0) === 1;
+    }
+
+    public function isActive(): bool
+    {
+        return in_array($this->state, self::ACTIVE_STATES, true);
+    }
+
+    /** Entry of the API; the log only with $withLog (get/backup-job/<id>) */
+    public function toArray(bool $withLog = false): array
+    {
+        $job = [
+            'id' => $this->id,
+            'type' => $this->type,
+            'state' => $this->state,
+            'step' => $this->step,
+            'params' => $this->params ?: new \stdClass(),
+            'result' => $this->result,
+            'error' => $this->error,
+            'domain' => $this->domain,
+            'requested_by' => $this->requestedBy,
+            'created' => $this->created,
+            'started' => $this->started,
+            'finished' => $this->finished,
+        ];
+        if ($withLog) {
+            $job['log'] = (string)$this->log;
+        }
+        return $job;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/RestoreRun.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+use Mailcow\Core\Context;
+
+/**
+ * The merge of one mailbox of a restore job: what the backup holds, who asked for the restore and what the merge
+ * did per component (added, skipped, errors), plus the caches to refresh afterwards.
+ */
+class RestoreRun
+{
+    /** @var array<string, array{added: int, skipped: int, errors: string[]}> */
+    public array $results = [];
+    /** Mailboxes whose SOGo identities (aliases, send-as addresses) changed */
+    public array $sogoUsers = [];
+    /** Mailboxes whose userdb or passdb values changed (Dovecot auth cache) */
+    public array $authUsers = [];
+
+    /**
+     * @param array $mailbox  the mailbox row as it is now
+     * @param array $params   params of the restore job (requested_role, acl of a domain admin, scope, ...)
+     * @param array $data     the mailbox's section of the domain export in the snapshot
+     * @param string[] $domains domains whose aliases and send-as addresses the restore may change, [] = all (admins)
+     * @param string $requestedBy who requested the restore
+     */
+    /** The restore created the mailbox (its own alias gets the forwarding of the backup) */
+    public bool $created = false;
+
+    public function __construct(
+        public readonly string $username,
+        public readonly string $domain,
+        public readonly array $mailbox,
+        public readonly array $params,
+        public readonly array $data,
+        public readonly array $domains = [],
+        public readonly string $requestedBy = ''
+    ) {
+    }
+
+    /**
+     * The caller the group module checks: an admin, or the domain admin who requested the restore with his ACL of then
+     * and his domains now
+     */
+    public function requester(): Context
+    {
+        if ($this->byAdmin()) {
+            return new Context('admin', 'backup', [], true);
+        }
+        return new Context('domainadmin', $this->requestedBy, array_map(fn(mixed $flag): string => (string)(int)$flag, (array)($this->params['acl'] ?? [])));
+    }
+
+    public function start(string $component): void
+    {
+        $this->results[$component] = ['added' => 0, 'skipped' => 0, 'errors' => []];
+    }
+
+    public function added(string $component, int $count = 1): void
+    {
+        $this->results[$component]['added'] += $count;
+    }
+
+    public function skipped(string $component, int $count = 1): void
+    {
+        $this->results[$component]['skipped'] += $count;
+    }
+
+    public function error(string $component, string $error): void
+    {
+        $this->results[$component]['errors'][] = $error;
+    }
+
+    /** The part of a component was rolled back: nothing of it counts */
+    public function failed(string $component, string $error): void
+    {
+        $this->start($component);
+        $this->error($component, $error);
+    }
+
+    /** The restore was requested by an admin (domain admins are limited by their ACL and the job's domain) */
+    public function byAdmin(): bool
+    {
+        return ($this->params['requested_role'] ?? '') === 'admin';
+    }
+
+    /** Admins, or a domain admin with this ACL when he requested the restore */
+    public function may(string $acl): bool
+    {
+        return $this->byAdmin() || (int)($this->params['acl'][$acl] ?? 0) === 1;
+    }
+
+    public function mayChangeDomain(string $domain): bool
+    {
+        return $this->domains === [] || in_array(strtolower($domain), $this->domains, true);
+    }
+
+    /** Rows of a list in the backup (sieve_filters, tfa, ...) */
+    public function rows(string $key): array
+    {
+        return array_values(array_filter((array)($this->data[$key] ?? []), 'is_array'));
+    }
+
+    public function sogo(string ...$usernames): void
+    {
+        array_push($this->sogoUsers, ...$usernames);
+        $this->sogoUsers = array_values(array_unique($this->sogoUsers));
+    }
+
+    public function auth(string ...$usernames): void
+    {
+        array_push($this->authUsers, ...$usernames);
+        $this->authUsers = array_values(array_unique($this->authUsers));
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Settings.php

@@ -0,0 +1,189 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * Backup settings (table `backup_setting`, name/value pairs) over their defaults. Secrets are stored sealed for the
+ * runner (`sealed:v1:<base64>`) and never leave toArray(): the API gets `<key>_set` instead. The state_* values, the
+ * SSH public key and the SFTP host key are written by the runner in the restic container and cannot be set through
+ * the API (PHP writes state_target_changed when the target changes).
+ */
+class Settings
+{
+    public const DEFAULTS = [
+        'enabled' => '0',
+        'target' => 's3',
+        's3_endpoint' => 'https://s3.amazonaws.com',
+        's3_bucket' => '',
+        's3_path' => '',
+        's3_region' => '',
+        's3_access_key' => '',
+        's3_secret_key' => '',
+        's3_path_style' => '0',
+        'sftp_host' => '',
+        'sftp_port' => '22',
+        'sftp_user' => '',
+        'sftp_path' => '',
+        'sftp_public_key' => '',
+        'sftp_host_key' => '',
+        'rest_url' => '',
+        'rest_user' => '',
+        'rest_password' => '',
+        'local_path' => 'mailcow',
+        'password' => '',
+        'password_confirmed' => '0',
+        // JSON list of the scheduled backups: {"days": [weekdays, 0 = Sunday], "time": "HH:MM"}
+        'schedule' => '[{"days":[0,1,2,3,4,5,6],"time":"01:00"}]',
+        'keep_hourly' => '0',
+        'keep_daily' => '7',
+        'keep_weekly' => '4',
+        'keep_monthly' => '12',
+        'keep_yearly' => '0',
+        'prune_weekday' => '0',
+        'max_unused' => '10%',
+        'check_subset' => '30',
+        'include_index' => '1',
+        'include_fts' => '1',
+        'limit_upload' => '0',
+        'notify_email' => '',
+        'state_initialized' => '0',
+        'state_runner_seen' => '',
+        'state_last_backup' => '',
+        'state_last_success' => '',
+        'state_repo_size' => '',
+        'state_repo_stats' => '',
+        'state_schedule_since' => '',
+        'state_schedule' => '',
+        // public key (base64, X25519) the secrets are sealed with; the private key stays in the restic container
+        'state_secrets_pubkey' => '',
+        'state_secrets_error' => '',
+        // JSON {old_target, old_addresses, changed, changed_by, time}: the runner mails the old addresses and clears it
+        'state_target_changed' => '',
+        // RESTIC_BACKUP_MODE of mailcow.conf as the runner sees it
+        'state_backup_mode' => '',
+    ];
+    public const SECRETS = ['password', 's3_secret_key', 'rest_password'];
+    /** Written by the runner only */
+    public const RUNNER = ['sftp_public_key', 'sftp_host_key', 'state_initialized', 'state_runner_seen', 'state_last_backup',
+        'state_last_success', 'state_repo_size', 'state_repo_stats', 'state_schedule_since', 'state_schedule', 'state_secrets_pubkey',
+        'state_secrets_error', 'state_target_changed', 'state_backup_mode'];
+    /** Prefix of a sealed secret (sodium_crypto_box_seal with state_secrets_pubkey) */
+    public const SEALED = 'sealed:v1:';
+    public const TARGETS = ['s3', 'sftp', 'rest', 'local'];
+    /** The connection of each target: a change (or another target or repository password) resets state_initialized */
+    public const CONNECTION = [
+        's3' => ['s3_endpoint', 's3_bucket', 's3_path', 's3_region', 's3_access_key', 's3_secret_key', 's3_path_style'],
+        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
+        'rest' => ['rest_url', 'rest_user', 'rest_password'],
+        'local' => ['local_path'],
+    ];
+    /** Values a target cannot work without (besides the repository password) */
+    public const REQUIRED = [
+        's3' => ['s3_endpoint', 's3_bucket', 's3_access_key', 's3_secret_key'],
+        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
+        'rest' => ['rest_url'],
+        'local' => ['local_path'],
+    ];
+    /** Returned as numbers by the API ('' = not known yet: null; prune_weekday '' = never stays '') */
+    private const NUMBERS = ['enabled', 's3_path_style', 'sftp_port', 'password_confirmed', 'keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly',
+        'keep_yearly', 'prune_weekday', 'check_subset', 'include_index', 'include_fts', 'limit_upload', 'state_initialized',
+        'state_runner_seen', 'state_last_backup', 'state_last_success', 'state_repo_size', 'state_schedule_since'];
+    private const JSON = ['schedule', 'state_repo_stats', 'state_target_changed'];
+
+    /** @param array<string, string> $values every key of DEFAULTS */
+    public function __construct(public array $values = self::DEFAULTS)
+    {
+    }
+
+    /** Stored values (name => value) over the defaults; unknown names are dropped */
+    public static function fromStored(array $stored): self
+    {
+        $values = self::DEFAULTS;
+        foreach ($stored as $name => $value) {
+            if (array_key_exists($name, $values)) {
+                $values[$name] = (string)$value;
+            }
+        }
+        return new self($values);
+    }
+
+    public function get(string $name): string
+    {
+        return $this->values[$name] ?? '';
+    }
+
+    /** The target has its required values and the repository password is set */
+    public function configured(): bool
+    {
+        $required = self::REQUIRED[$this->get('target')] ?? null;
+        if ($required === null || $this->get('password') === '') {
+            return false;
+        }
+        foreach ($required as $name) {
+            if ($this->get($name) === '') {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * The schedule as [weekday (0 = Sunday), hour, minute] slots, sorted and unique
+     *
+     * @return array<int, array{int, int, int}>
+     */
+    public function scheduleSlots(): array
+    {
+        $entries = json_decode($this->get('schedule'), true);
+        $slots = [];
+        foreach (is_array($entries) ? $entries : [] as $entry) {
+            if (!is_array($entry) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', (string)($entry['time'] ?? ''), $time) || !is_array($entry['days'] ?? null)) {
+                continue;
+            }
+            foreach ($entry['days'] as $day) {
+                if (is_int($day) && $day >= 0 && $day <= 6) {
+                    $slots[] = $day * 1440 + (int)$time[1] * 60 + (int)$time[2];
+                }
+            }
+        }
+        $slots = array_unique($slots);
+        sort($slots);
+        return array_map(fn(int $slot): array => [intdiv($slot, 1440), intdiv($slot % 1440, 60), $slot % 60], $slots);
+    }
+
+    /** The longest time between two scheduled backups of a week in seconds, null without a schedule */
+    public function scheduleGap(): ?int
+    {
+        $slots = array_map(fn(array $slot): int => $slot[0] * 86400 + $slot[1] * 3600 + $slot[2] * 60, $this->scheduleSlots());
+        if (empty($slots)) {
+            return null;
+        }
+        $gaps = [$slots[0] + 7 * 86400 - end($slots)];
+        for ($i = 1; $i < count($slots); $i++) {
+            $gaps[] = $slots[$i] - $slots[$i - 1];
+        }
+        return max($gaps);
+    }
+
+    /** Entry of get/backup-settings: every key, secrets as <key>_set, plus `configured` */
+    public function toArray(): array
+    {
+        $settings = [];
+        foreach ($this->values as $name => $value) {
+            if (in_array($name, self::SECRETS, true)) {
+                $settings[$name . '_set'] = (int)($value !== '');
+            } elseif (in_array($name, self::JSON, true)) {
+                $decoded = json_decode($value, true);
+                $settings[$name] = is_array($decoded) ? $decoded : null;
+            } elseif (in_array($name, self::NUMBERS, true)) {
+                $settings[$name] = ($value === '') ? (($name === 'prune_weekday') ? '' : null) : (int)$value;
+            } else {
+                $settings[$name] = $value;
+            }
+        }
+        $settings['configured'] = (int)$this->configured();
+        return $settings;
+    }
+}

data/web/inc/lib/mailcow/Backup/Model/Snapshot.php

@@ -0,0 +1,142 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Model;
+
+/**
+ * A snapshot of the repository (table `backup_snapshot`, the runner's cache of `restic snapshots --json`).
+ */
+class Snapshot
+{
+    public string $id = '';
+    public string $shortId = '';
+    /** unix timestamp */
+    public int $time = 0;
+    public string $hostname = '';
+    /** @var string[] */
+    public array $tags = [];
+    /** restic's summary of the backup (data_added, files_new, ...) */
+    public ?array $summary = null;
+    /** @var ?string[] domains of its manifest, null for snapshots not made by mailcow */
+    public ?array $domains = null;
+    /** @var array<string, ?int> domain => its creation time (unix) when the snapshot was made, from the snapshot row */
+    public array $created = [];
+    /**
+     * @var ?array<string, array{created: ?int, mailboxes: array<string, string>}> per domain its creation time (unix, null
+     *      when not recorded) and username => name of its mailboxes; null = not known (snapshots this installation did
+     *      not make itself, e.g. of another server in the same repository)
+     */
+    public ?array $content = null;
+
+    /** Row of `backup_snapshot` with the time as unix timestamp */
+    public static function fromRow(array $row): self
+    {
+        $snapshot = new self();
+        $snapshot->id = (string)$row['id'];
+        $snapshot->shortId = (string)$row['short_id'];
+        $snapshot->time = (int)$row['time'];
+        $snapshot->hostname = (string)($row['hostname'] ?? '');
+        $tags = (string)($row['tags'] ?? '');
+        $decoded = json_decode($tags, true);
+        $tags = is_array($decoded) ? $decoded : explode(',', $tags);
+        $snapshot->tags = array_values(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen'));
+        $summary = is_string($row['summary'] ?? null) ? json_decode($row['summary'], true) : null;
+        $snapshot->summary = is_array($summary) ? $summary : null;
+        // {domain: creation time}: lists of snapshots need no content
+        $domains = is_string($row['domains'] ?? null) ? json_decode($row['domains'], true) : null;
+        if (is_array($domains)) {
+            $snapshot->domains = [];
+            foreach ($domains as $domain => $created) {
+                $snapshot->domains[] = (string)$domain;
+                $snapshot->created[strtolower((string)$domain)] = is_numeric($created) ? (int)$created : null;
+            }
+        }
+        if (is_string($row['content'] ?? null) && $row['content'] !== '') {
+            $snapshot->content = self::decodeContent($row['content']);
+        }
+        return $snapshot;
+    }
+
+    /**
+     * `content` (gzip JSON) written by the runner: {"domains": {"<domain>": {"created": <unix>, "mailboxes": {"<username>": "<name>"}}}}.
+     * Keys lower case; null when broken.
+     */
+    public static function decodeContent(string $gzip): ?array
+    {
+        $json = @gzdecode($gzip);
+        $data = is_string($json) ? json_decode($json, true) : null;
+        if (!is_array($data) || !is_array($data['domains'] ?? null)) {
+            return null;
+        }
+        $content = [];
+        foreach ($data['domains'] as $domain => $entry) {
+            $mailboxes = is_array($entry) ? ($entry['mailboxes'] ?? []) : [];
+            $created = is_numeric($entry['created'] ?? null) ? (int)$entry['created'] : null;
+            $names = [];
+            foreach (is_array($mailboxes) ? $mailboxes : [] as $username => $name) {
+                $names[strtolower((string)$username)] = is_scalar($name) ? (string)$name : '';
+            }
+            $content[strtolower((string)$domain)] = ['created' => $created, 'mailboxes' => $names];
+        }
+        return $content;
+    }
+
+    /** Domains of the snapshot: those of its content and of its manifest (an export may have failed) */
+    public function domainList(): array
+    {
+        return array_values(array_unique(array_merge(array_map('strval', array_keys($this->content ?? [])), array_map('strtolower', $this->domains ?? []))));
+    }
+
+    /** @return ?array<string, string> username => name of the domain's mailboxes, null when the content does not list the domain */
+    public function mailboxesOf(string $domain): ?array
+    {
+        return $this->content[strtolower($domain)]['mailboxes'] ?? null;
+    }
+
+    /** Creation time of the domain when the snapshot was made, null when not recorded */
+    public function createdOf(string $domain): ?int
+    {
+        return $this->content[strtolower($domain)]['created'] ?? $this->created[strtolower($domain)] ?? null;
+    }
+
+    public function hasDomain(string $domain): bool
+    {
+        return in_array(strtolower($domain), $this->domainList(), true);
+    }
+
+    /**
+     * RESTIC_BACKUP_MODE of the backup (tag mode:<mode>, none for all): all, data (without the mailcow directory) or
+     * config (without mails)
+     */
+    public function mode(): string
+    {
+        foreach ($this->tags as $tag) {
+            if (str_starts_with($tag, 'mode:')) {
+                return substr($tag, 5);
+            }
+        }
+        return 'all';
+    }
+
+    /** Snapshots not made by mailcow know neither domains nor mailboxes */
+    public function knowsDomains(): bool
+    {
+        return $this->content !== null || $this->domains !== null;
+    }
+
+    /** Entry of the API for admins */
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->id,
+            'short_id' => $this->shortId,
+            'time' => $this->time,
+            'hostname' => $this->hostname,
+            'tags' => $this->tags,
+            'summary' => $this->summary,
+            'domains' => $this->domains,
+            'mode' => $this->mode(),
+        ];
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/ExportRepository.php

@@ -0,0 +1,169 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * Reads what the export of a domain holds: the domain with its settings, its aliases and a page of its mailboxes with
+ * every row that belongs to a mailbox. Rows are read with all their columns (SELECT *).
+ */
+class ExportRepository extends Repository
+{
+    /** @return string[] all domains */
+    public function domains(): array
+    {
+        return $this->query("SELECT `domain` FROM `domain` ORDER BY `domain`")->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    public function dbVersion(): string
+    {
+        return (string)$this->query("SELECT `version` FROM `versions` WHERE `application` = 'db_schema'")->fetchColumn();
+    }
+
+    /** @return array[] rows of a table matching column => value (table and column names from code) */
+    public function rows(string $table, string $column, string $value): array
+    {
+        self::identifier($table);
+        self::identifier($column);
+        return $this->query("SELECT * FROM `$table` WHERE `$column` = :value", ['value' => $value])->fetchAll();
+    }
+
+    /** @return string[] usernames of the domain's mailboxes */
+    public function usernames(string $domain): array
+    {
+        return $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain", ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** @return array[] the domain's mailbox rows after $after in username order (keyset paging), at most $limit */
+    public function mailboxesAfter(string $domain, string $after, int $limit): array
+    {
+        return $this->query("SELECT * FROM `mailbox` WHERE `domain` = :domain AND `username` > :after ORDER BY `username` LIMIT " . max(1, $limit),
+            ['domain' => $domain, 'after' => $after])->fetchAll();
+    }
+
+    /** @return array[] alias rows of the domain and its alias domains */
+    public function aliases(string $domain): array
+    {
+        return $this->query("SELECT * FROM `alias` WHERE `domain` = :domain
+            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target) ORDER BY `id`",
+            ['domain' => $domain, 'target' => $domain])->fetchAll();
+    }
+
+    /** @return array[] alias rows of any domain whose goto may name a mailbox of the domain (the caller checks) */
+    public function aliasesPointingTo(string $domain): array
+    {
+        return $this->query("SELECT * FROM `alias` WHERE `goto` LIKE :pattern ORDER BY `id`",
+            ['pattern' => '%@' . addcslashes($domain, '%_\\') . '%'])->fetchAll();
+    }
+
+    /**
+     * @param string $table  table with rows of mailboxes
+     * @param string $column its column holding the username
+     * @param string[] $usernames
+     * @return array[] rows of these mailboxes
+     */
+    public function rowsOfMailboxes(string $table, string $column, array $usernames): array
+    {
+        self::identifier($table);
+        self::identifier($column);
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT * FROM `$table` WHERE `$column` IN ($in)", $params)->fetchAll();
+    }
+
+    /** @return array[] username, id, name, domain, source of the group memberships of these mailboxes */
+    public function groupMemberships(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT `gm`.`username`, `g`.`id`, `g`.`name`, `g`.`domain`, `gm`.`source` FROM `mailbox_group_member` `gm`
+            JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id` WHERE `gm`.`username` IN ($in) ORDER BY `g`.`id`", $params)->fetchAll();
+    }
+
+    /**
+     * @param string $table  mail_policy or retention_rule
+     * @param string $member its member table
+     * @param string $key    the member table's column with the id
+     * @return array[] username, id, name, domain of the direct assignments of these mailboxes
+     */
+    public function assignments(string $table, string $member, string $key, array $usernames): array
+    {
+        self::identifier($table);
+        self::identifier($member);
+        self::identifier($key);
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        return $this->query("SELECT `a`.`username`, `p`.`id`, `p`.`name`, `p`.`domain` FROM `$member` `a`
+            JOIN `$table` `p` ON `p`.`id` = `a`.`$key` WHERE `a`.`username` IN ($in) ORDER BY `p`.`id`", $params)->fetchAll();
+    }
+
+    /** @return array[] sync jobs of these mailboxes (without their log) with host1, port1 and name of the source */
+    public function syncJobs(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        [$in, $params] = self::in($usernames);
+        $rows = $this->query("SELECT `i`.*, `s`.`host1` AS `source_host1`, `s`.`port1` AS `source_port1`, `s`.`name` AS `source_name`
+            FROM `imapsync` `i` LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` IN ($in) ORDER BY `i`.`id`", $params)->fetchAll();
+        foreach ($rows as &$row) {
+            unset($row['returned_text']);
+        }
+        return $rows;
+    }
+
+    /** @return array[] BCC maps of the whole domain (local_dest "@domain", older rows "domain") */
+    public function domainBccMaps(string $domain): array
+    {
+        return $this->query("SELECT * FROM `bcc_maps` WHERE `local_dest` IN (:domain, :at)", ['domain' => $domain, 'at' => '@' . $domain])->fetchAll();
+    }
+
+    /**
+     * @param string $table  mail_policy or retention_rule
+     * @param string $member its member table
+     * @param string $key    the member table's column with the id
+     * @return array[] rows of the domain, each with `group_members`: the names of the groups it is assigned to
+     */
+    public function domainRules(string $table, string $member, string $key, string $domain): array
+    {
+        self::identifier($table);
+        self::identifier($member);
+        self::identifier($key);
+        $rows = $this->query("SELECT * FROM `$table` WHERE `domain` = :domain ORDER BY `id`", ['domain' => $domain])->fetchAll();
+        foreach ($rows as &$row) {
+            $row['group_members'] = $this->query("SELECT `g`.`name` FROM `$member` `a` JOIN `mailbox_group` `g` ON `g`.`id` = `a`.`group_id`
+                WHERE `a`.`$key` = :id ORDER BY `g`.`name`", ['id' => $row['id']])->fetchAll(PDO::FETCH_COLUMN);
+        }
+        return $rows;
+    }
+
+    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
+    private static function in(array $values): array
+    {
+        $names = [];
+        $params = [];
+        foreach (array_values($values) as $i => $value) {
+            $names[] = ':in' . $i;
+            $params['in' . $i] = $value;
+        }
+        return [implode(', ', $names), $params];
+    }
+
+    /** Table and column names come from code only; this keeps it that way */
+    private static function identifier(string $name): void
+    {
+        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
+            throw new \InvalidArgumentException('invalid identifier');
+        }
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/JobRepository.php

@@ -0,0 +1,176 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Backup\Model\Job;
+use Mailcow\Core\Context;
+use Mailcow\Core\Repository;
+use Mailcow\Core\UserError;
+use PDO;
+
+/**
+ * Table `backup_job` (the runners claim, log and finish the jobs) and the lookups a new restore job needs.
+ */
+class JobRepository extends Repository
+{
+    private const COLUMNS = "`id`, `type`, `state`, `step`, `params`, `result`, `error`, `domain`, `requested_by`,
+        UNIX_TIMESTAMP(`created`) AS `created`, UNIX_TIMESTAMP(`started`) AS `started`, UNIX_TIMESTAMP(`finished`) AS `finished`";
+    /** Serializes the check for active jobs and the insert of a new one */
+    private const LOCK = 'mailcow_backup_job';
+
+    public function find(int $id, bool $withLog = false): ?Job
+    {
+        $row = $this->query("SELECT " . self::COLUMNS . ($withLog ? ", `log`" : "") . " FROM `backup_job` WHERE `id` = :id", ['id' => $id])->fetch();
+        return is_array($row) ? Job::fromRow($row) : null;
+    }
+
+    /**
+     * @param ?string $type only jobs of this type
+     * @return Job[] newest first: all jobs for admins, the restores of their domains (since the domain was created) for
+     *         domain admins
+     */
+    public function visible(Context $context, int $limit, ?string $type = null): array
+    {
+        if ($context->hasAdminRights()) {
+            [$where, $params] = ['1=1', []];
+        } elseif ($context->isDomainAdmin()) {
+            // restores of their domains, requested since the domain was created (not those of a former domain of that name)
+            [$where, $params] = ["`type` = 'restore' AND `domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :da)
+              AND `created` >= (SELECT `d`.`created` FROM `domain` `d` WHERE `d`.`domain` = `backup_job`.`domain`)", ['da' => $context->username]];
+        } else {
+            return [];
+        }
+        if ($type !== null) {
+            $where .= " AND `type` = :type";
+            $params['type'] = $type;
+        }
+        return array_map(fn(array $row): Job => Job::fromRow($row),
+            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE $where ORDER BY `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll());
+    }
+
+    /** @return Job[] queued and running jobs */
+    public function active(): array
+    {
+        return array_map(fn(array $row): Job => Job::fromRow($row),
+            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `state` IN ('queued', 'running') ORDER BY `id`")->fetchAll());
+    }
+
+    /** @return array<string, Job> type => its newest job */
+    public function latestOfEachType(): array
+    {
+        $jobs = [];
+        foreach ($this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `id` IN (SELECT MAX(`id`) FROM `backup_job` GROUP BY `type`)")->fetchAll() as $row) {
+            $jobs[(string)$row['type']] = Job::fromRow($row);
+        }
+        return $jobs;
+    }
+
+    /** A backup job was created by the schedule since this unix time */
+    public function scheduledSince(int $since): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `backup_job` WHERE `type` = 'backup' AND `requested_by` = 'schedule' AND `created` >= FROM_UNIXTIME(:since) LIMIT 1",
+            ['since' => $since])->fetchColumn();
+    }
+
+    /**
+     * Inserts the job after $check(active jobs) passed; $check throws a UserError on a conflict. A named lock keeps two
+     * requests from queueing the same work at once.
+     *
+     * @param callable(Job[]): void $check
+     * @throws UserError
+     */
+    public function insertExclusive(Job $job, callable $check): int
+    {
+        if ((int)$this->query("SELECT GET_LOCK('" . self::LOCK . "', 10)")->fetchColumn() !== 1) {
+            throw new UserError('backup_job_active', [$job->type]);
+        }
+        try {
+            $check($this->active());
+            $job->id = $this->insertRow('backup_job', $job->toRow());
+            return $job->id;
+        } finally {
+            $this->query("SELECT RELEASE_LOCK('" . self::LOCK . "')");
+        }
+    }
+
+    /**
+     * Cancels a queued or running job; the restic runner stops at its next check. A restore in the dovecot step
+     * (mails, mails_running) cannot be cancelled. False when the job finished or reached that step meanwhile.
+     */
+    public function cancel(int $id): bool
+    {
+        return $this->query("UPDATE `backup_job` SET `state` = 'cancelled', `finished` = NOW() WHERE `id` = :id
+            AND `state` IN ('queued', 'running') AND NOT (`type` = 'restore' AND COALESCE(`step`, '') IN ('mails', 'mails_running'))",
+            ['id' => $id])->rowCount() > 0;
+    }
+
+    public function isDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+    }
+
+    /** Domain of a mailbox, null if there is none */
+    public function mailboxDomain(string $username): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username", ['username' => $username])->fetchColumn();
+        return is_string($domain) ? strtolower($domain) : null;
+    }
+
+    /** Unix time the domain was created, null if it does not exist */
+    public function domainCreated(string $domain): ?int
+    {
+        $created = $this->query("SELECT UNIX_TIMESTAMP(`created`) FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+        return ($created === false || $created === null) ? null : (int)$created;
+    }
+
+    /**
+     * @param string[] $domains
+     * @return array{0: array<string, true>, 1: array<string, array<string, true>>} the domains that exist, and per domain
+     *         the usernames of its mailboxes; one query each per 500 domains
+     */
+    public function domainsAndUsernames(array $domains): array
+    {
+        $existing = [];
+        $usernames = [];
+        foreach (array_chunk(array_values(array_unique($domains)), 500) as $chunk) {
+            $params = [];
+            foreach ($chunk as $i => $domain) {
+                $params['d' . $i] = $domain;
+            }
+            $in = implode(', ', array_map(fn(string $name): string => ':' . $name, array_keys($params)));
+            foreach ($this->query("SELECT `domain` FROM `domain` WHERE `domain` IN ($in)", $params)->fetchAll(PDO::FETCH_COLUMN) as $domain) {
+                $existing[strtolower((string)$domain)] = true;
+            }
+            foreach ($this->query("SELECT `domain`, `username` FROM `mailbox` WHERE `domain` IN ($in)", $params)->fetchAll() as $row) {
+                $usernames[strtolower((string)$row['domain'])][strtolower((string)$row['username'])] = true;
+            }
+        }
+        return [$existing, $usernames];
+    }
+
+    /** @return array<string, string> username => name of the mailboxes of a domain */
+    public function mailboxNames(string $domain): array
+    {
+        $names = [];
+        foreach ($this->query("SELECT `username`, `name` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`", ['domain' => $domain])->fetchAll() as $row) {
+            $names[strtolower((string)$row['username'])] = (string)$row['name'];
+        }
+        return $names;
+    }
+
+    /** @return string[] all mailboxes of a domain */
+    public function mailboxesOf(string $domain): array
+    {
+        return array_map('strtolower', $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`",
+            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /** @return string[] alias domains of a domain */
+    public function aliasDomainsOf(string $domain): array
+    {
+        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
+            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/RestoreRepository.php

@@ -0,0 +1,237 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+use Throwable;
+
+/**
+ * Reads and writes of the restore merge. Rows of a backup may come from an older schema, so inserts and updates only
+ * use the columns the table has now. Table and column names come from code, never from the backup.
+ */
+class RestoreRepository extends Repository
+{
+    /** @var array<string, string[]> table => columns */
+    private array $columns = [];
+
+    /** Runs $work in one transaction */
+    public function inTransaction(callable $work): mixed
+    {
+        return $this->transaction($work);
+    }
+
+    /** Runs $work in a savepoint of the running transaction: an error undoes only its writes and is thrown on */
+    public function savepoint(string $name, callable $work): mixed
+    {
+        self::identifier($name);
+        $this->pdo->exec("SAVEPOINT `$name`");
+        try {
+            $result = $work();
+            $this->pdo->exec("RELEASE SAVEPOINT `$name`");
+            return $result;
+        } catch (Throwable $e) {
+            $this->pdo->exec("ROLLBACK TO SAVEPOINT `$name`");
+            throw $e;
+        }
+    }
+
+    /**
+     * Runs $work while holding a named lock, so a retried request of the runner waits for the first one instead of
+     * merging the same mailbox twice; null when the lock was not free within $timeout seconds
+     */
+    public function locked(string $key, callable $work, int $timeout = 120): mixed
+    {
+        $name = 'mailcow_backup_' . md5($key);
+        if ((int)$this->query("SELECT GET_LOCK(:name, " . max(0, $timeout) . ")", ['name' => $name])->fetchColumn() !== 1) {
+            return null;
+        }
+        try {
+            return $work();
+        } finally {
+            $this->query("SELECT RELEASE_LOCK(:name)", ['name' => $name]);
+        }
+    }
+
+    /** Quota rows of a new mailbox (like the legacy mailbox add) */
+    public function resetQuota(string $username): void
+    {
+        foreach (['quota2', 'quota2replica'] as $table) {
+            $this->query("INSERT INTO `$table` (`username`, `bytes`, `messages`) VALUES (:username, 0, 0)
+              ON DUPLICATE KEY UPDATE `bytes` = 0, `messages` = 0", ['username' => $username]);
+        }
+    }
+
+    /** Regular mailboxes of a domain (the domain's mailbox limit counts these) */
+    public function mailboxCount(string $domain): int
+    {
+        return (int)$this->query("SELECT COUNT(*) FROM `mailbox` WHERE `domain` = :domain AND `kind` = ''", ['domain' => $domain])->fetchColumn();
+    }
+
+    /** External send-as rights for a domain that is local now (legacy domain and alias domain add) */
+    public function dropExternalSenderAcl(string $domain): void
+    {
+        $this->query("DELETE FROM `sender_acl` WHERE `external` = 1 AND `send_as` LIKE :domain", ['domain' => '%@' . addcslashes($domain, '%_\\')]);
+    }
+
+    /** @return string[] the columns of a table in the current schema */
+    public function columns(string $table): array
+    {
+        self::identifier($table);
+        return $this->columns[$table] ??= $this->query("SELECT `COLUMN_NAME` FROM `information_schema`.`COLUMNS`
+            WHERE `TABLE_SCHEMA` = DATABASE() AND `TABLE_NAME` = :table ORDER BY `ORDINAL_POSITION`", ['table' => $table])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /** @return array[] rows of a table matching every condition (column => value) */
+    public function rows(string $table, array $where): array
+    {
+        [$condition, $params] = $this->where($table, $where);
+        return $this->query("SELECT * FROM `$table` WHERE $condition", $params)->fetchAll();
+    }
+
+    /** Inserts a row with the values of the columns the table has now; returns the new id (0 without one) */
+    public function insert(string $table, array $values): int
+    {
+        return $this->insertRow($table, $this->existing($table, $values));
+    }
+
+    /** Updates the rows matching $where with the values of the columns the table has now */
+    public function update(string $table, array $where, array $values): int
+    {
+        $values = $this->existing($table, $values);
+        if (empty($values)) {
+            return 0;
+        }
+        [$condition, $params] = $this->where($table, $where);
+        $set = [];
+        foreach (array_keys($values) as $i => $column) {
+            $set[] = "`$column` = :v$i";
+            $params['v' . $i] = $values[$column];
+        }
+        return $this->query("UPDATE `$table` SET " . implode(', ', $set) . " WHERE $condition", $params)->rowCount();
+    }
+
+    /** A primary or alias domain */
+    public function domainExists(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias",
+            ['domain' => $domain, 'alias' => $domain])->fetchColumn();
+    }
+
+    /** The primary domain of a domain or alias domain, null if mailcow has neither */
+    public function primaryDomain(string $domain): ?string
+    {
+        $primary = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain
+            UNION SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :alias", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
+        return is_string($primary) ? strtolower($primary) : null;
+    }
+
+    /** An active primary or alias domain (like the legacy is_local_mailcow_domain()) */
+    public function isLocalDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain AND `active` = 1
+            UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias AND `active` = 1", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
+    }
+
+    /** @return string[] alias domains of a domain */
+    public function aliasDomainsOf(string $domain): array
+    {
+        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
+            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /**
+     * What an address is now: 'mailbox' (also local@alias-domain of a mailbox), 'resource' (a mailbox of kind
+     * location, thing or group), 'alias', 'spamalias' or null
+     */
+    public function addressKind(string $address): ?string
+    {
+        $kind = $this->query("SELECT `kind` FROM `mailbox` WHERE `username` = :address", ['address' => $address])->fetchColumn();
+        if (is_string($kind)) {
+            return preg_match('/location|thing|group/', $kind) ? 'resource' : 'mailbox';
+        }
+        [$local, $domain] = array_pad(explode('@', $address, 2), 2, '');
+        if ((bool)$this->query("SELECT 1 FROM `mailbox` `m` JOIN `alias_domain` `ad` ON `ad`.`alias_domain` = :domain
+            WHERE `m`.`username` = CONCAT(:local, '@', `ad`.`target_domain`)", ['domain' => $domain, 'local' => $local])->fetchColumn()) {
+            return 'mailbox';
+        }
+        if ((bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
+            return 'alias';
+        }
+        if ((bool)$this->query("SELECT 1 FROM `spamalias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
+            return 'spamalias';
+        }
+        return null;
+    }
+
+    /** Aliases the domain may still get (domain limit minus its aliases and those of its alias domains, like the legacy code) */
+    public function aliasesLeft(string $domain): int
+    {
+        $limit = $this->query("SELECT `aliases` FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+        if ($limit === false) {
+            return 0;
+        }
+        $used = (int)$this->query("SELECT COUNT(*) FROM `alias` WHERE (`domain` = :domain
+            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target))
+            AND `address` NOT IN (SELECT `username` FROM `mailbox`)", ['domain' => $domain, 'target' => $domain])->fetchColumn();
+        return (int)$limit - $used;
+    }
+
+    /** Quota (bytes) the other mailboxes of the domain have */
+    public function quotaInUse(string $domain, string $exceptUsername): int
+    {
+        return (int)$this->query("SELECT COALESCE(SUM(`quota`), 0) FROM `mailbox` WHERE `domain` = :domain AND `kind` = '' AND `username` != :username",
+            ['domain' => $domain, 'username' => $exceptUsername])->fetchColumn();
+    }
+
+    /** @return array[] user1, host1 and port1 of the mailbox's sync jobs */
+    public function syncJobsOf(string $username): array
+    {
+        return $this->query("SELECT `i`.`user1`, `s`.`host1`, `s`.`port1` FROM `imapsync` `i`
+            LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` = :username", ['username' => $username])->fetchAll();
+    }
+
+    /** @return int[] ids of the sync sources with this host and port the mailbox may use, oldest first */
+    public function syncSourcesFor(string $host, int $port, string $username, string $domain): array
+    {
+        return array_map('intval', $this->query("SELECT `s`.`id` FROM `imapsync_source` `s` WHERE `s`.`host1` = :host AND `s`.`port1` = :port
+            AND (`s`.`scope` = 'all'
+              OR (`s`.`scope` = 'domain' AND EXISTS (SELECT 1 FROM `imapsync_source_domain` `d` WHERE `d`.`source_id` = `s`.`id` AND `d`.`domain` = :domain))
+              OR (`s`.`scope` = 'user' AND EXISTS (SELECT 1 FROM `imapsync_source_user` `u` WHERE `u`.`source_id` = `s`.`id` AND `u`.`username` = :username)))
+            ORDER BY `s`.`id`", ['host' => $host, 'port' => $port, 'domain' => $domain, 'username' => $username])->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    /** @return array{0: string, 1: array} condition and parameters of column => value pairs */
+    private function where(string $table, array $where): array
+    {
+        self::identifier($table);
+        $parts = [];
+        $params = [];
+        foreach (array_keys($where) as $i => $column) {
+            self::identifier((string)$column);
+            if ($where[$column] === null) {
+                $parts[] = "`$column` IS NULL";
+                continue;
+            }
+            $parts[] = "`$column` = :w$i";
+            $params['w' . $i] = $where[$column];
+        }
+        return [empty($parts) ? '1=1' : implode(' AND ', $parts), $params];
+    }
+
+    /** The values of the columns the table has now */
+    private function existing(string $table, array $values): array
+    {
+        return array_intersect_key($values, array_flip($this->columns($table)));
+    }
+
+    /** Table, column and savepoint names come from code only; this keeps it that way */
+    private static function identifier(string $name): void
+    {
+        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
+            throw new \InvalidArgumentException('invalid identifier');
+        }
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/SettingsRepository.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Core\Repository;
+use PDO;
+
+/**
+ * Table `backup_setting` (name/value pairs; the runner writes its state_* values into it as well).
+ */
+class SettingsRepository extends Repository
+{
+    /** @return array<string, string> name => value of the stored settings */
+    public function load(): array
+    {
+        return $this->query("SELECT `name`, `value` FROM `backup_setting`")->fetchAll(PDO::FETCH_KEY_PAIR);
+    }
+
+    /** Stores the given values (name => value) */
+    public function save(array $values): void
+    {
+        $this->transaction(function () use ($values): void {
+            foreach ($values as $name => $value) {
+                $this->query("INSERT INTO `backup_setting` (`name`, `value`) VALUES (:name, :value)
+                  ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)", ['name' => (string)$name, 'value' => (string)$value]);
+            }
+        });
+    }
+}

data/web/inc/lib/mailcow/Backup/Repository/SnapshotRepository.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Repository;
+
+use Mailcow\Backup\Model\Snapshot;
+use Mailcow\Core\Repository;
+
+/**
+ * Table `backup_snapshot`: the runner's cache of the repository's snapshots (read only here).
+ */
+class SnapshotRepository extends Repository
+{
+    private const COLUMNS = "`id`, `short_id`, UNIX_TIMESTAMP(`time`) AS `time`, `hostname`, `tags`, `summary`, `domains`";
+
+    /** By id or short id, with its content (the mailboxes in it) */
+    public function find(string $id): ?Snapshot
+    {
+        $row = $this->query("SELECT " . self::COLUMNS . ", `content` FROM `backup_snapshot` WHERE `id` = :id OR `short_id` = :short_id ORDER BY `id` = :exact DESC LIMIT 1",
+            ['id' => $id, 'short_id' => $id, 'exact' => $id])->fetch();
+        return is_array($row) ? Snapshot::fromRow($row) : null;
+    }
+
+    /** @return Snapshot[] newest first; the content (the mailboxes in it) only with $withContent */
+    public function all(bool $withContent = false): array
+    {
+        return array_map(fn(array $row): Snapshot => Snapshot::fromRow($row),
+            $this->query("SELECT " . self::COLUMNS . ($withContent ? ", `content`" : "") . " FROM `backup_snapshot` ORDER BY `time` DESC, `id`")->fetchAll());
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/DomainRestoreService.php

@@ -0,0 +1,320 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use Mailcow\Backup\Legacy\LegacyDomain;
+use Mailcow\Backup\Model\Job;
+use Mailcow\Backup\Repository\JobRepository;
+use Mailcow\Backup\Repository\RestoreRepository;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Throwable;
+
+/**
+ * Action `restore_domain`: recreates a domain deleted since the snapshot from its export (`domain_data`), before its
+ * mailboxes are restored. Only for running restores an admin requested while the domain did not exist
+ * (params.create_domain).
+ *
+ * Creates the domain like the legacy domain add (row, tags, domain map, external send-as rights for it dropped, SOGo
+ * restarted so it knows the domain), unless it exists by now. Then adds what is missing: alias domains, DKIM keys, rate
+ * limit, spam settings, BCC maps, footer, MTA-STS, domain admin assignments of accounts that still exist, groups (no
+ * members, send-as rights or shares; members follow with the mailboxes), mail policies and retention rules with their
+ * group assignments (matched by name).
+ */
+class DomainRestoreService
+{
+    private const NEW_ROW = ['id', 'created', 'modified'];
+    private const POLICY_LISTS = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
+    private const RATE_LIMIT = '/^\d+ \/ 1[smhd]$/';
+
+    public function __construct(private RestoreRepository $repo, private JobRepository $jobs, private GroupMembers $groups,
+        private LegacyDomain $legacy)
+    {
+    }
+
+    /**
+     * @return array{domain: array{added: int, skipped: int, errors: string[], created: int}}
+     * @throws UserError
+     */
+    public function restore(mixed $jobId, mixed $data): array
+    {
+        $id = Input::intInRange($jobId, 1, PHP_INT_MAX);
+        $job = ($id === null) ? null : $this->jobs->find($id);
+        if ($job === null || $job->type !== 'restore' || $job->state !== 'running') {
+            throw new UserError('backup_job_not_running');
+        }
+        if (!$job->createsDomain() || ($job->params['requested_role'] ?? '') !== 'admin') {
+            throw new AccessDenied();
+        }
+        $domain = strtolower((string)$job->domain);
+        if (!is_array($data) || !is_array($data['domain'] ?? null) || strtolower((string)($data['domain']['domain'] ?? '')) !== $domain) {
+            throw new UserError('backup_export_invalid');
+        }
+        $result = $this->repo->locked($job->id . ':domain', fn(): array => $this->restoreDomain($job, $domain, $data));
+        return is_array($result) ? ['domain' => $result] : throw new UserError('backup_job_busy');
+    }
+
+    private function restoreDomain(Job $job, string $domain, array $data): array
+    {
+        $result = ['added' => 0, 'skipped' => 0, 'errors' => [], 'created' => 0];
+        if (!empty($this->repo->rows('alias_domain', ['alias_domain' => $domain]))) {
+            $result['errors'][] = "$domain is an alias domain now";
+            return $result;
+        }
+        if (!empty($this->repo->rows('domain', ['domain' => $domain]))) {
+            // created meanwhile: only what is missing is added
+            $result['skipped']++;
+        } else {
+            $this->createDomain($domain, $data, $result);
+        }
+        $this->repo->inTransaction(function () use ($domain, $data, &$result): void {
+            $this->aliasDomains($domain, $data, $result);
+            $this->domainSettings($domain, $data, $result);
+        });
+        $this->redisSettings($domain, $data, $result);
+        $groupIds = $this->createGroups($domain, $data, $result);
+        $this->repo->inTransaction(function () use ($domain, $data, $groupIds, &$result): void {
+            $this->rules($domain, $data, 'mail_policies', 'mail_policy', 'mail_policy_member', 'policy_id', $groupIds, $result);
+            $this->rules($domain, $data, 'retention_rules', 'retention_rule', 'retention_rule_member', 'rule_id', $groupIds, $result);
+        });
+        return $result;
+    }
+
+    /** The domain row, its tags and the effects of the legacy domain add */
+    private function createDomain(string $domain, array $data, array &$result): void
+    {
+        // the domain keeps its creation time: domain admins may use the snapshots made after it
+        $row = array_diff_key(RestoreService::decode($data['domain']), array_flip(['modified']));
+        $relayhost = (string)($row['relayhost'] ?? '0');
+        if ($relayhost !== '0' && $relayhost !== '' && empty($this->repo->rows('relayhosts', ['id' => (int)$relayhost]))) {
+            $row['relayhost'] = '0';
+            $result['errors'][] = 'relayhost: does not exist';
+        }
+        $this->repo->inTransaction(function () use ($domain, $row, $data): void {
+            $this->repo->insert('domain', ['domain' => $domain] + $row);
+            $this->repo->dropExternalSenderAcl($domain);
+            foreach (array_unique(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', (array)($data['tags'] ?? [])), 'strlen')) as $tag) {
+                $this->repo->insert('tags_domain', ['domain' => $domain, 'tag_name' => $tag]);
+            }
+        });
+        $result['added']++;
+        $result['created'] = 1;
+        $this->effect($result, 'domain map', fn() => $this->legacy->addDomainMap($domain));
+        $error = $this->legacy->restartSogo();
+        if ($error !== null) {
+            $result['errors'][] = 'SOGo restart failed: ' . $error;
+        }
+    }
+
+    private function aliasDomains(string $domain, array $data, array &$result): void
+    {
+        foreach (self::rows($data, 'alias_domains') as $row) {
+            $alias = strtolower(trim((string)($row['alias_domain'] ?? '')));
+            if ($alias === '' || $this->repo->domainExists($alias)) {
+                $result['skipped']++;
+                continue;
+            }
+            $this->repo->insert('alias_domain', ['alias_domain' => $alias, 'target_domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $this->repo->dropExternalSenderAcl($alias);
+            $this->effect($result, "domain map of $alias", fn() => $this->legacy->addDomainMap($alias));
+            $result['added']++;
+        }
+    }
+
+    /** Spam settings, BCC maps, footer, MTA-STS and domain admins of the domain that are missing */
+    private function domainSettings(string $domain, array $data, array &$result): void
+    {
+        foreach (self::rows($data, 'filterconf') as $row) {
+            $option = (string)($row['option'] ?? '');
+            $value = (string)($row['value'] ?? '');
+            $current = $this->repo->rows('filterconf', ['object' => $domain]);
+            $present = in_array($option, ['whitelist_from', 'blacklist_from'], true)
+                ? array_filter($current, fn(array $c): bool => in_array($c['option'], ['whitelist_from', 'blacklist_from'], true) && strtolower((string)$c['value']) === strtolower($value))
+                : array_filter($current, fn(array $c): bool => $c['option'] === $option && (in_array($option, ['lowspamlevel', 'highspamlevel'], true) || (string)$c['value'] === $value));
+            if ($option === '' || !empty($present)) {
+                $result['skipped']++;
+                continue;
+            }
+            $this->repo->insert('filterconf', ['object' => $domain, 'option' => $option, 'value' => $value]);
+            $result['added']++;
+        }
+        foreach (self::rows($data, 'bcc_maps') as $row) {
+            $localDest = strtolower((string)($row['local_dest'] ?? ''));
+            if (!in_array($localDest, [$domain, '@' . $domain], true) || !filter_var((string)($row['bcc_dest'] ?? ''), FILTER_VALIDATE_EMAIL)
+                || !in_array($row['type'] ?? '', ['sender', 'rcpt'], true)) {
+                $result['errors'][] = "BCC map $localDest: invalid";
+            } elseif (!empty($this->repo->rows('bcc_maps', ['local_dest' => $localDest, 'type' => $row['type']]))) {
+                $result['skipped']++;
+            } else {
+                $this->repo->insert('bcc_maps', ['domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
+                $result['added']++;
+            }
+        }
+        foreach (['footer' => 'domain_wide_footer', 'mta_sts' => 'mta_sts'] as $key => $table) {
+            if (!is_array($data[$key] ?? null)) {
+                continue;
+            }
+            if (!empty($this->repo->rows($table, ['domain' => $domain]))) {
+                $result['skipped']++;
+                continue;
+            }
+            $this->repo->insert($table, ['domain' => $domain] + array_diff_key(RestoreService::decode($data[$key]), array_flip(['created', 'modified'])));
+            $result['added']++;
+        }
+        foreach (self::rows($data, 'domain_admins') as $row) {
+            $username = (string)($row['username'] ?? '');
+            if (empty($this->repo->rows('admin', ['username' => $username, 'superadmin' => 0]))) {
+                $result['errors'][] = "domain admin $username: does not exist";
+            } elseif (!empty($this->repo->rows('domain_admins', ['username' => $username, 'domain' => $domain]))) {
+                $result['skipped']++;
+            } else {
+                $this->repo->insert('domain_admins', ['username' => $username, 'domain' => $domain, 'active' => (int)($row['active'] ?? 1)]);
+                $result['added']++;
+            }
+        }
+    }
+
+    /** DKIM keys of the domain and its alias domains, and the rate limit of the domain (Redis), when missing */
+    private function redisSettings(string $domain, array $data, array &$result): void
+    {
+        $domains = array_merge([$domain], $this->repo->aliasDomainsOf($domain));
+        foreach (self::rows($data, 'dkim') as $key) {
+            $name = strtolower((string)($key['domain'] ?? ''));
+            $selector = (string)($key['selector'] ?? '');
+            if (!in_array($name, $domains, true)) {
+                continue;
+            }
+            if (!preg_match('/^[a-z0-9._-]{1,63}$/i', $selector) || (string)($key['pubkey'] ?? '') === '' || (string)($key['privkey'] ?? '') === '') {
+                $result['errors'][] = "DKIM key of $name: invalid";
+                continue;
+            }
+            $this->effect($result, "DKIM key of $name", function () use ($name, $selector, $key, &$result): void {
+                if ($this->legacy->dkim($name) !== null) {
+                    $result['skipped']++;
+                    return;
+                }
+                $this->legacy->addDkim($name, $selector, (string)$key['pubkey'], (string)$key['privkey']);
+                $result['added']++;
+            });
+        }
+        $rateLimit = $data['ratelimit'] ?? null;
+        if (is_string($rateLimit) && preg_match(self::RATE_LIMIT, $rateLimit)) {
+            $this->effect($result, 'rate limit', function () use ($domain, $rateLimit, &$result): void {
+                if (isset($this->legacy->rateLimits([$domain])[$domain])) {
+                    $result['skipped']++;
+                    return;
+                }
+                $this->legacy->setRateLimit($domain, $rateLimit);
+                $result['added']++;
+            });
+        }
+    }
+
+    /**
+     * Groups of the domain missing by name, through the group module (address, flags, priority, IdP link)
+     *
+     * @return array<int, int> id in the backup => id now, of all groups found by name
+     */
+    private function createGroups(string $domain, array $data, array &$result): array
+    {
+        $ids = [];
+        $admin = new Context('admin', 'backup', [], true);
+        foreach (self::rows($data, 'groups') as $row) {
+            $name = (string)($row['name'] ?? '');
+            $current = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => $name])[0] ?? null;
+            if ($current !== null) {
+                $ids[(int)($row['id'] ?? 0)] = (int)$current['id'];
+                $result['skipped']++;
+                continue;
+            }
+            $input = ['domain' => $domain, 'name' => $name, 'description' => (string)($row['description'] ?? '')];
+            foreach (['sender_allowed', 'sogo_visible', 'internal', 'active', 'priority'] as $key) {
+                if (isset($row[$key])) {
+                    $input[$key] = (int)$row[$key];
+                }
+            }
+            foreach (['address', 'idp_ref'] as $key) {
+                if (($row[$key] ?? null) !== null && $row[$key] !== '') {
+                    $input[$key] = (string)$row[$key];
+                }
+            }
+            try {
+                $ids[(int)($row['id'] ?? 0)] = $this->groups->create($admin, $input);
+                $result['added']++;
+            } catch (UserError $e) {
+                $result['errors'][] = "group $name: " . $e->key;
+            }
+        }
+        return $ids;
+    }
+
+    /**
+     * Mail policies or retention rules missing by name, with their group assignments; group:<id> entries of policies
+     * follow the groups' new ids
+     */
+    private function rules(string $domain, array $data, string $key, string $table, string $member, string $column, array $groupIds, array &$result): void
+    {
+        foreach (self::rows($data, $key) as $row) {
+            $name = (string)($row['name'] ?? '');
+            if (!empty($this->repo->rows($table, ['domain' => $domain, 'name' => $name]))) {
+                $result['skipped']++;
+                continue;
+            }
+            $values = array_diff_key($row, array_flip([...self::NEW_ROW, 'group_members']));
+            foreach (self::POLICY_LISTS as $list) {
+                if (isset($values[$list])) {
+                    $values[$list] = json_encode($this->remapGroups((string)$values[$list], $groupIds, "$name ($list)", $result));
+                }
+            }
+            $id = $this->repo->insert($table, ['domain' => $domain, 'name' => $name] + $values);
+            foreach ((array)($row['group_members'] ?? []) as $groupName) {
+                $group = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => (string)$groupName])[0] ?? null;
+                if ($group === null) {
+                    $result['errors'][] = "$name: group $groupName does not exist";
+                    continue;
+                }
+                $this->repo->insert($member, [$column => $id, 'group_id' => (int)$group['id']]);
+            }
+            $result['added']++;
+        }
+    }
+
+    /** @return string[] entries of a JSON list with group:<id> of the backup turned into the group's id now */
+    private function remapGroups(string $json, array $groupIds, string $what, array &$result): array
+    {
+        $entries = [];
+        foreach ((array)json_decode($json, true) as $entry) {
+            if (!is_string($entry)) {
+                continue;
+            }
+            if (preg_match('/^group:(\d+)$/', $entry, $match)) {
+                if (!isset($groupIds[(int)$match[1]])) {
+                    $result['errors'][] = "$what: $entry dropped (group does not exist)";
+                    continue;
+                }
+                $entry = 'group:' . $groupIds[(int)$match[1]];
+            }
+            $entries[] = $entry;
+        }
+        return $entries;
+    }
+
+    /** A Redis or dockerapi step: a failure is reported and the rest goes on */
+    private function effect(array &$result, string $what, callable $work): void
+    {
+        try {
+            $work();
+        } catch (Throwable) {
+            $result['errors'][] = "$what: not stored (Redis)";
+        }
+    }
+
+    private static function rows(array $data, string $key): array
+    {
+        return array_values(array_filter((array)($data[$key] ?? []), 'is_array'));
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/ExportService.php

@@ -0,0 +1,227 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use Mailcow\Backup\Legacy\LegacyDomain;
+use Mailcow\Backup\Repository\ExportRepository;
+use Mailcow\Core\UserError;
+
+/**
+ * Export of a domain for the backup (/mailcow/export/<domain>.json in a snapshot), in pages of mailboxes the runner
+ * merges into one file. The first page holds version, db_version, domain, domain_data (what recreating the domain
+ * needs), the aliases and `mailbox_aliases` (username => the aliases of any domain whose goto names the mailbox,
+ * without its own alias and group addresses; the runner moves them into the mailbox sections when it reads the file);
+ * every page a part of `mailboxes` and `next_after`, the last username of the page to ask for the next one with
+ * (keyset paging; null after the last page).
+ *
+ * Per mailbox every row a restore needs, with all columns. Password hashes, DKIM keys and the secrets of app
+ * passwords, 2FA and sync jobs are part of it on purpose; only the runner gets it (backup.php with the token). Binary
+ * values (fido2) are written as {"base64": "..."}. Beside the keys of the spec, a mailbox has `own_alias` (its alias
+ * row: forwarding with or without a copy), `aliases` (empty, see mailbox_aliases) and `ratelimit` (Redis).
+ */
+class ExportService
+{
+    public const VERSION = 1;
+    public const PAGE_SIZE = 500;
+    /** Lists of a mailbox: key => [table, column with the username] */
+    private const MAILBOX_ROWS = [
+        'sender_acl' => ['sender_acl', 'logged_in_as'],
+        'sender_acl_granted' => ['sender_acl', 'send_as'],
+        'sieve_filters' => ['sieve_filters', 'username'],
+        'filterconf' => ['filterconf', 'object'],
+        'spamalias' => ['spamalias', 'goto'],
+        'bcc_maps' => ['bcc_maps', 'local_dest'],
+        'app_passwd' => ['app_passwd', 'mailbox'],
+        'tfa' => ['tfa', 'username'],
+        // FIDO2 keys of mailbox users (passwordless login on the user page), like those of admins
+        'fido2' => ['fido2', 'username'],
+        'identity_link' => ['identity_link', 'username'],
+    ];
+    /** Single rows of a mailbox: key => [table, column with the username] */
+    private const MAILBOX_ROW = [
+        'user_acl' => ['user_acl', 'username'],
+        'pushover' => ['pushover', 'username'],
+    ];
+
+    public function __construct(private ExportRepository $repo, private LegacyDomain $legacy)
+    {
+    }
+
+    /** @return string[] all domains */
+    public function domains(): array
+    {
+        return array_values(array_map('strval', $this->repo->domains()));
+    }
+
+    /**
+     * @param ?string $after the last username of the previous page (keyset paging), null for the first page
+     * @throws UserError domain_not_found
+     */
+    public function export(string $domain, ?string $after = null, int $pageSize = self::PAGE_SIZE): array
+    {
+        $domain = strtolower(trim($domain));
+        $row = ($domain === '') ? null : ($this->repo->rows('domain', 'domain', $domain)[0] ?? null);
+        if ($row === null) {
+            throw new UserError('domain_not_found', [htmlspecialchars($domain)]);
+        }
+        $rows = $this->repo->mailboxesAfter($domain, strtolower($after ?? ''), $pageSize + 1);
+        $more = count($rows) > $pageSize;
+        $rows = array_slice($rows, 0, $pageSize);
+        $export = ['version' => self::VERSION, 'domain' => $domain];
+        if ($after === null) {
+            $export += [
+                'db_version' => $this->repo->dbVersion(),
+                'domain_data' => $this->domainData($domain, $row),
+                'aliases' => array_map(fn(array $alias): array => self::encode($alias), $this->repo->aliases($domain)),
+                'mailbox_aliases' => $this->mailboxAliases($domain) ?: new \stdClass(),
+            ];
+        }
+        $export['mailboxes'] = $this->mailboxes($domain, $rows) ?: new \stdClass();
+        $export['next_after'] = ($more && !empty($rows)) ? strtolower((string)end($rows)['username']) : null;
+        return $export;
+    }
+
+    /** What recreating the domain needs: its row and settings, alias domains, DKIM keys, groups, policies, rules */
+    private function domainData(string $domain, array $row): array
+    {
+        $aliasDomains = $this->repo->rows('alias_domain', 'target_domain', $domain);
+        $dkim = [];
+        foreach (array_merge([$domain], array_map(fn(array $alias): string => strtolower((string)$alias['alias_domain']), $aliasDomains)) as $name) {
+            $key = $this->legacy->dkim($name);
+            if ($key !== null) {
+                $dkim[] = ['domain' => $name] + $key;
+            }
+        }
+        return [
+            'domain' => self::encode($row),
+            'tags' => array_map(fn(array $tag): string => (string)$tag['tag_name'], $this->repo->rows('tags_domain', 'domain', $domain)),
+            'alias_domains' => $aliasDomains,
+            'dkim' => $dkim,
+            'ratelimit' => $this->legacy->rateLimits([$domain])[$domain] ?? null,
+            'filterconf' => $this->repo->rows('filterconf', 'object', $domain),
+            'bcc_maps' => $this->repo->domainBccMaps($domain),
+            'footer' => $this->repo->rows('domain_wide_footer', 'domain', $domain)[0] ?? null,
+            'mta_sts' => $this->repo->rows('mta_sts', 'domain', $domain)[0] ?? null,
+            'domain_admins' => $this->repo->rows('domain_admins', 'domain', $domain),
+            'groups' => $this->repo->rows('mailbox_group', 'domain', $domain),
+            'mail_policies' => $this->repo->domainRules('mail_policy', 'mail_policy_member', 'policy_id', $domain),
+            'retention_rules' => $this->repo->domainRules('retention_rule', 'retention_rule_member', 'rule_id', $domain),
+        ];
+    }
+
+    /** @return array<string, array> username => its section */
+    private function mailboxes(string $domain, array $rows): array
+    {
+        $mailboxes = [];
+        foreach ($rows as $row) {
+            $mailboxes[strtolower((string)$row['username'])] = [
+                'mailbox' => self::encode($row) + ['tags' => []],
+                'user_acl' => null,
+                'pushover' => null,
+                'ratelimit' => null,
+                'own_alias' => null,
+                'aliases' => [],
+                'groups' => [],
+                'mail_policies' => [],
+                'retention_rules' => [],
+                'imapsync' => [],
+            ] + array_fill_keys(array_keys(self::MAILBOX_ROWS), []);
+        }
+        $usernames = array_map('strval', array_keys($mailboxes));
+        if (empty($usernames)) {
+            return [];
+        }
+        // the mailbox of the page a row belongs to, null for anything else
+        $owner = fn(array $row, string $column): ?string => isset($mailboxes[strtolower((string)$row[$column])]) ? strtolower((string)$row[$column]) : null;
+        foreach ($this->repo->rowsOfMailboxes('tags_mailbox', 'username', $usernames) as $row) {
+            if (($username = $owner($row, 'username')) !== null) {
+                $mailboxes[$username]['mailbox']['tags'][] = (string)$row['tag_name'];
+            }
+        }
+        foreach (self::MAILBOX_ROW as $key => [$table, $column]) {
+            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
+                if (($username = $owner($row, $column)) !== null) {
+                    $mailboxes[$username][$key] = self::encode($row);
+                }
+            }
+        }
+        foreach (self::MAILBOX_ROWS as $key => [$table, $column]) {
+            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
+                if (($username = $owner($row, $column)) !== null) {
+                    $mailboxes[$username][$key][] = self::encode($row);
+                }
+            }
+        }
+        foreach ($this->legacy->rateLimits($usernames) as $username => $value) {
+            if (isset($mailboxes[strtolower((string)$username)])) {
+                $mailboxes[strtolower((string)$username)]['ratelimit'] = $value;
+            }
+        }
+        foreach ($this->repo->groupMemberships($usernames) as $row) {
+            if (($username = $owner($row, 'username')) !== null) {
+                $mailboxes[$username]['groups'][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain'],
+                    'source' => (string)$row['source']];
+            }
+        }
+        foreach (['mail_policies' => ['mail_policy', 'mail_policy_member', 'policy_id'], 'retention_rules' => ['retention_rule', 'retention_rule_member', 'rule_id']]
+            as $key => [$table, $member, $column]) {
+            foreach ($this->repo->assignments($table, $member, $column, $usernames) as $row) {
+                if (($username = $owner($row, 'username')) !== null) {
+                    $mailboxes[$username][$key][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain']];
+                }
+            }
+        }
+        foreach ($this->repo->syncJobs($usernames) as $row) {
+            if (($username = $owner($row, 'user2')) !== null) {
+                $mailboxes[$username]['imapsync'][] = self::encode($row);
+            }
+        }
+        foreach ($this->repo->rowsOfMailboxes('alias', 'address', $usernames) as $row) {
+            if (($username = $owner($row, 'address')) !== null) {
+                $mailboxes[$username]['own_alias'] = self::encode($row);
+            }
+        }
+        return $mailboxes;
+    }
+
+    /**
+     * @return array<string, array[]> username => aliases of any domain whose goto names the mailbox (once per domain: the
+     * alias table is scanned, not once per page)
+     */
+    private function mailboxAliases(string $domain): array
+    {
+        $usernames = array_fill_keys(array_map(fn(mixed $username): string => strtolower((string)$username), $this->repo->usernames($domain)), true);
+        $aliases = [];
+        foreach ($this->repo->aliasesPointingTo($domain) as $row) {
+            if ($row['group_id'] !== null) {
+                continue;
+            }
+            $address = strtolower((string)$row['address']);
+            foreach (array_unique(array_map('strtolower', self::gotos((string)$row['goto']))) as $goto) {
+                if ($goto !== $address && isset($usernames[$goto])) {
+                    $aliases[$goto][] = self::encode($row);
+                }
+            }
+        }
+        return $aliases;
+    }
+
+    /** @return string[] addresses of a goto list */
+    public static function gotos(string $goto): array
+    {
+        return array_values(array_filter(array_map('trim', explode(',', $goto)), 'strlen'));
+    }
+
+    /** Binary values (not UTF-8) as {"base64": "..."}, so the row survives JSON */
+    public static function encode(array $row): array
+    {
+        foreach ($row as $column => $value) {
+            if (is_string($value) && !mb_check_encoding($value, 'UTF-8')) {
+                $row[$column] = ['base64' => base64_encode($value)];
+            }
+        }
+        return $row;
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/GroupMembers.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use Mailcow\Core\Context;
+use Mailcow\Core\UserError;
+use Mailcow\MailboxGroup\Repository\GroupRepository;
+use Mailcow\MailboxGroup\Repository\LookupRepository;
+use Mailcow\MailboxGroup\Service\ChangeEffects;
+use Mailcow\MailboxGroup\Service\GroupService;
+use Mailcow\MailboxGroup\Service\SharingService;
+use Mailcow\MailboxGroup\Service\TemplateService;
+
+/**
+ * Group writes of a restore through the group module (addresses, send-as rights, shares and caches follow), as the
+ * requester of the restore: an admin, or the domain admin with his rights now (GroupService::canEdit). The module is
+ * wired without group templates (Legacy\NoTemplates).
+ */
+class GroupMembers
+{
+    public function __construct(private GroupRepository $repo, private LookupRepository $lookup, private TemplateService $templates,
+        private ChangeEffects $effects, private SharingService $sharing)
+    {
+    }
+
+    /** @throws UserError e.g. access_denied when the group is read-only for the requester */
+    public function add(Context $context, int $groupId, string $username): void
+    {
+        $this->service($context)->update($groupId, ['add_members' => [$username]]);
+    }
+
+    /**
+     * Creates a group (restore of a deleted domain); members follow with the mailboxes.
+     *
+     * @throws UserError
+     */
+    public function create(Context $context, array $input): int
+    {
+        return (int)$this->service($context)->create($input)->id;
+    }
+
+    protected function service(Context $context): GroupService
+    {
+        return new GroupService($this->repo, $this->lookup, $this->templates, $this->effects, $this->sharing, $context);
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/JobService.php

@@ -0,0 +1,653 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use DateTimeImmutable;
+use DateTimeZone;
+use Mailcow\Backup\Model\Job;
+use Mailcow\Backup\Model\Settings;
+use Mailcow\Backup\Model\Snapshot;
+use Mailcow\Backup\Repository\JobRepository;
+use Mailcow\Backup\Repository\SnapshotRepository;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+
+/**
+ * Jobs of the backup runners, the snapshots with their content and the status page.
+ *
+ * Backup, check, prune and init jobs: admins (not read-only API keys) queue them, one of each type at a time.
+ * Restores: admins, and domain admins for mailboxes or a domain they administer now. Mailboxes and domains deleted
+ * since the snapshot are created from it (deleted domains by admins only). A component a domain admin could not
+ * manage himself (e.g. sync jobs without the `syncjobs` ACL) is refused; his ACL and domains are stored with the job,
+ * so the merge and the runners apply the same limits.
+ * Reading: admins see every job and snapshot; domain admins the restores of their domains and, of each snapshot made
+ * after their domain was created, their domains in it.
+ */
+class JobService
+{
+    /**
+     * Restore components in their order. mails, imap_acl, sieve_activate and the Sieve scripts of filters are restored by
+     * the dovecot runner, sogo by the restic runner, the others by RestoreService.
+     */
+    public const COMPONENTS = ['mails', 'imap_acl', 'sogo', 'aliases', 'groups', 'sender_acl', 'mail_policies', 'retention_rules',
+        'filters', 'sieve_activate', 'spam', 'bcc', 'syncjobs', 'app_passwords', 'tfa', 'settings', 'password'];
+    /** Components that replace current data with the backup's (all others only add what is missing) */
+    public const OVERWRITE = ['sogo', 'settings', 'password'];
+    /** Components that need the mails: not in snapshots without them (RESTIC_BACKUP_MODE=config) */
+    public const MAIL_COMPONENTS = ['mails', 'imap_acl', 'sogo', 'sieve_activate'];
+    public const MAIL_TARGETS = ['folder', 'original'];
+    /** Domain admin ACL a component needs */
+    private const COMPONENT_ACL = [
+        'groups' => 'mailbox_groups',
+        'mail_policies' => 'mail_policies',
+        'retention_rules' => 'retention',
+        'filters' => 'filters',
+        'sieve_activate' => 'filters',
+        'spam' => 'spam_policy',
+        'bcc' => 'bcc_maps',
+        'syncjobs' => 'syncjobs',
+        'app_passwords' => 'app_passwds',
+    ];
+    /** Components that only work together with another one */
+    private const REQUIRES = ['sieve_activate' => 'filters'];
+    /** Domain admin ACLs the merge respects, stored with the job */
+    private const RESTORE_ACL = ['filters', 'mailbox_groups', 'mail_policies', 'retention', 'spam_policy', 'bcc_maps', 'syncjobs',
+        'app_passwds', 'protocol_access', 'mailbox_relayhost', 'unlimited_quota', 'sogo_access', 'pushover', 'extend_sender_acl',
+        'alias_external_goto', 'ratelimit', 'tls_policy', 'quarantine', 'quarantine_notification', 'quarantine_category'];
+    private const MAX_MAILBOXES = 1000;
+    private const LIST_LIMIT = 100;
+    private const MAX_LIST = 1000;
+    /** Mailboxes a search returns at most */
+    private const SEARCH_LIMIT = 500;
+    /** The runner reports every 10 seconds */
+    private const RUNNER_TIMEOUT = 120;
+    /** @var array<string, array{bool, ?int}> domain => [access of the domain admin, creation time now], per request */
+    private array $domainState = [];
+
+    public function __construct(private JobRepository $jobs, private SnapshotRepository $snapshots, private SettingsService $settings,
+        private Context $context, private string $timezone = 'UTC')
+    {
+    }
+
+    /**
+     * @param mixed $type only jobs of this type ('' or null = all; an unknown type finds nothing)
+     * @return Job[] newest first, at most $limit (0 = LIST_LIMIT)
+     */
+    public function list(int $limit = 0, mixed $type = null): array
+    {
+        $type = Input::text($type);
+        if ($type !== '' && !in_array($type, Job::TYPES, true)) {
+            return [];
+        }
+        return $this->jobs->visible($this->context, ($limit > 0) ? min($limit, self::MAX_LIST) : self::LIST_LIMIT, ($type === '') ? null : $type);
+    }
+
+    /**
+     * Entry of the API. Domain admins get it restricted: e-mail addresses outside the job's domain and its alias domains
+     * and repository details (URLs, restic repository strings, paths of the repository) are shown as "…".
+     */
+    public function view(Job $job, bool $withLog = false): array
+    {
+        $entry = $job->toArray($withLog);
+        if ($this->context->hasAdminRights()) {
+            return $entry;
+        }
+        $domains = array_merge([strtolower($job->domain)], $this->jobs->aliasDomainsOf(strtolower($job->domain)));
+        foreach (['result', 'error', 'log'] as $key) {
+            if (isset($entry[$key])) {
+                $entry[$key] = self::restricted($entry[$key], $domains);
+            }
+        }
+        return $entry;
+    }
+
+    /** The job with its log, null when it does not exist or the caller may not see it */
+    public function get(int $id): ?Job
+    {
+        $job = $this->jobs->find($id, true);
+        return ($job !== null && $this->visible($job)) ? $job : null;
+    }
+
+    /**
+     * Queues a backup, check, prune or init job (admins).
+     *
+     * @throws UserError
+     */
+    public function create(array $input): Job
+    {
+        $this->context->requireAdmin();
+        $type = Input::text($input['type'] ?? null);
+        if (!in_array($type, Job::MANUAL_TYPES, true)) {
+            throw new UserError('backup_job_type_invalid', [htmlspecialchars((string)$type)]);
+        }
+        $settings = $this->settings->load();
+        if (!$settings->configured()) {
+            throw new UserError('backup_not_configured');
+        }
+        if ($type === 'backup' && $settings->get('password_confirmed') !== '1') {
+            throw new UserError('backup_password_not_confirmed');
+        }
+        $job = new Job($type);
+        $job->requestedBy = (string)$this->context->username;
+        $this->jobs->insertExclusive($job, function (array $active) use ($type): void {
+            foreach ($active as $other) {
+                if ($other->type === $type) {
+                    throw new UserError('backup_job_active', [$type]);
+                }
+            }
+        });
+        return $job;
+    }
+
+    /**
+     * Queues a restore: {snapshot, scope: mailbox|domain, mailboxes | domain, components, mail_target: folder|original}.
+     * All mailboxes belong to one domain. Mailboxes deleted since the snapshot are created from it (create_mailboxes);
+     * an admin may restore a deleted domain (create_domain). Scope domain restores the snapshot's mailboxes of the
+     * domain (the current ones for snapshots without content).
+     *
+     * @throws UserError
+     */
+    public function restore(array $input): Job
+    {
+        $this->requireRestorer();
+        if (!$this->settings->load()->configured()) {
+            throw new UserError('backup_not_configured');
+        }
+        $snapshot = $this->snapshot($input['snapshot'] ?? null);
+        $scope = Input::text($input['scope'] ?? 'mailbox');
+        [$domain, $mailboxes] = match ($scope) {
+            'domain' => $this->domainScope($snapshot, $input['domain'] ?? null),
+            'mailbox' => $this->mailboxScope($snapshot, $input['mailboxes'] ?? null, $input['domain'] ?? null),
+            default => throw new UserError('backup_scope_invalid'),
+        };
+        $components = $this->components($input['components'] ?? null);
+        $missing = array_values(array_intersect($components, self::MAIL_COMPONENTS));
+        if ($snapshot->mode() === 'config' && !empty($missing)) {
+            throw new UserError('backup_components_no_mails', [htmlspecialchars(implode(', ', $missing))]);
+        }
+        $mailTarget = Input::text($input['mail_target'] ?? null) ?: 'folder';
+        if (!in_array($mailTarget, self::MAIL_TARGETS, true)) {
+            throw new UserError('backup_mail_target_invalid');
+        }
+        $job = new Job('restore');
+        $job->step = 'files';
+        $job->domain = $domain;
+        $job->requestedBy = (string)$this->context->username;
+        $job->params = [
+            'snapshot' => $snapshot->id,
+            'snapshot_time' => $snapshot->time,
+            'scope' => $scope,
+            'domain' => $domain,
+            'mailboxes' => $mailboxes,
+            'components' => $components,
+            'mail_target' => $mailTarget,
+            'requested_role' => $this->context->hasAdminRights() ? 'admin' : 'domainadmin',
+            'create_domain' => (int)!$this->jobs->isDomain($domain),
+            'create_mailboxes' => array_values(array_filter($mailboxes, fn(string $username): bool => $this->jobs->mailboxDomain($username) === null)),
+        ];
+        if (!$this->context->hasAdminRights()) {
+            $job->params['acl'] = array_combine(self::RESTORE_ACL, array_map(fn(string $acl): int => (int)$this->context->hasAcl($acl), self::RESTORE_ACL));
+            $job->params['allowed_domains'] = array_values(array_unique(array_merge([$domain], $this->jobs->aliasDomainsOf($domain))));
+        }
+        $domainAdmin = !$this->context->hasAdminRights();
+        $this->jobs->insertExclusive($job, function (array $active) use ($mailboxes, $domain, $domainAdmin): void {
+            foreach ($active as $other) {
+                if ($other->type !== 'restore') {
+                    continue;
+                }
+                // a domain admin has one restore per domain at a time
+                if ($domainAdmin && strtolower($other->domain) === $domain) {
+                    throw new UserError('backup_restore_domain_active', [htmlspecialchars($domain)]);
+                }
+                $both = array_intersect($mailboxes, $other->mailboxes());
+                if (!empty($both)) {
+                    throw new UserError('backup_restore_active', [htmlspecialchars((string)reset($both))]);
+                }
+            }
+        });
+        return $job;
+    }
+
+    /**
+     * Cancels a queued or running job (the restic runner stops at its next check): admins any, domain admins the
+     * restores of their domains that domain admins requested. A restore in the dovecot step (mails, mails_running) runs
+     * to its end.
+     *
+     * @throws UserError
+     */
+    public function cancel(int $id): Job
+    {
+        if ($this->context->isReadOnlyApi()) {
+            throw new AccessDenied();
+        }
+        $job = $this->jobs->find($id);
+        // domain admins cancel only restores domain admins requested
+        if ($job === null || !$this->visible($job) || (!$this->context->hasAdminRights() && ($job->params['requested_role'] ?? '') !== 'domainadmin')) {
+            throw new AccessDenied();
+        }
+        if (!$job->isActive() || ($job->type === 'restore' && in_array($job->step, ['mails', 'mails_running'], true)) || !$this->jobs->cancel($id)) {
+            throw new UserError('backup_job_not_cancellable', [$id]);
+        }
+        $job->state = 'cancelled';
+        return $job;
+    }
+
+    /**
+     * Snapshots, newest first. Domain admins get those holding one of their domains (see mayUse()), with id, short_id,
+     * time and those domains.
+     */
+    public function snapshots(): array
+    {
+        if ($this->context->hasAdminRights()) {
+            return array_map(fn(Snapshot $snapshot): array => $snapshot->toArray(), $this->snapshots->all());
+        }
+        if (!$this->context->isDomainAdmin()) {
+            return [];
+        }
+        $snapshots = [];
+        // the domains with their creation time come from the snapshot row, the content is not needed here
+        foreach ($this->snapshots->all() as $snapshot) {
+            $domains = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
+            if (!empty($domains)) {
+                $snapshots[] = ['id' => $snapshot->id, 'short_id' => $snapshot->shortId, 'time' => $snapshot->time, 'domains' => $domains,
+                    'mode' => $snapshot->mode()];
+            }
+        }
+        return $snapshots;
+    }
+
+    /**
+     * Domains of a snapshot with their state now: GET backup-snapshot-content/<snapshot>. `available` 0: the snapshot has
+     * no content (not made by this installation), the list comes from its manifest and only existing mailboxes can be restored. A domain
+     * of the manifest missing in the content (its export failed) lists its current mailboxes as well.
+     *
+     * @return array{snapshot: string, time: int, available: int, domains: array[]}|array{} [] when not found or not visible
+     */
+    public function content(mixed $snapshotId): array
+    {
+        $snapshot = $this->visibleSnapshot($snapshotId);
+        if ($snapshot === null) {
+            return [];
+        }
+        $visible = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
+        [$existing, $usernames] = $this->jobs->domainsAndUsernames($visible);
+        $domains = [];
+        foreach ($visible as $domain) {
+            $current = $usernames[$domain] ?? [];
+            $mailboxes = $snapshot->mailboxesOf($domain) ?? $current;
+            $domains[] = ['domain' => $domain, 'exists' => (int)isset($existing[$domain]), 'mailboxes' => count($mailboxes),
+                'missing' => count(array_diff_key($mailboxes, $current))];
+        }
+        return ['snapshot' => $snapshot->id, 'time' => $snapshot->time, 'available' => (int)($snapshot->content !== null), 'domains' => $domains];
+    }
+
+    /**
+     * Search in a snapshot: GET backup-snapshot-search/<snapshot>/<term>. Matches domain names and mailboxes by local
+     * part, address or name ("x@": local part, "@d": domain, "x@d": address), with the rules of content(); at most
+     * SEARCH_LIMIT mailboxes. A matching domain comes with all its mailboxes.
+     *
+     * @return array{domains: array[], truncated: int}|array{} [] when the snapshot is not found or not visible
+     */
+    public function search(mixed $snapshotId, mixed $term): array
+    {
+        $snapshot = $this->visibleSnapshot($snapshotId);
+        $term = mb_strtolower((string)Input::text($term));
+        if ($snapshot === null) {
+            return [];
+        }
+        if (trim($term, '@') === '') {
+            return ['domains' => [], 'truncated' => 0];
+        }
+        $domains = [];
+        $hits = 0;
+        $truncated = 0;
+        [$local, $at, $domainTerm] = str_contains($term, '@') ? [strstr($term, '@', true), true, substr((string)strstr($term, '@'), 1)] : [$term, false, ''];
+        foreach ($snapshot->domainList() as $domain) {
+            if (!$this->mayUse($snapshot, $domain)) {
+                continue;
+            }
+            $domainMatch = $at ? ($domainTerm !== '' && str_contains($domain, $domainTerm)) : str_contains($domain, $term);
+            if ($at && $domainTerm !== '' && !$domainMatch) {
+                continue;
+            }
+            $current = $this->jobs->mailboxNames($domain);
+            $mailboxes = [];
+            foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
+                $username = (string)$username;
+                $part = (string)strstr($username, '@', true);
+                $match = match (true) {
+                    // "x@d": the address, "x@": the local part, "@d": every mailbox of the matching domains
+                    $at && $local !== '' && $domainTerm !== '' => str_contains($username, $term),
+                    $at && $local !== '' => str_contains($part, $local),
+                    $at => true,
+                    default => $domainMatch || str_contains($username, $term) || str_contains(mb_strtolower($name), $term),
+                };
+                if (!$match) {
+                    continue;
+                }
+                if ($hits >= self::SEARCH_LIMIT) {
+                    $truncated = 1;
+                    break;
+                }
+                $hits++;
+                $mailboxes[] = ['username' => $username, 'name' => $name, 'exists' => (int)isset($current[$username])];
+            }
+            // a domain without matching mailboxes only for a search by domain
+            if (!empty($mailboxes) || ($domainMatch && !($at && $local !== '') && !$truncated)) {
+                $domains[] = ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
+            }
+            if ($truncated) {
+                break;
+            }
+        }
+        return ['domains' => $domains, 'truncated' => $truncated];
+    }
+
+    /**
+     * Mailboxes of a domain in a snapshot with their state now: GET backup-snapshot-content/<snapshot>/<domain>
+     *
+     * @return array{domain: string, exists: int, mailboxes: array[]}|array{} [] when not found or not visible
+     */
+    public function domainContent(mixed $snapshotId, mixed $domain): array
+    {
+        $snapshot = $this->visibleSnapshot($snapshotId);
+        $domain = strtolower((string)Input::text($domain));
+        if ($snapshot === null || !$snapshot->hasDomain($domain) || !$this->mayUse($snapshot, $domain)) {
+            return [];
+        }
+        $current = $this->jobs->mailboxNames($domain);
+        $mailboxes = [];
+        foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
+            $mailboxes[] = ['username' => (string)$username, 'name' => $name, 'exists' => (int)isset($current[$username])];
+        }
+        return ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
+    }
+
+    /**
+     * State of the backup for admins: settings state, runner heartbeat, repository, newest job of each type, queued and
+     * running jobs, the next scheduled run. [] for everybody else.
+     */
+    public function status(?int $now = null): array
+    {
+        if (!$this->context->hasAdminRights()) {
+            return [];
+        }
+        $now ??= time();
+        $settings = $this->settings->load();
+        $seen = ($settings->get('state_runner_seen') === '') ? null : (int)$settings->get('state_runner_seen');
+        $snapshots = $this->snapshots->all();
+        $latest = $this->jobs->latestOfEachType();
+        $stats = json_decode($settings->get('state_repo_stats'), true);
+        return [
+            'enabled' => (int)$settings->get('enabled'),
+            'target' => $settings->get('target'),
+            'configured' => (int)$settings->configured(),
+            'password_confirmed' => (int)$settings->get('password_confirmed'),
+            'initialized' => (int)$settings->get('state_initialized'),
+            // the runner cannot decrypt the stored credentials (e.g. a database restored on another server): enter them again
+            'secrets_error' => ($settings->get('state_secrets_error') === '') ? null : $settings->get('state_secrets_error'),
+            // RESTIC_BACKUP_MODE (null until the runner reported it)
+            'mode' => ($settings->get('state_backup_mode') === '') ? null : $settings->get('state_backup_mode'),
+            'runner_seen' => $seen,
+            'runner_age' => ($seen === null) ? null : max(0, $now - $seen),
+            'runner_ok' => (int)($seen !== null && $now - $seen <= self::RUNNER_TIMEOUT),
+            'last_success' => ($settings->get('state_last_success') === '') ? null : (int)$settings->get('state_last_success'),
+            'repo_size' => ($settings->get('state_repo_size') === '') ? null : (int)$settings->get('state_repo_size'),
+            'repo_stats' => is_array($stats) ? $stats : null,
+            'snapshots' => count($snapshots),
+            'latest_snapshot' => empty($snapshots) ? null : $snapshots[0]->time,
+            'next_run' => $this->nextRun($settings, $now),
+            // the longest time between two scheduled backups (seconds): a later last success is overdue
+            'schedule_gap' => $settings->scheduleGap(),
+            'jobs' => array_map(fn(string $type): ?array => isset($latest[$type]) ? $latest[$type]->toArray() : null,
+                array_combine(Job::TYPES, Job::TYPES)),
+            'active' => array_map(fn(Job $job): array => $job->toArray(), $this->jobs->active()),
+        ];
+    }
+
+    /**
+     * Unix time of the next scheduled backup, null while the schedule is off. Like the runner: the schedule is armed
+     * (state_schedule_since/state_schedule) when it is switched on or changed, a run is due at the first schedule time
+     * after that; a due run without a scheduled backup since its time is the next one (now).
+     */
+    private function nextRun(Settings $settings, int $now): ?int
+    {
+        if ($settings->get('enabled') !== '1' || $settings->get('password_confirmed') !== '1' || !$settings->configured()) {
+            return null;
+        }
+        $slots = $settings->scheduleSlots();
+        try {
+            $zone = new DateTimeZone($this->timezone);
+        } catch (\Exception) {
+            $zone = new DateTimeZone('UTC');
+        }
+        $today = (new DateTimeImmutable('@' . $now))->setTimezone($zone);
+        // the last schedule time up to now, and the one after it
+        $due = self::slot($slots, $today, -1);
+        $next = self::slot($slots, $today, 1);
+        $since = $settings->get('state_schedule_since');
+        if ($due === null || $since === '' || $settings->get('state_schedule') !== $settings->get('schedule') || (int)$since > $due) {
+            return $next;
+        }
+        return $this->jobs->scheduledSince($due) ? $next : $due;
+    }
+
+    /**
+     * The newest schedule time up to $now ($direction -1) or the first one after it (1) as Unix time, null without
+     * slots ([weekday, hour, minute], sorted)
+     */
+    private static function slot(array $slots, DateTimeImmutable $now, int $direction): ?int
+    {
+        for ($i = 0; $i <= 7; $i++) {
+            $day = $now->modify(sprintf('%+d day', $direction * $i));
+            foreach (($direction < 0) ? array_reverse($slots) : $slots as [$weekday, $hour, $minute]) {
+                if ($weekday !== (int)$day->format('w')) {
+                    continue;
+                }
+                $slot = $day->setTime($hour, $minute)->getTimestamp();
+                if (($direction < 0) ? $slot <= $now->getTimestamp() : $slot > $now->getTimestamp()) {
+                    return $slot;
+                }
+            }
+        }
+        return null;
+    }
+
+    /** @throws AccessDenied unless the caller may request restores */
+    private function requireRestorer(): void
+    {
+        if ($this->context->isReadOnlyApi() || !($this->context->hasAdminRights() || $this->context->isDomainAdmin())) {
+            throw new AccessDenied();
+        }
+    }
+
+    /** Admins every job; domain admins the restores of their domains requested after the domain was created */
+    private function visible(Job $job): bool
+    {
+        if ($this->context->hasAdminRights()) {
+            return true;
+        }
+        if (!$this->context->isDomainAdmin() || $job->type !== 'restore' || $job->domain === '' || !$this->context->canAccessDomain($job->domain)) {
+            return false;
+        }
+        $created = $this->jobs->domainCreated(strtolower($job->domain));
+        return $created !== null && (int)$job->created >= $created;
+    }
+
+    /**
+     * A value of a job as a domain admin sees it: e-mail addresses outside $domains and repository details become "…"
+     *
+     * @param string[] $domains
+     */
+    private static function restricted(mixed $value, array $domains): mixed
+    {
+        if (is_array($value)) {
+            $restricted = [];
+            foreach ($value as $key => $item) {
+                $restricted[is_string($key) ? (string)self::restricted($key, $domains) : $key] = self::restricted($item, $domains);
+            }
+            return $restricted;
+        }
+        if (!is_string($value)) {
+            return $value;
+        }
+        $value = preg_replace(['~\b(?:s3|sftp|rest|b2|azure|gs|swift|rclone):\S+~i', '~\bhttps?://\S+~i', '~/repository\S*~'], '…', $value);
+        return preg_replace_callback('/[^\s@<>"\'(),;:\[\]]+@([a-z0-9.-]+\.[a-z]{2,})/i',
+            fn(array $match): string => in_array(strtolower($match[1]), $domains, true) ? $match[0] : '…', (string)$value);
+    }
+
+    /** @throws UserError backup_snapshot_not_found (also for snapshots a domain admin may not use) */
+    private function snapshot(mixed $value): Snapshot
+    {
+        $id = (string)Input::text($value);
+        $snapshot = ($id === '') ? null : $this->snapshots->find($id);
+        if ($snapshot === null || (!$this->context->hasAdminRights() && empty(array_filter($snapshot->domainList(),
+            fn(string $domain): bool => $this->mayUse($snapshot, $domain))))) {
+            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($id)]);
+        }
+        return $snapshot;
+    }
+
+    private function visibleSnapshot(mixed $value): ?Snapshot
+    {
+        try {
+            return ($this->context->hasAdminRights() || $this->context->isDomainAdmin()) ? $this->snapshot($value) : null;
+        } catch (UserError) {
+            return null;
+        }
+    }
+
+    /**
+     * Admins every domain of the snapshot; domain admins the domains they administer now, when the snapshot holds this
+     * domain and not a former one of the same name: its recorded creation time is the domain's (snapshots without it:
+     * made after the domain was created)
+     */
+    private function mayUse(Snapshot $snapshot, string $domain): bool
+    {
+        if ($this->context->hasAdminRights()) {
+            return true;
+        }
+        if (!$this->context->isDomainAdmin()) {
+            return false;
+        }
+        $this->domainState[$domain] ??= [$this->context->canAccessDomain($domain), $this->jobs->domainCreated($domain)];
+        [$access, $created] = $this->domainState[$domain];
+        if (!$access) {
+            return false;
+        }
+        $recorded = $snapshot->createdOf($domain);
+        return $created !== null && (($recorded !== null) ? $recorded === $created : $snapshot->time > $created);
+    }
+
+    /**
+     * @return array{0: string, 1: string[]} the domain and its mailboxes in the snapshot (now, for snapshots without content)
+     * @throws UserError
+     */
+    private function domainScope(Snapshot $snapshot, mixed $value): array
+    {
+        $domain = strtolower((string)Input::text($value));
+        $exists = $domain !== '' && $this->jobs->isDomain($domain);
+        // a deleted domain is created from the snapshot's export: only when the content lists it
+        if (!$exists && !($this->context->hasAdminRights() && $snapshot->mailboxesOf($domain) !== null)) {
+            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
+        }
+        $this->requireDomain($snapshot, $domain);
+        $listed = $snapshot->mailboxesOf($domain);
+        $mailboxes = ($listed === null) ? $this->jobs->mailboxesOf($domain) : array_map('strval', array_keys($listed));
+        // a deleted domain without mailboxes comes back with its aliases and settings
+        if (empty($mailboxes) && $exists) {
+            throw new UserError('backup_mailboxes_empty');
+        }
+        return [$domain, $mailboxes];
+    }
+
+    /**
+     * @return array{0: string, 1: string[]} the domain of the mailboxes and the mailboxes
+     * @throws UserError
+     */
+    private function mailboxScope(Snapshot $snapshot, mixed $value, mixed $domainValue): array
+    {
+        $mailboxes = array_values(array_unique(array_map('strtolower', self::values($value))));
+        if (empty($mailboxes)) {
+            throw new UserError('backup_mailboxes_empty');
+        }
+        if (count($mailboxes) > self::MAX_MAILBOXES) {
+            throw new UserError('backup_entries_exceeded', [self::MAX_MAILBOXES]);
+        }
+        $domain = strtolower((string)Input::text($domainValue));
+        foreach ($mailboxes as $username) {
+            // a deleted mailbox can be restored when the snapshot knows it
+            $mailboxDomain = $this->jobs->mailboxDomain($username) ?? (isset($snapshot->mailboxesOf(self::domainOf($username))[$username]) ? self::domainOf($username) : null);
+            if ($mailboxDomain === null) {
+                throw new UserError('backup_mailbox_not_found', [htmlspecialchars($username)]);
+            }
+            if ($domain !== '' && $mailboxDomain !== $domain) {
+                throw new UserError('backup_mailboxes_one_domain');
+            }
+            $domain = $mailboxDomain;
+        }
+        if (!$this->jobs->isDomain($domain) && !$this->context->hasAdminRights()) {
+            throw new AccessDenied();
+        }
+        $this->requireDomain($snapshot, $domain);
+        return [$domain, $mailboxes];
+    }
+
+    /** @throws UserError the snapshot does not hold the domain, or the caller may not restore into it */
+    private function requireDomain(Snapshot $snapshot, string $domain): void
+    {
+        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
+            throw new AccessDenied();
+        }
+        if ($snapshot->knowsDomains() && !$snapshot->hasDomain($domain)) {
+            throw new UserError('backup_domain_not_in_snapshot', [htmlspecialchars($domain)]);
+        }
+        if (!$this->mayUse($snapshot, $domain)) {
+            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($snapshot->shortId)]);
+        }
+    }
+
+    /**
+     * @return string[] the requested components in their order
+     * @throws UserError
+     */
+    private function components(mixed $value): array
+    {
+        $requested = array_values(array_unique(self::values($value)));
+        if (empty($requested)) {
+            throw new UserError('backup_components_invalid', ['']);
+        }
+        foreach ($requested as $component) {
+            if (!in_array($component, self::COMPONENTS, true)) {
+                throw new UserError('backup_components_invalid', [htmlspecialchars($component)]);
+            }
+            if (!$this->context->hasAdminRights() && isset(self::COMPONENT_ACL[$component]) && !$this->context->hasAcl(self::COMPONENT_ACL[$component])) {
+                throw new UserError('backup_component_denied', [$component]);
+            }
+            if (isset(self::REQUIRES[$component]) && !in_array(self::REQUIRES[$component], $requested, true)) {
+                throw new UserError('backup_component_requires', [$component, self::REQUIRES[$component]]);
+            }
+        }
+        return array_values(array_intersect(self::COMPONENTS, $requested));
+    }
+
+    private static function domainOf(string $address): string
+    {
+        return strtolower(substr((string)strrchr($address, '@'), 1));
+    }
+
+    /**
+     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
+     *
+     * @return string[]
+     */
+    private static function values(mixed $value): array
+    {
+        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
+        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/RestoreService.php

@@ -0,0 +1,996 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use Mailcow\Backup\Legacy\LegacyDomain;
+use Mailcow\Backup\Model\Job;
+use Mailcow\Backup\Model\RestoreRun;
+use Mailcow\Backup\Repository\JobRepository;
+use Mailcow\Backup\Repository\RestoreRepository;
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Mailcow\Dovecot\Service\DovecotService;
+use Mailcow\Sogo\Service\SogoService;
+use PDOException;
+
+/**
+ * Merges the settings of a mailbox from a snapshot's export into the mailbox as it is now (the runner of a restore job
+ * posts one mailbox at a time to backup.php); a mailbox deleted since the snapshot is created first. The aliases of a
+ * domain restore come in parts before the mailboxes (mergeAliases()).
+ *
+ * Only what is missing is added, nothing is deleted; `settings` and `password` overwrite. Components and limits come
+ * from the job in the database, not from the request: the job must be running and hold the mailbox. A domain admin's
+ * restore keeps to his ACL at the time of the request, to his rights now for groups and to the job's domain (and its
+ * alias domains). A named lock per job and mailbox makes a retried request wait for the first one.
+ *
+ * The components run in one transaction per mailbox, each in a savepoint: a database error drops only that component.
+ * Group memberships go through the group module afterwards (its own transaction and effects, no group templates),
+ * then the SOGo view and the Dovecot auth cache of the touched mailboxes are refreshed.
+ */
+class RestoreService
+{
+    /** Components merged here, in this order */
+    public const COMPONENTS = ['aliases', 'sender_acl', 'mail_policies', 'retention_rules', 'filters', 'spam', 'bcc', 'syncjobs', 'app_passwords',
+        'tfa', 'settings', 'password', 'groups'];
+    /** Aliases per restore_aliases request */
+    public const MAX_ALIASES = 500;
+    private const SPECIAL_GOTOS = ['null@localhost', 'spam@localhost', 'ham@localhost'];
+    /** Attributes `settings` leaves alone: the password component's, the storage format, the IdP's */
+    private const KEEP_ATTRIBUTES = ['passwd_update', 'mailbox_format', 'attribute_hash'];
+    /** Attributes a domain admin needs an ACL for */
+    private const ATTRIBUTE_ACL = ['imap_access' => 'protocol_access', 'pop3_access' => 'protocol_access', 'smtp_access' => 'protocol_access',
+        'sieve_access' => 'protocol_access', 'eas_access' => 'protocol_access', 'dav_access' => 'protocol_access',
+        'relayhost' => 'mailbox_relayhost', 'sogo_access' => 'sogo_access', 'tls_enforce_in' => 'tls_policy', 'tls_enforce_out' => 'tls_policy',
+        'quarantine_notification' => 'quarantine_notification', 'quarantine_category' => 'quarantine_category'];
+    /** Columns a restored row never takes from the backup */
+    private const NEW_ROW = ['id', 'created', 'modified'];
+    /** @var array<string, int> primary domain => aliases it may still get, counted once per request */
+    private array $aliasesLeft = [];
+    private const MIB = 1048576;
+
+    public function __construct(private RestoreRepository $repo, private JobRepository $jobs, private GroupMembers $groups,
+        private SogoService $sogo, private DovecotService $dovecot, private LegacyDomain $legacy, private bool $externalGotoDisabled = false)
+    {
+    }
+
+    /**
+     * Action `restore`: one mailbox of a running restore job. A mailbox in params.create_mailboxes that does not exist
+     * is created from the backup first (result key `mailbox`); when that fails, nothing else is merged.
+     *
+     * @param mixed $data the mailbox's section of the domain export
+     * @return array<string, array{added: int, skipped: int, errors: string[]}> per component of the job merged here
+     * @throws UserError
+     */
+    public function merge(mixed $jobId, mixed $username, mixed $data): array
+    {
+        $job = $this->runningRestore($jobId);
+        $username = strtolower((string)Input::text($username));
+        if ($username === '' || !in_array($username, array_map('strtolower', $job->mailboxes()), true)) {
+            throw new AccessDenied();
+        }
+        if (!is_array($data) || !is_array($data['mailbox'] ?? null) || strtolower((string)($data['mailbox']['username'] ?? '')) !== $username) {
+            throw new UserError('backup_export_invalid');
+        }
+        return $this->locked($job, $username, fn(): array => $this->mergeMailbox($job, $username, $data));
+    }
+
+    /**
+     * Action `restore_aliases`: a part of the aliases of the domain (scope domain), before its mailboxes. Missing aliases
+     * are created with their gotos that are valid now; existing ones are left to the mailboxes' merge.
+     *
+     * @return array{aliases: array{added: int, skipped: int, errors: string[]}}
+     * @throws UserError
+     */
+    public function mergeAliases(mixed $jobId, mixed $aliases): array
+    {
+        $job = $this->runningRestore($jobId);
+        if (($job->params['scope'] ?? '') !== 'domain' || !in_array('aliases', $job->components(), true)) {
+            throw new AccessDenied();
+        }
+        if (!is_array($aliases)) {
+            throw new UserError('backup_export_invalid');
+        }
+        if (count($aliases) > self::MAX_ALIASES) {
+            throw new UserError('backup_entries_exceeded', [self::MAX_ALIASES]);
+        }
+        return $this->locked($job, '', function () use ($job, $aliases): array {
+            if ($this->permissionRevoked($job)) {
+                return ['aliases' => ['added' => 0, 'skipped' => 0, 'errors' => ['permission revoked']]];
+            }
+            $domain = strtolower((string)$job->domain);
+            $run = new RestoreRun('', $domain, [], $job->params, [], $this->allowedDomains($job, $domain), $job->requestedBy);
+            $run->start('aliases');
+            try {
+                $this->repo->inTransaction(fn() => $this->aliases($run, array_values(array_filter($aliases, 'is_array'))));
+            } catch (PDOException $e) {
+                $run->failed('aliases', 'database error ' . $e->getCode());
+            }
+            $this->sogo->refreshStaticView($run->sogoUsers);
+            return $run->results;
+        });
+    }
+
+    /** The mailbox's merge under the lock: create it if needed, then the components */
+    private function mergeMailbox(Job $job, string $username, array $data): array
+    {
+        if ($this->permissionRevoked($job)) {
+            return ['mailbox' => ['added' => 0, 'skipped' => 0, 'errors' => ['permission revoked'], 'created' => 0]];
+        }
+        $domain = strtolower((string)$job->domain);
+        $run = new RestoreRun($username, $domain, $this->mailbox($username) ?? [], $job->params, $data, $this->allowedDomains($job, $domain), $job->requestedBy);
+        $results = [];
+        if (in_array($username, array_map('strtolower', $job->createMailboxes()), true)) {
+            $created = ['added' => 0, 'skipped' => 0, 'errors' => [], 'created' => 0];
+            if (!empty($run->mailbox)) {
+                $created['skipped'] = 1;
+            } else {
+                $error = $this->createMailbox($run);
+                if ($error !== null) {
+                    $created['errors'][] = $error;
+                    return ['mailbox' => $created];
+                }
+                $created = ['added' => 1, 'skipped' => 0, 'errors' => [], 'created' => 1];
+                $run = new RestoreRun($username, $domain, $this->mailbox($username) ?? [], $job->params, $data, $run->domains, $job->requestedBy);
+                $run->created = true;
+                $run->sogo($username);
+                $run->auth($username);
+            }
+            $results['mailbox'] = $created;
+        }
+        if (empty($run->mailbox)) {
+            throw new UserError('backup_mailbox_not_found', [htmlspecialchars($username)]);
+        }
+        $components = array_values(array_intersect(self::COMPONENTS, $job->components()));
+        $this->repo->inTransaction(function () use ($run, $components): void {
+            foreach (array_diff($components, ['groups']) as $component) {
+                $run->start($component);
+                try {
+                    $this->repo->savepoint($component, fn() => $this->component($run, $component));
+                } catch (PDOException $e) {
+                    $run->failed($component, 'database error ' . $e->getCode());
+                }
+            }
+        });
+        if (in_array('groups', $components, true)) {
+            $run->start('groups');
+            $this->groups($run);
+        }
+        $this->sogo->refreshStaticView($run->sogoUsers);
+        $this->dovecot->flushAuthCache($run->authUsers);
+        return $results + $run->results;
+    }
+
+    /**
+     * Creates a mailbox deleted since the snapshot from its exported row, with the side effects of the legacy mailbox
+     * add (own alias, quota rows, user ACL, tags) and within the domain's limits
+     *
+     * @return ?string null on success, else why it was not created
+     */
+    private function createMailbox(RestoreRun $run): ?string
+    {
+        $backup = self::decode($run->data['mailbox']);
+        $username = $run->username;
+        [$local, $domain] = array_pad(explode('@', $username, 2), 2, '');
+        if (!filter_var($username, FILTER_VALIDATE_EMAIL) || $domain !== $run->domain) {
+            return 'invalid mailbox';
+        }
+        $domainRow = $this->repo->rows('domain', ['domain' => $domain])[0] ?? null;
+        if ($domainRow === null) {
+            return 'the domain does not exist';
+        }
+        $kind = $this->repo->addressKind($username);
+        // deleting a forwarding mailbox leaves its address as an alias to the forward targets: that one is taken back
+        $leftover = ($kind === 'alias') ? $this->leftoverAlias($run) : null;
+        if ($kind !== null && $leftover === null) {
+            return "the address is in use ($kind)";
+        }
+        $quota = max(0, (int)($backup['quota'] ?? 0));
+        if ((string)($backup['kind'] ?? '') === '') {
+            if ($this->repo->mailboxCount($domain) >= (int)($domainRow['mailboxes'] ?? 0)) {
+                return 'mailbox limit of the domain reached';
+            }
+            $error = $this->quotaError($run, $quota);
+            if ($error !== null) {
+                return "quota: $error";
+            }
+        }
+        $this->repo->inTransaction(function () use ($run, $backup, $username, $local, $domain, $leftover): void {
+            $this->repo->insert('mailbox', ['username' => $username, 'local_part' => $local, 'domain' => $domain]
+                + array_diff_key($backup, array_flip(['modified', 'tags'])));
+            $this->repo->resetQuota($username);
+            if ($leftover !== null) {
+                $this->repo->update('alias', ['id' => (int)$leftover['id']], ['goto' => $username, 'active' => (int)($backup['active'] ?? 1)]);
+            } elseif (empty($this->repo->rows('alias', ['address' => $username]))) {
+                $this->repo->insert('alias', ['address' => $username, 'goto' => $username, 'domain' => $domain,
+                    'active' => (int)($backup['active'] ?? 1)]);
+            }
+            // the user ACL of the backup (a domain admin's restore: defaults for what he may not grant), else the defaults
+            if (empty($this->repo->rows('user_acl', ['username' => $username]))) {
+                $acl = is_array($run->data['user_acl'] ?? null) ? self::decode($run->data['user_acl']) : [];
+                $this->repo->insert('user_acl', ['username' => $username] + array_diff_key($acl, array_flip($this->lockedUserAcl($run))));
+            }
+            foreach (array_unique(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '',
+                (array)($run->data['mailbox']['tags'] ?? [])), 'strlen')) as $tag) {
+                $this->repo->insert('tags_mailbox', ['username' => $username, 'tag_name' => $tag]);
+            }
+        });
+        return null;
+    }
+
+    /**
+     * A domain admin's restore stops when he lost what it was requested with: his account, the domain, or an ACL he had
+     * then (columns of da_acl)
+     */
+    private function permissionRevoked(Job $job): bool
+    {
+        if (($job->params['requested_role'] ?? '') === 'admin') {
+            return false;
+        }
+        $username = $job->requestedBy;
+        if (empty($this->repo->rows('admin', ['username' => $username, 'superadmin' => 0, 'active' => 1]))
+            || empty($this->repo->rows('domain_admins', ['username' => $username, 'domain' => strtolower($job->domain), 'active' => 1]))) {
+            return true;
+        }
+        $acl = $this->repo->rows('da_acl', ['username' => $username])[0] ?? [];
+        foreach ((array)($job->params['acl'] ?? []) as $name => $flag) {
+            if ((int)$flag === 1 && array_key_exists((string)$name, $acl) && (int)$acl[$name] !== 1) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * User ACL columns a domain admin may not grant: those of da_acl he does not have himself (legacy acl('get', 'user'));
+     * none for admins
+     *
+     * @return string[]
+     */
+    private function lockedUserAcl(RestoreRun $run): array
+    {
+        if ($run->byAdmin()) {
+            return [];
+        }
+        return array_values(array_filter(array_diff($this->repo->columns('da_acl'), ['username']), fn(string $column): bool => !$run->may($column)));
+    }
+
+    /** @throws UserError backup_job_busy while another request merges the same mailbox of the job */
+    private function locked(Job $job, string $username, callable $work): array
+    {
+        $result = $this->repo->locked($job->id . ':' . $username, $work);
+        return is_array($result) ? $result : throw new UserError('backup_job_busy');
+    }
+
+    /** @throws UserError backup_job_not_running */
+    private function runningRestore(mixed $jobId): Job
+    {
+        $id = Input::intInRange($jobId, 1, PHP_INT_MAX);
+        $job = ($id === null) ? null : $this->jobs->find($id);
+        if ($job === null || $job->type !== 'restore' || $job->state !== 'running') {
+            throw new UserError('backup_job_not_running');
+        }
+        return $job;
+    }
+
+    /** @return string[] domains whose objects the restore may change: [] = all (admins), else the job's domain and its alias domains */
+    private function allowedDomains(Job $job, string $domain): array
+    {
+        return (($job->params['requested_role'] ?? '') === 'admin') ? [] : array_values(array_unique(array_merge([$domain], $this->repo->aliasDomainsOf($domain))));
+    }
+
+    private function component(RestoreRun $run, string $component): void
+    {
+        match ($component) {
+            'aliases' => $this->aliases($run, []),
+            'sender_acl' => $this->senderAcl($run),
+            'mail_policies' => $this->policies($run, 'mail_policies', 'mail_policies', 'mail_policy', 'mail_policy_member', 'policy_id'),
+            'retention_rules' => $this->policies($run, 'retention_rules', 'retention', 'retention_rule', 'retention_rule_member', 'rule_id'),
+            'filters' => $this->filters($run),
+            'spam' => $this->spam($run),
+            'bcc' => $this->bcc($run),
+            'syncjobs' => $this->syncJobs($run),
+            'app_passwords' => $this->appPasswords($run),
+            'tfa' => $this->tfa($run),
+            'settings' => $this->settings($run),
+            'password' => $this->password($run),
+        };
+    }
+
+    /**
+     * Aliases whose goto named the mailbox, or a part of the domain's aliases: a missing alias is created with its gotos
+     * that are valid now, an existing one gets the mailbox back into its goto. Group addresses belong to their group
+     * and are left alone.
+     */
+    private function aliases(RestoreRun $run, array $domainAliases): void
+    {
+        if ($run->created) {
+            $this->ownAlias($run);
+        }
+        $seen = [];
+        $created = array_map('strtolower', (array)($run->params['create_mailboxes'] ?? []));
+        foreach (array_merge($domainAliases, $run->rows('aliases')) as $row) {
+            $row = self::decode($row);
+            $address = strtolower(trim((string)($row['address'] ?? '')));
+            $gotos = ExportService::gotos(strtolower((string)($row['goto'] ?? '')));
+            // the own alias of a mailbox comes with the mailbox (ownAlias())
+            if ($address === '' || isset($seen[$address]) || $gotos === [$address] || in_array($address, $created, true)) {
+                continue;
+            }
+            $seen[$address] = true;
+            if (($row['group_id'] ?? null) !== null) {
+                $run->skipped('aliases');
+                continue;
+            }
+            $domain = self::domainOf($address);
+            if (!$run->mayChangeDomain($domain)) {
+                $run->error('aliases', "$address: outside the domain of the restore");
+                continue;
+            }
+            $current = $this->repo->rows('alias', ['address' => $address])[0] ?? null;
+            if ($current !== null) {
+                $this->aliasGoto($run, $current, $gotos);
+                continue;
+            }
+            // naming itself: the own alias of a deleted mailbox (forwarding with a copy), as an alias it would block the mailbox
+            if (in_array($address, $gotos, true)) {
+                $run->skipped('aliases');
+                continue;
+            }
+            $this->createAlias($run, $row, $address, $domain, $gotos);
+        }
+    }
+
+    /**
+     * The alias at the address of a deleted mailbox when it is what the deletion left of the mailbox's forwarding: the
+     * forward targets of the backup's own alias, without the mailbox. Any other alias (e.g. one added since) blocks.
+     */
+    private function leftoverAlias(RestoreRun $run): ?array
+    {
+        $backup = is_array($run->data['own_alias'] ?? null) ? self::decode($run->data['own_alias']) : [];
+        $forward = array_unique(array_diff(ExportService::gotos(strtolower((string)($backup['goto'] ?? ''))), [$run->username]));
+        $current = $this->repo->rows('alias', ['address' => $run->username])[0] ?? null;
+        if ($current === null || empty($forward) || ($current['group_id'] ?? null) !== null) {
+            return null;
+        }
+        $now = array_unique(ExportService::gotos(strtolower((string)$current['goto'])));
+        sort($forward);
+        sort($now);
+        return ($now === $forward) ? $current : null;
+    }
+
+    /**
+     * The own alias of a mailbox the restore created gets the forwarding of the backup: the gotos that are valid now, with
+     * the mailbox itself when it kept a copy. Without a valid one it keeps delivering to the mailbox.
+     */
+    private function ownAlias(RestoreRun $run): void
+    {
+        $backup = is_array($run->data['own_alias'] ?? null) ? self::decode($run->data['own_alias']) : [];
+        $gotos = ExportService::gotos(strtolower((string)($backup['goto'] ?? '')));
+        $forward = array_values(array_diff($gotos, [$run->username]));
+        $current = $this->repo->rows('alias', ['address' => $run->username])[0] ?? null;
+        if (empty($forward) || $current === null) {
+            return;
+        }
+        $valid = $this->validGotos($run, $run->username, $forward);
+        if (empty($valid)) {
+            return;
+        }
+        $goto = in_array($run->username, $gotos, true) ? array_merge([$run->username], $valid) : $valid;
+        $this->repo->update('alias', ['id' => (int)$current['id']], ['goto' => implode(',', $goto)]);
+        $run->added('aliases');
+    }
+
+    /** The mailbox goes back into the goto of an existing alias (never into a group address or null/spam/ham) */
+    private function aliasGoto(RestoreRun $run, array $current, array $backupGotos): void
+    {
+        $gotos = ExportService::gotos((string)$current['goto']);
+        $lower = array_map('strtolower', $gotos);
+        if ($run->username === '' || !in_array($run->username, $backupGotos, true) || in_array($run->username, $lower, true) || ($current['group_id'] ?? null) !== null
+            || !empty(array_intersect($lower, self::SPECIAL_GOTOS))) {
+            $run->skipped('aliases');
+            return;
+        }
+        $gotos[] = $run->username;
+        $this->repo->update('alias', ['id' => (int)$current['id']], ['goto' => implode(',', $gotos)]);
+        $run->added('aliases');
+        $run->sogo($run->username);
+    }
+
+    /** Creates a missing alias with the checks of the legacy alias form */
+    private function createAlias(RestoreRun $run, array $row, string $address, string $domain, array $backupGotos): void
+    {
+        if (!self::validAddress($address)) {
+            $run->error('aliases', "$address: invalid address");
+            return;
+        }
+        $primary = $this->repo->primaryDomain($domain);
+        if ($primary === null) {
+            $run->error('aliases', "$address: domain does not exist");
+            return;
+        }
+        $kind = str_starts_with($address, '@') ? null : $this->repo->addressKind($address);
+        if ($kind !== null) {
+            $run->error('aliases', "$address: address is in use ($kind)");
+            return;
+        }
+        $this->aliasesLeft[$primary] ??= $this->repo->aliasesLeft($primary);
+        if ($this->aliasesLeft[$primary] <= 0) {
+            $run->error('aliases', "$address: alias limit of the domain reached");
+            return;
+        }
+        $gotos = $this->validGotos($run, $address, $backupGotos);
+        if (empty($gotos)) {
+            $run->error('aliases', "$address: no valid goto");
+            return;
+        }
+        $values = array_diff_key($row, array_flip([...self::NEW_ROW, 'group_id']));
+        $this->repo->insert('alias', ['address' => $address, 'goto' => implode(',', $gotos), 'domain' => $domain] + $values);
+        $this->aliasesLeft[$primary]--;
+        $run->added('aliases');
+        $run->sogo(...array_filter($gotos, fn(string $goto): bool => $this->repo->addressKind($goto) === 'mailbox'));
+    }
+
+    /**
+     * Gotos of a new alias that are valid now: null/spam/ham alone, local addresses that exist or that the restore creates
+     * (no resources), external
+     * ones unless external gotos are disabled (globally or by the domain admin's ACL). Dropped ones are reported.
+     *
+     * @return string[]
+     */
+    private function validGotos(RestoreRun $run, string $address, array $gotos): array
+    {
+        $special = array_intersect($gotos, self::SPECIAL_GOTOS);
+        if (!empty($special)) {
+            return (count($gotos) === 1) ? $gotos : [];
+        }
+        $valid = [];
+        foreach (array_unique($gotos) as $goto) {
+            if ($goto === $address || !filter_var($goto, FILTER_VALIDATE_EMAIL)) {
+                $run->error('aliases', "$address: goto $goto dropped (invalid)");
+                continue;
+            }
+            if ($this->repo->isLocalDomain(self::domainOf($goto))) {
+                // mailboxes the restore creates count as existing (the domain's aliases come before its mailboxes)
+                if (!in_array($this->repo->addressKind($goto), ['mailbox', 'alias'], true)
+                    && !in_array($goto, array_map('strtolower', (array)($run->params['create_mailboxes'] ?? [])), true)) {
+                    $run->error('aliases', "$address: goto $goto dropped (no mailbox or alias)");
+                    continue;
+                }
+            } elseif ($this->externalGotoDisabled || !$run->may('alias_external_goto')) {
+                $run->error('aliases', "$address: goto $goto dropped (external gotos are not allowed)");
+                continue;
+            }
+            $valid[] = $goto;
+        }
+        return $valid;
+    }
+
+    /**
+     * Send-as rights of the mailbox and rights of others to send as it, for addresses and mailboxes that exist now.
+     * Rows of groups are rebuilt by the group module and left out.
+     */
+    private function senderAcl(RestoreRun $run): void
+    {
+        foreach ($run->rows('sender_acl') as $row) {
+            if (($row['group_id'] ?? null) !== null) {
+                continue;
+            }
+            $sendAs = strtolower(trim((string)($row['send_as'] ?? '')));
+            $external = (int)($row['external'] ?? 0);
+            $error = $this->sendAsError($run, $sendAs, $external);
+            if ($error !== null) {
+                $run->error('sender_acl', "$sendAs: $error");
+            } elseif (!empty($this->repo->rows('sender_acl', ['logged_in_as' => $run->username, 'send_as' => $sendAs]))) {
+                $run->skipped('sender_acl');
+            } else {
+                $this->repo->insert('sender_acl', ['logged_in_as' => $run->username, 'send_as' => $sendAs, 'external' => $external]);
+                $run->added('sender_acl');
+                $run->sogo($run->username);
+            }
+        }
+        foreach ($run->rows('sender_acl_granted') as $row) {
+            if (($row['group_id'] ?? null) !== null) {
+                continue;
+            }
+            $login = strtolower(trim((string)($row['logged_in_as'] ?? '')));
+            if (empty($this->repo->rows('mailbox', ['username' => $login]))) {
+                $run->error('sender_acl', "$login: mailbox does not exist");
+            } elseif (!$run->mayChangeDomain(self::domainOf($login))) {
+                $run->error('sender_acl', "$login: outside the domain of the restore");
+            } elseif (!empty($this->repo->rows('sender_acl', ['logged_in_as' => $login, 'send_as' => $run->username]))) {
+                $run->skipped('sender_acl');
+            } else {
+                $this->repo->insert('sender_acl', ['logged_in_as' => $login, 'send_as' => $run->username, 'external' => 0]);
+                $run->added('sender_acl');
+                $run->sogo($login);
+            }
+        }
+    }
+
+    /** Why the mailbox may not send as this address (anymore), null when it may */
+    private function sendAsError(RestoreRun $run, string $sendAs, int $external): ?string
+    {
+        if ($sendAs === '*') {
+            return $run->byAdmin() ? null : 'only admins grant *';
+        }
+        $domain = self::domainOf($sendAs);
+        if (!self::validAddress($sendAs)) {
+            return 'invalid address';
+        }
+        if ($external === 1) {
+            if (!$run->may('extend_sender_acl')) {
+                return 'not allowed';
+            }
+            return $this->repo->domainExists($domain) ? 'domain is local now' : null;
+        }
+        if (!$run->mayChangeDomain($domain)) {
+            return 'outside the domain of the restore';
+        }
+        if (str_starts_with($sendAs, '@')) {
+            return $this->repo->domainExists($domain) ? null : 'domain does not exist';
+        }
+        return in_array($this->repo->addressKind($sendAs), ['mailbox', 'alias'], true) ? null : 'address does not exist';
+    }
+
+    /**
+     * Assignments to mail policies or retention rules of the mailbox's domain that exist now (by id, else by name)
+     *
+     * @param string $key    the list in the backup and the component
+     * @param string $acl    the domain admin ACL it needs
+     * @param string $member the member table and its $column with the id
+     */
+    private function policies(RestoreRun $run, string $key, string $acl, string $table, string $member, string $column): void
+    {
+        $entries = $run->rows($key);
+        if (!empty($entries) && !$run->may($acl)) {
+            $run->error($key, 'not allowed');
+            return;
+        }
+        foreach ($entries as $entry) {
+            $name = (string)($entry['name'] ?? '');
+            $target = $this->repo->rows($table, ['id' => (int)($entry['id'] ?? 0), 'domain' => $run->domain])[0]
+                ?? $this->repo->rows($table, ['domain' => $run->domain, 'name' => $name])[0] ?? null;
+            if ($target === null) {
+                $run->error($key, "$name: does not exist");
+            } elseif (!empty($this->repo->rows($member, [$column => (int)$target['id'], 'username' => $run->username]))) {
+                $run->skipped($key);
+            } else {
+                $this->repo->insert($member, [$column => (int)$target['id'], 'username' => $run->username]);
+                $run->added($key);
+            }
+        }
+    }
+
+    /**
+     * mailcow filters (sieve_filters) missing with the same type and script; a restored filter is active only if it
+     * was and the mailbox has no active filter of its type now.
+     */
+    private function filters(RestoreRun $run): void
+    {
+        foreach ($run->rows('sieve_filters') as $row) {
+            $type = (string)($row['filter_type'] ?? '');
+            $script = (string)($row['script_data'] ?? '');
+            $description = (string)($row['script_desc'] ?? '');
+            if (!in_array($type, ['prefilter', 'postfilter'], true) || trim($script) === '') {
+                $run->error('filters', "$description: invalid filter");
+                continue;
+            }
+            $current = $this->repo->rows('sieve_filters', ['username' => $run->username, 'filter_type' => $type]);
+            if (!empty(array_filter($current, fn(array $filter): bool => $filter['script_data'] === $script))) {
+                $run->skipped('filters');
+                continue;
+            }
+            $active = ($row['script_name'] ?? '') === 'active' && empty(array_filter($current, fn(array $filter): bool => $filter['script_name'] === 'active'));
+            $this->repo->insert('sieve_filters', ['username' => $run->username, 'script_name' => $active ? 'active' : 'inactive']
+                + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('filters');
+        }
+    }
+
+    /**
+     * Spam settings (filterconf): missing allow/deny list entries and spam score levels (an existing level stays);
+     * spam aliases that are still valid, whose address is free and in a domain of the mailbox.
+     */
+    private function spam(RestoreRun $run): void
+    {
+        foreach ($run->rows('filterconf') as $row) {
+            $option = (string)($row['option'] ?? '');
+            $value = (string)($row['value'] ?? '');
+            $current = $this->repo->rows('filterconf', ['object' => $run->username]);
+            $present = match ($option) {
+                'whitelist_from', 'blacklist_from' => array_filter($current, fn(array $c): bool => in_array($c['option'], ['whitelist_from', 'blacklist_from'], true)
+                    && strtolower((string)$c['value']) === strtolower($value)),
+                'lowspamlevel', 'highspamlevel' => array_filter($current, fn(array $c): bool => $c['option'] === $option),
+                default => array_filter($current, fn(array $c): bool => $c['option'] === $option && (string)$c['value'] === $value),
+            };
+            if ($option === '' || !empty($present)) {
+                $run->skipped('spam');
+                continue;
+            }
+            $this->repo->insert('filterconf', ['object' => $run->username, 'option' => $option, 'value' => $value]);
+            $run->added('spam');
+        }
+        foreach ($run->rows('spamalias') as $row) {
+            $address = strtolower(trim((string)($row['address'] ?? '')));
+            if ((int)($row['permanent'] ?? 0) !== 1 && (int)($row['validity'] ?? 0) < time()) {
+                $run->skipped('spam');
+                continue;
+            }
+            $current = $this->repo->rows('spamalias', ['address' => $address])[0] ?? null;
+            if ($current !== null && strtolower((string)$current['goto']) === $run->username) {
+                $run->skipped('spam');
+                continue;
+            }
+            // like the legacy form: spam aliases in the mailbox's domain and its alias domains only
+            if (!filter_var($address, FILTER_VALIDATE_EMAIL) || !in_array(self::domainOf($address),
+                array_merge([$run->domain], $this->repo->aliasDomainsOf($run->domain)), true)) {
+                $run->error('spam', "$address: not a domain of the mailbox");
+                continue;
+            }
+            $kind = $this->repo->addressKind($address);
+            if ($kind !== null) {
+                $run->error('spam', "$address: address is in use ($kind)");
+                continue;
+            }
+            $this->repo->insert('spamalias', ['address' => $address, 'goto' => $run->username] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('spam');
+        }
+    }
+
+    /** BCC maps of the mailbox: one per type */
+    private function bcc(RestoreRun $run): void
+    {
+        foreach ($run->rows('bcc_maps') as $row) {
+            $type = (string)($row['type'] ?? '');
+            $destination = (string)($row['bcc_dest'] ?? '');
+            if (!in_array($type, ['sender', 'rcpt'], true) || !filter_var($destination, FILTER_VALIDATE_EMAIL)) {
+                $run->error('bcc', "$destination: invalid BCC map");
+                continue;
+            }
+            if (!empty($this->repo->rows('bcc_maps', ['local_dest' => $run->username, 'type' => $type]))) {
+                $run->skipped('bcc');
+                continue;
+            }
+            $this->repo->insert('bcc_maps', ['local_dest' => $run->username, 'domain' => $run->domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('bcc');
+        }
+    }
+
+    /**
+     * Sync jobs missing by user1, host and port. The source must exist now (same id with that host and port, else
+     * another source the mailbox may use). Restored jobs are inactive: an old migration must not start on its own.
+     */
+    private function syncJobs(RestoreRun $run): void
+    {
+        $current = array_map(fn(array $job): string => self::syncKey((string)$job['user1'], (string)$job['host1'], (int)$job['port1']),
+            $this->repo->syncJobsOf($run->username));
+        foreach ($run->rows('imapsync') as $row) {
+            $row = self::decode($row);
+            $user1 = (string)($row['user1'] ?? '');
+            $host = (string)($row['source_host1'] ?? '');
+            $port = (int)($row['source_port1'] ?? 0);
+            if (in_array(self::syncKey($user1, $host, $port), $current, true)) {
+                $run->skipped('syncjobs');
+                continue;
+            }
+            $source = $this->repo->rows('imapsync_source', ['id' => (int)($row['source_id'] ?? 0)])[0] ?? null;
+            $sourceId = ($source !== null && strtolower((string)$source['host1']) === strtolower($host) && (int)$source['port1'] === $port)
+                ? (int)$source['id'] : ($this->repo->syncSourcesFor($host, $port, $run->username, $run->domain)[0] ?? null);
+            if ($sourceId === null || $user1 === '') {
+                $run->error('syncjobs', "$user1 ($host:$port): sync source does not exist");
+                continue;
+            }
+            $values = array_diff_key($row, array_flip([...self::NEW_ROW, 'is_running', 'returned_text', 'last_run', 'success', 'exit_status', 'prio']));
+            $this->repo->insert('imapsync', ['user2' => $run->username, 'source_id' => $sourceId, 'active' => 0] + $values);
+            $current[] = self::syncKey($user1, $host, $port);
+            $run->added('syncjobs');
+        }
+    }
+
+    /** App passwords missing by name */
+    private function appPasswords(RestoreRun $run): void
+    {
+        foreach ($run->rows('app_passwd') as $row) {
+            $name = (string)($row['name'] ?? '');
+            if ($name === '' || (string)($row['password'] ?? '') === '') {
+                $run->error('app_passwords', "$name: invalid app password");
+                continue;
+            }
+            if (!empty($this->repo->rows('app_passwd', ['mailbox' => $run->username, 'name' => $name]))) {
+                $run->skipped('app_passwords');
+                continue;
+            }
+            $this->repo->insert('app_passwd', ['mailbox' => $run->username, 'domain' => $run->domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('app_passwords');
+            $run->auth($run->username);
+        }
+    }
+
+    /**
+     * 2FA entries missing (same key_id, keyHandle or secret = present) and FIDO2 keys of the passwordless login
+     * missing by credential id.
+     */
+    private function tfa(RestoreRun $run): void
+    {
+        foreach ($run->rows('tfa') as $row) {
+            $current = $this->repo->rows('tfa', ['username' => $run->username]);
+            $present = array_filter($current, fn(array $tfa): bool => (string)$tfa['key_id'] === (string)($row['key_id'] ?? '')
+                || (($row['keyHandle'] ?? '') !== '' && ($row['keyHandle'] ?? null) !== null && $tfa['keyHandle'] === $row['keyHandle'])
+                || (($row['secret'] ?? '') !== '' && ($row['secret'] ?? null) !== null && $tfa['secret'] === $row['secret']));
+            if (!empty($present)) {
+                $run->skipped('tfa');
+                continue;
+            }
+            $this->repo->insert('tfa', ['username' => $run->username] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('tfa');
+        }
+        foreach ($run->rows('fido2') as $row) {
+            $row = self::decode($row);
+            $credentialId = (string)($row['credentialId'] ?? '');
+            if ($credentialId === '') {
+                $run->error('tfa', 'FIDO2 key without credential id');
+                continue;
+            }
+            $present = array_filter($this->repo->rows('fido2', ['username' => $run->username]), fn(array $key): bool => $key['credentialId'] === $credentialId);
+            if (!empty($present)) {
+                $run->skipped('tfa');
+                continue;
+            }
+            $this->repo->insert('fido2', ['username' => $run->username] + array_diff_key($row, array_flip(self::NEW_ROW)));
+            $run->added('tfa');
+        }
+    }
+
+    /**
+     * Overwrites quota (within the domain's limits), attributes (protocols, relayhost, ...), custom attributes, user
+     * ACL, Pushover settings and the rate limit; adds missing tags. Counts changed values as added, equal ones as skipped.
+     */
+    private function settings(RestoreRun $run): void
+    {
+        $backup = self::decode($run->data['mailbox']);
+        $mailbox = $this->mailbox($run->username) ?? $run->mailbox;
+        $changes = [];
+        if (isset($backup['quota']) && is_numeric($backup['quota'])) {
+            $quota = (int)$backup['quota'];
+            $error = ($quota === (int)$mailbox['quota']) ? null : $this->quotaError($run, $quota);
+            if ($error !== null) {
+                $run->error('settings', "quota: $error");
+            } elseif ($quota === (int)$mailbox['quota']) {
+                $run->skipped('settings');
+            } else {
+                $changes['quota'] = $quota;
+                $run->added('settings');
+            }
+        }
+        $attributes = self::json($mailbox['attributes'] ?? null);
+        $changed = false;
+        foreach (self::json($backup['attributes'] ?? null) as $key => $value) {
+            if (in_array($key, self::KEEP_ATTRIBUTES, true)) {
+                continue;
+            }
+            if (array_key_exists($key, $attributes) && $attributes[$key] === $value) {
+                $run->skipped('settings');
+                continue;
+            }
+            if (isset(self::ATTRIBUTE_ACL[$key]) && !$run->may(self::ATTRIBUTE_ACL[$key])) {
+                $run->error('settings', "$key: not allowed");
+                continue;
+            }
+            if ($key === 'relayhost' && (string)$value !== '0' && (string)$value !== '' && empty($this->repo->rows('relayhosts', ['id' => (int)$value]))) {
+                $run->error('settings', 'relayhost: does not exist');
+                continue;
+            }
+            $attributes[$key] = $value;
+            $changed = true;
+            $run->added('settings');
+        }
+        if ($changed) {
+            $changes['attributes'] = self::jsonText($attributes);
+        }
+        if (array_key_exists('custom_attributes', $backup)) {
+            $custom = self::json($mailbox['custom_attributes'] ?? null);
+            $restored = array_merge($custom, self::json($backup['custom_attributes']));
+            if ($restored !== $custom) {
+                $changes['custom_attributes'] = self::jsonText($restored);
+                $run->added('settings');
+            }
+        }
+        if (!empty($changes)) {
+            $this->repo->update('mailbox', ['username' => $run->username], $changes);
+            $run->auth($run->username);
+            $run->sogo($run->username);
+        }
+        $this->tags($run, (array)($run->data['mailbox']['tags'] ?? []));
+        if (is_array($run->data['user_acl'] ?? null)) {
+            $this->overwriteRow($run, 'user_acl', self::decode($run->data['user_acl']));
+        }
+        if (is_array($run->data['pushover'] ?? null)) {
+            if ($run->may('pushover')) {
+                $this->overwriteRow($run, 'pushover', self::decode($run->data['pushover']));
+            } else {
+                $run->error('settings', 'pushover: not allowed');
+            }
+        }
+        $rateLimit = $run->data['ratelimit'] ?? null;
+        if (is_string($rateLimit) && preg_match('/^\d+ \/ 1[smhd]$/', $rateLimit)) {
+            if (($this->legacy->rateLimits([$run->username])[$run->username] ?? null) === $rateLimit) {
+                $run->skipped('settings');
+            } elseif ($run->may('ratelimit')) {
+                $this->legacy->setRateLimit($run->username, $rateLimit);
+                $run->added('settings');
+            } else {
+                $run->error('settings', 'rate limit: not allowed');
+            }
+        }
+    }
+
+    /** Why the quota cannot be restored, null when it can (domain limits like the legacy mailbox form) */
+    private function quotaError(RestoreRun $run, int $quota): ?string
+    {
+        if ($quota < 0) {
+            return 'invalid';
+        }
+        if ($quota === 0) {
+            return $run->may('unlimited_quota') ? null : 'unlimited quota not allowed';
+        }
+        $domain = $this->repo->rows('domain', ['domain' => $run->domain])[0] ?? null;
+        if ($domain === null) {
+            return 'domain does not exist';
+        }
+        if ($quota > (int)$domain['maxquota'] * self::MIB) {
+            return 'exceeds the maximum quota of a mailbox in the domain';
+        }
+        if ($quota > (int)$domain['quota'] * self::MIB - $this->repo->quotaInUse($run->domain, $run->username)) {
+            return 'exceeds the quota left in the domain';
+        }
+        return null;
+    }
+
+    /** @param array $tags tags of the backup: missing ones are added */
+    private function tags(RestoreRun $run, array $tags): void
+    {
+        $current = array_map(fn(array $row): string => (string)$row['tag_name'], $this->repo->rows('tags_mailbox', ['username' => $run->username]));
+        foreach (array_unique(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen')) as $tag) {
+            if (in_array($tag, $current, true)) {
+                $run->skipped('settings');
+                continue;
+            }
+            $this->repo->insert('tags_mailbox', ['username' => $run->username, 'tag_name' => $tag]);
+            $run->added('settings');
+        }
+    }
+
+    /** Overwrites the mailbox's row of user_acl or pushover with the backup's values (creates it if missing) */
+    private function overwriteRow(RestoreRun $run, string $table, array $backup): void
+    {
+        $values = array_diff_key($backup, ['username' => true]);
+        $current = $this->repo->rows($table, ['username' => $run->username])[0] ?? null;
+        if ($table === 'user_acl') {
+            // a domain admin's restore keeps what he may not grant
+            foreach ($this->lockedUserAcl($run) as $column) {
+                if (array_key_exists($column, $values) && $current !== null && array_key_exists($column, $current) && (string)$current[$column] !== (string)$values[$column]) {
+                    $run->error('settings', "user ACL $column: not allowed");
+                }
+                unset($values[$column]);
+            }
+        }
+        if ($current === null) {
+            $this->repo->insert($table, ['username' => $run->username] + $values);
+            $run->added('settings');
+            return;
+        }
+        $changed = array_filter($values, fn(mixed $value, string $column): bool => array_key_exists($column, $current)
+            && (string)$current[$column] !== (string)$value, ARRAY_FILTER_USE_BOTH);
+        $run->skipped('settings', count(array_intersect_key($values, $current)) - count($changed));
+        if (!empty($changed)) {
+            $this->repo->update($table, ['username' => $run->username], $changed);
+            $run->added('settings', count($changed));
+        }
+    }
+
+    /** Overwrites the password hash (mailboxes that log in with mailcow passwords only) */
+    private function password(RestoreRun $run): void
+    {
+        $backup = $run->data['mailbox'];
+        $hash = (string)($backup['password'] ?? '');
+        $mailbox = $this->mailbox($run->username) ?? $run->mailbox;
+        if ($hash === '') {
+            $run->error('password', 'no password in the backup');
+        } elseif (($mailbox['authsource'] ?? 'mailcow') !== 'mailcow') {
+            $run->error('password', 'the mailbox logs in through an identity provider');
+        } elseif (hash_equals((string)$mailbox['password'], $hash)) {
+            $run->skipped('password');
+        } else {
+            $attributes = self::json($mailbox['attributes'] ?? null);
+            $attributes['passwd_update'] = self::json($backup['attributes'] ?? null)['passwd_update'] ?? date('Y-m-d H:i:s');
+            $this->repo->update('mailbox', ['username' => $run->username], ['password' => $hash, 'attributes' => self::jsonText($attributes)]);
+            $run->added('password');
+            $run->auth($run->username);
+        }
+    }
+
+    /**
+     * Manual memberships in groups of the mailbox's domain that exist now (by id, else by name), through the group
+     * module as the requester (a domain admin with his rights now: groups read-only for him are not allowed).
+     * IdP memberships are the IdP sync's.
+     */
+    private function groups(RestoreRun $run): void
+    {
+        $context = $run->requester();
+        foreach ($run->rows('groups') as $entry) {
+            $name = (string)($entry['name'] ?? '');
+            if (($entry['source'] ?? 'manual') === 'idp') {
+                $run->skipped('groups');
+                continue;
+            }
+            $group = $this->repo->rows('mailbox_group', ['id' => (int)($entry['id'] ?? 0), 'domain' => $run->domain])[0]
+                ?? $this->repo->rows('mailbox_group', ['domain' => $run->domain, 'name' => $name])[0] ?? null;
+            if ($group === null) {
+                $run->error('groups', "$name: does not exist");
+                continue;
+            }
+            if (!empty($this->repo->rows('mailbox_group_member', ['group_id' => (int)$group['id'], 'username' => $run->username]))) {
+                $run->skipped('groups');
+                continue;
+            }
+            try {
+                $this->groups->add($context, (int)$group['id'], $run->username);
+                $run->added('groups');
+            } catch (AccessDenied) {
+                $run->error('groups', "$name: not allowed");
+            } catch (UserError $e) {
+                $run->error('groups', "$name: " . $e->key);
+            } catch (PDOException $e) {
+                $run->error('groups', "$name: database error " . $e->getCode());
+            }
+        }
+    }
+
+    private function mailbox(string $username): ?array
+    {
+        return $this->repo->rows('mailbox', ['username' => $username])[0] ?? null;
+    }
+
+    private static function syncKey(string $user1, string $host, int $port): string
+    {
+        return strtolower($user1) . '|' . strtolower($host) . '|' . $port;
+    }
+
+    /** user@domain or @domain */
+    private static function validAddress(string $address): bool
+    {
+        return str_starts_with($address, '@') ? (bool)preg_match('/^@[a-z0-9.-]+\.[a-z0-9-]+$/i', $address)
+            : filter_var($address, FILTER_VALIDATE_EMAIL) !== false;
+    }
+
+    private static function domainOf(string $address): string
+    {
+        return strtolower(substr((string)strrchr($address, '@'), 1));
+    }
+
+    /** Values of a backup row: {"base64": ...} back to binary, other lists as JSON text */
+    public static function decode(array $row): array
+    {
+        foreach ($row as $column => $value) {
+            if (is_array($value)) {
+                $row[$column] = (array_keys($value) === ['base64']) ? base64_decode((string)$value['base64'], true) : json_encode($value);
+            }
+        }
+        return $row;
+    }
+
+    /** A JSON column as array ({} when empty or broken) */
+    private static function json(mixed $value): array
+    {
+        $decoded = is_string($value) ? json_decode($value, true) : $value;
+        return is_array($decoded) ? $decoded : [];
+    }
+
+    private static function jsonText(array $value): string
+    {
+        return (string)json_encode($value ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+    }
+}

data/web/inc/lib/mailcow/Backup/Service/SettingsService.php

@@ -0,0 +1,327 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Backup\Service;
+
+use Mailcow\Backup\Model\Settings;
+use Mailcow\Backup\Repository\SettingsRepository;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+
+/**
+ * Backup settings: admins read them (also read-only API keys) and save them. Values are checked strictly (at least as
+ * strictly as the runner), since the runner passes them to restic and ssh. Secrets are write-only and stored sealed
+ * with the public key the runner published (only the runner can decrypt them); an empty secret keeps the stored one,
+ * and without a published key no secret can be saved. A changed target, connection or repository password resets
+ * state_initialized (the runner tests the repository again); a changed SFTP host or port drops the known host key. A
+ * new repository password needs a new confirmation that it is stored outside mailcow. A change of a configured target,
+ * its connection or the notify addresses leaves state_target_changed for the runner, which mails the notify addresses
+ * of before the change.
+ */
+class SettingsService
+{
+    private const FLAGS = ['enabled', 's3_path_style', 'password_confirmed', 'include_index', 'include_fts'];
+    /** Integer settings: name => [min, max] */
+    private const RANGES = [
+        'sftp_port' => [1, 65535],
+        'keep_hourly' => [0, 1000],
+        'keep_daily' => [0, 1000],
+        'keep_weekly' => [0, 1000],
+        'keep_monthly' => [0, 1000],
+        'keep_yearly' => [0, 1000],
+        'prune_weekday' => [0, 6],
+        'check_subset' => [0, 365],
+        'limit_upload' => [0, 100000000],
+    ];
+    private const MIN_PASSWORD = 8;
+    private const MAX_LENGTH = 255;
+    private const MAX_NOTIFY = 20;
+    /** Entries of the schedule */
+    private const MAX_SCHEDULE = 24;
+    private const HOSTNAME = '/^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i';
+
+    public function __construct(private SettingsRepository $repo, private Context $context)
+    {
+    }
+
+    /** The settings for admins, null for everybody else */
+    public function get(): ?Settings
+    {
+        return $this->context->hasAdminRights() ? $this->load() : null;
+    }
+
+    /** The settings without permission check (status, job rules) */
+    public function load(): Settings
+    {
+        return Settings::fromStored($this->repo->load());
+    }
+
+    /**
+     * Saves the given settings; missing ones keep their value, unknown and runner-owned ones are ignored.
+     *
+     * @throws UserError
+     */
+    public function save(array $input): Settings
+    {
+        $this->context->requireAdmin();
+        $current = $this->load();
+        $values = $current->values;
+        foreach ($input as $name => $value) {
+            if (!is_string($name) || !array_key_exists($name, Settings::DEFAULTS) || in_array($name, Settings::RUNNER, true)) {
+                continue;
+            }
+            $value = $this->normalize($name, $value);
+            if (in_array($name, Settings::SECRETS, true)) {
+                if ($value === '') {
+                    continue;
+                }
+                $value = self::seal($value, $current->get('state_secrets_pubkey'));
+            }
+            $values[$name] = $value;
+        }
+        if ($values['password'] !== $current->get('password')
+            && !(array_key_exists('password_confirmed', $input) && $values['password_confirmed'] === '1')) {
+            $values['password_confirmed'] = '0';
+        }
+        foreach (array_merge(['target', 'password'], Settings::CONNECTION[$values['target']]) as $name) {
+            if ($values[$name] !== $current->get($name)) {
+                $values['state_initialized'] = '0';
+                break;
+            }
+        }
+        if ($values['sftp_host'] !== $current->get('sftp_host') || $values['sftp_port'] !== $current->get('sftp_port')) {
+            $values['sftp_host_key'] = '';
+        }
+        $settings = new Settings($values);
+        if ($values['enabled'] === '1' && !$settings->configured()) {
+            throw new UserError('backup_not_configured');
+        }
+        // switched on without a time
+        if ($values['enabled'] === '1' && empty($settings->scheduleSlots())) {
+            throw self::invalid('schedule');
+        }
+        $values['state_target_changed'] = $this->targetChange($current, $values);
+        $settings = new Settings($values);
+        $changed = array_diff_assoc($values, $current->values);
+        if (!empty($changed)) {
+            $this->repo->save($changed);
+        }
+        return $settings;
+    }
+
+    /**
+     * state_target_changed after a save: when the target, its connection or the notify addresses change, what the target
+     * was, whom to tell (the notify addresses before the change) and what changed ("target", "notify_email"). An earlier
+     * change the runner has not mailed yet keeps its state and addresses and only gets the new kinds of change, so new
+     * addresses followed by a new target still go to the original addresses. Nothing is recorded for a target that was
+     * not configured or without notify addresses.
+     */
+    private function targetChange(Settings $current, array $values): string
+    {
+        $pending = json_decode($current->get('state_target_changed'), true);
+        if (!$current->configured() && !is_array($pending)) {
+            return $current->get('state_target_changed');
+        }
+        $old = $current->get('target');
+        $watched = array_unique(array_merge(['target'], Settings::CONNECTION[$old] ?? [], Settings::CONNECTION[$values['target']] ?? []));
+        $changed = [];
+        if (!empty(array_filter($watched, fn(string $name): bool => $values[$name] !== $current->get($name)))) {
+            $changed[] = 'target';
+        }
+        if ($values['notify_email'] !== $current->get('notify_email')) {
+            $changed[] = 'notify_email';
+        }
+        if (is_array($pending)) {
+            $pending['changed'] = array_values(array_unique(array_merge((array)($pending['changed'] ?? ['target']), $changed)));
+            return (string)json_encode($pending, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+        }
+        $addresses = array_values(array_filter(explode(',', $current->get('notify_email')), 'strlen'));
+        if (empty($changed) || empty($addresses)) {
+            return '';
+        }
+        $summary = ['target' => $old];
+        foreach (array_diff(Settings::CONNECTION[$old] ?? [], Settings::SECRETS) as $name) {
+            $summary[$name] = $current->get($name);
+        }
+        return (string)json_encode(['old_target' => $summary, 'old_addresses' => $addresses, 'changed' => $changed,
+            'changed_by' => (string)$this->context->username, 'time' => time()], JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+    }
+
+    /**
+     * A secret sealed for the runner: sealed:v1:<base64 of sodium_crypto_box_seal>
+     *
+     * @throws UserError backup_secrets_key_missing while the runner has not published its key
+     */
+    private static function seal(string $secret, string $publicKey): string
+    {
+        $key = base64_decode($publicKey, true);
+        if ($key === false || strlen($key) !== SODIUM_CRYPTO_BOX_PUBLICKEYBYTES) {
+            throw new UserError('backup_secrets_key_missing');
+        }
+        return Settings::SEALED . base64_encode(sodium_crypto_box_seal($secret, $key));
+    }
+
+    /**
+     * The value to store, '' for an empty optional value.
+     *
+     * @throws UserError backup_setting_invalid, backup_password_too_short
+     */
+    private function normalize(string $name, mixed $value): string
+    {
+        if (in_array($name, self::FLAGS, true)) {
+            return self::flag($value) ?? throw self::invalid($name);
+        }
+        if ($name === 'notify_email' && is_string($value)) {
+            // one address per line is fine as well
+            $value = preg_replace('/[\r\n\t]+/', ',', $value);
+        }
+        if ($name === 'schedule') {
+            return self::schedule($value) ?? throw self::invalid($name);
+        }
+        // secrets as typed: spaces at the ends are part of a password
+        $text = (in_array($name, Settings::SECRETS, true) && is_string($value)) ? $value : Input::text($value);
+        if ($text === null || mb_strlen($text) > (($name === 'notify_email') ? self::MAX_LENGTH * self::MAX_NOTIFY : self::MAX_LENGTH) || preg_match('/[\x00-\x1f\x7f]/', $text) || !mb_check_encoding($text, 'UTF-8')) {
+            throw self::invalid($name);
+        }
+        // no scheduled prune (append-only or object-lock targets); a prune job still works
+        if ($name === 'prune_weekday' && $text === '') {
+            return '';
+        }
+        if (isset(self::RANGES[$name])) {
+            $int = Input::intInRange($text, ...self::RANGES[$name]);
+            return ($int === null) ? throw self::invalid($name) : (string)$int;
+        }
+        $valid = match ($name) {
+            'target' => in_array($text, Settings::TARGETS, true),
+            // like the runner: scheme, host and port only
+            's3_endpoint' => self::url($text) && in_array(parse_url($text, PHP_URL_PATH) ?? '', ['', '/'], true),
+            'rest_url' => $text === '' || self::url($text),
+            's3_bucket' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{1,62}$/i', $text),
+            's3_path' => $text === '' || self::path(trim($text, '/'), false),
+            's3_region' => $text === '' || preg_match('/^[a-z0-9-]{1,64}$/i', $text),
+            's3_access_key' => preg_match('/^[\x21-\x7e]*$/', $text),
+            's3_secret_key', 'rest_password' => true,
+            'sftp_host' => $text === '' || preg_match(self::HOSTNAME, $text) || filter_var($text, FILTER_VALIDATE_IP),
+            // the runner's character set (ssh -l)
+            'sftp_user' => $text === '' || preg_match('/^[a-z0-9_][a-z0-9._-]{0,63}$/i', $text),
+            'sftp_path' => $text === '' || self::path($text, true),
+            'rest_user' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{0,63}$/i', $text),
+            // the runner's character set, below the repository folder
+            'local_path' => preg_match('~^[a-z0-9._-]+(/[a-z0-9._-]+)*$~i', $text) && self::path($text, false),
+            'password' => true,
+            'max_unused' => preg_match('/^(\d{1,3})\s*%?$/', $text, $percent) && (int)$percent[1] <= 100,
+            'notify_email' => true,
+            default => false,
+        };
+        if (!$valid) {
+            throw self::invalid($name);
+        }
+        return match ($name) {
+            's3_path' => trim($text, '/'),
+            's3_endpoint' => rtrim($text, '/'),
+            'sftp_path' => rtrim($text, '/'),
+            'password' => self::password($text),
+            'max_unused' => (int)$text . '%',
+            'notify_email' => self::emails($text),
+            default => $text,
+        };
+    }
+
+    /**
+     * The schedule as stored: a list (or its JSON) of {days: weekdays 0 (Sunday) to 6, time: "H:MM"}, up to MAX_SCHEDULE
+     * entries, each with at least one weekday; days sorted, times as HH:MM, repeated entries dropped. Null when invalid.
+     */
+    private static function schedule(mixed $value): ?string
+    {
+        if (is_string($value)) {
+            $value = json_decode($value, true);
+        }
+        if (!is_array($value) || !array_is_list($value) || count($value) > self::MAX_SCHEDULE) {
+            return null;
+        }
+        $entries = [];
+        foreach ($value as $entry) {
+            $days = $entry['days'] ?? null;
+            $time = is_array($entry) && is_string($entry['time'] ?? null) ? $entry['time'] : '';
+            if (!is_array($days) || empty($days) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', $time, $match)) {
+                return null;
+            }
+            foreach ($days as $day) {
+                if (Input::intInRange($day, 0, 6) === null) {
+                    return null;
+                }
+            }
+            $days = array_values(array_unique(array_map('intval', $days)));
+            sort($days);
+            $entries[] = ['days' => $days, 'time' => sprintf('%02d:%02d', $match[1], $match[2])];
+        }
+        return (string)json_encode(array_values(array_unique($entries, SORT_REGULAR)));
+    }
+
+    /** http(s) URL without credentials, query or fragment (rest_user/rest_password hold the credentials) */
+    private static function url(string $url): bool
+    {
+        $parts = parse_url($url);
+        return filter_var($url, FILTER_VALIDATE_URL) !== false && is_array($parts) && in_array(strtolower($parts['scheme'] ?? ''), ['http', 'https'], true)
+            && !isset($parts['user']) && !isset($parts['pass']) && !isset($parts['query']) && !isset($parts['fragment'])
+            && (preg_match(self::HOSTNAME, (string)($parts['host'] ?? '')) || filter_var(trim((string)($parts['host'] ?? ''), '[]'), FILTER_VALIDATE_IP));
+    }
+
+    /** A path of plain names separated by /, without . and .. parts; a leading / only where $absolute is allowed */
+    private static function path(string $path, bool $absolute): bool
+    {
+        if ($path === '' || (str_starts_with($path, '/') && !$absolute)) {
+            return false;
+        }
+        foreach (explode('/', trim($path, '/')) as $part) {
+            if ($part === '.' || $part === '..' || !preg_match('/^[a-z0-9_~+@][a-z0-9._~+@-]*$/i', $part)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /** @throws UserError backup_password_too_short ('' keeps the stored password) */
+    private static function password(string $password): string
+    {
+        if ($password !== '' && mb_strlen($password) < self::MIN_PASSWORD) {
+            throw new UserError('backup_password_too_short', [self::MIN_PASSWORD]);
+        }
+        return $password;
+    }
+
+    /** @throws UserError for an invalid address or too many */
+    private static function emails(string $text): string
+    {
+        $emails = array_values(array_unique(array_filter(preg_split('/[\s,;]+/', $text), 'strlen')));
+        if (count($emails) > self::MAX_NOTIFY) {
+            throw self::invalid('notify_email');
+        }
+        foreach ($emails as $email) {
+            if (!filter_var($email, FILTER_VALIDATE_EMAIL)) {
+                throw self::invalid('notify_email');
+            }
+        }
+        return implode(',', $emails);
+    }
+
+    /** Checkbox or API flag as '0'/'1'; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked; null = invalid */
+    private static function flag(mixed $value): ?string
+    {
+        if (is_array($value)) {
+            $value = end($value);
+        }
+        if (!is_scalar($value) && $value !== null) {
+            return null;
+        }
+        $flag = filter_var($value ?? false, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
+        return ($flag === null) ? null : (string)(int)$flag;
+    }
+
+    private static function invalid(string $name): UserError
+    {
+        return new UserError('backup_setting_invalid', [$name]);
+    }
+}

data/web/inc/lib/mailcow/Identity/Service/SyncPlanner.php

@@ -44,6 +44,8 @@ class SyncPlanner
         /** @var array<string, DirectoryUser> $candidates email => user to create, template still open */
         $candidates = [];
         $duplicates = [];
+        /** @var array<string, true> $ambiguous unlinked mailboxes whose email several IdP users have, left as they are */
+        $ambiguous = [];
         foreach ($listing->users as $user) {
             if (isset($owners[$user->id])) {
                 $username = $owners[$user->id];
@@ -55,10 +57,19 @@ class SyncPlanner
                 $plan->skipped++;
             } elseif (isset($state->accounts[$user->email])) {
                 $account = $state->accounts[$user->email];
-                if ($account->externalId !== null || isset($plan->links[$user->email])) {
+                if ($account->externalId !== null) {
                     self::conflict($plan, $user->email . ' is linked to another IdP user');
                     continue;
                 }
+                if (isset($plan->links[$user->email]) || isset($ambiguous[$user->email])) {
+                    // Which IdP user owns the mailbox is unclear: link none of them
+                    if (!isset($ambiguous[$user->email])) {
+                        self::conflict($plan, 'Several IdP users have the email ' . $user->email);
+                    }
+                    unset($plan->links[$user->email], $matched[$user->email]);
+                    $ambiguous[$user->email] = true;
+                    continue;
+                }
                 $plan->links[$user->email] = $user->id;
                 $matched[$user->email] = $user;
             } elseif (isset($state->mailboxes[$user->email]) || !$settings->importUsers || !$user->enabled
@@ -85,7 +96,7 @@ class SyncPlanner
         if ($groups !== null) {
             $this->planGroups($settings, $groups, $state, $usernames, $matched, $plan);
         }
-        $this->planStates($settings, $state, $matched, $plan);
+        $this->planStates($settings, $state, $matched, $ambiguous, $plan);
         if ($settings->periodicSync) {
             $this->planUpdates($settings, $state, $matched, $plan);
         }
@@ -211,10 +222,13 @@ class SyncPlanner
     }
 
     /** @param array<string, DirectoryUser> $matched */
-    private function planStates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
+    private function planStates(Settings $settings, State $state, array $matched, array $ambiguous, SyncPlan $plan): void
     {
         foreach ($state->accounts as $username => $account) {
             $username = (string)$username;
+            if (isset($ambiguous[$username])) {
+                continue;
+            }
             $user = $matched[$username] ?? null;
             if ($user === null && $account->missingSince === null) {
                 $plan->missing[] = $username;

data/web/inc/lib/mailcow/Imapsync/Model/CustomParams.php

@@ -19,6 +19,14 @@ class CustomParams
 {
     private const MAX_PAIRS = 100;
     private const MAX_VALUE_LENGTH = 1000;
+    /**
+     * Never allowed, even when configured: code or command execution, local files, the environment
+     * of the container (tests*), passwords in the output and stopping other runs (abort).
+     * The runner (dovecot/imapsync_runner.py) skips the same options.
+     */
+    public const BLOCKED = ['abort', 'delete2foldersbutnot', 'delete2foldersonly', 'logdir', 'logfile', 'maxlinelengthcmd',
+        'oauthrefreshcmd1', 'oauthrefreshcmd2', 'passfile1', 'passfile2', 'pidfile', 'pipemess', 'regexflag', 'regexmess',
+        'regextrans2', 'showpasswords', 'skipmess', 'tests', 'testsdebug', 'testslive', 'testslive6', 'testsunit', 'tmpdir'];
 
     private static bool $warnedOldFormat = false;
 
@@ -57,7 +65,7 @@ class CustomParams
 
     /**
      * The effective allowlist: $IMAPSYNC_OPTIONS (after vars.local.inc.php) without the old
-     * 'whitelist' key and malformed entries; the defaults when nothing valid is left.
+     * 'whitelist' key, malformed entries and BLOCKED options; the defaults when nothing valid is left.
      */
     public static function allowlist(array $configured, array $defaults): array
     {
@@ -73,7 +81,7 @@ class CustomParams
             self::$warnedOldFormat = true;
             error_log("mailcow: \$IMAPSYNC_OPTIONS['whitelist'] (old format) is ignored, use 'option' => true|false|'regex' (see data/web/inc/vars.inc.php)");
         }
-        return empty($allowlist) ? $defaults : $allowlist;
+        return array_diff_key(empty($allowlist) ? $defaults : $allowlist, array_flip(self::BLOCKED));
     }
 
     /** The value to store for one option; throws when the allowlist rule is violated */

data/web/inc/lib/mailcow/Imapsync/Model/Source.php

@@ -29,6 +29,7 @@ class Source
     public string $host = '';
     public int $port = 0;
     public ?string $enc = 'TLS';
+    public int $verifyCert = 1;
     public ?string $authType = 'PLAIN';
     public string $oauthFlow = self::FLOW_CLIENT_CREDENTIALS;
     public ?string $tokenEndpoint = null;
@@ -65,6 +66,7 @@ class Source
         $source->scope = (string)($row['scope'] ?? 'user');
         $source->host = (string)($row['host1'] ?? '');
         $source->port = (int)($row['port1'] ?? 0);
+        $source->verifyCert = (int)($row['verify_cert'] ?? 1);
         $source->oauthFlow = (string)($row['oauth_flow'] ?? self::FLOW_CLIENT_CREDENTIALS);
         $source->tokenExpires = isset($row['oauth_token_expires']) ? (int)$row['oauth_token_expires'] : null;
         $source->refreshAttemptAge = isset($row['refresh_attempt_age']) ? (int)$row['refresh_attempt_age'] : null;
@@ -112,6 +114,7 @@ class Source
             'host1' => $this->host,
             'port1' => $this->port,
             'enc1' => $this->enc,
+            'verify_cert' => $this->verifyCert,
             'auth_type' => $this->authType,
             'oauth_flow' => $this->oauthFlow,
             'oauth_token_endpoint' => $this->tokenEndpoint,

data/web/inc/lib/mailcow/Imapsync/Repository/SourceRepository.php

@@ -48,6 +48,12 @@ class SourceRepository extends Repository
         return (bool)$this->query("SELECT 1 FROM `imapsync` WHERE `source_id` = :id LIMIT 1", ['id' => $id])->fetchColumn();
     }
 
+    /** @return string[] mailboxes whose sync jobs use the source */
+    public function jobMailboxes(int $id): array
+    {
+        return $this->query("SELECT DISTINCT `user2` FROM `imapsync` WHERE `source_id` = :id", ['id' => $id])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
     public function isSuperAdmin(string $username): bool
     {
         return (bool)$this->query("SELECT 1 FROM `admin` WHERE `username` = :username AND `superadmin` = '1'",
@@ -200,6 +206,7 @@ class SourceRepository extends Repository
             'host1' => $source->host,
             'port1' => $source->port,
             'enc1' => $source->enc,
+            'verify_cert' => $source->verifyCert,
             'auth_type' => $source->authType,
             'oauth_flow' => $source->oauthFlow,
             'oauth_token_endpoint' => $source->tokenEndpoint,

data/web/inc/lib/mailcow/Imapsync/Service/SourceService.php

@@ -121,7 +121,11 @@ class SourceService
         if ($writeTargets) {
             $this->applyScope($source, $input['scope'] ?? $current->scope, $input['domains'] ?? [], $input['users'] ?? []);
         }
-        $this->repo->update($source, $writeTargets, self::connection($current) !== self::connection($source));
+        $connectionChanged = self::connection($current) !== self::connection($source);
+        if ($connectionChanged || ($current->verifyCert && !$source->verifyCert)) {
+            $this->requireAllJobsAccessible($source);
+        }
+        $this->repo->update($source, $writeTargets, $connectionChanged);
         $this->fetchToken($source);
         return $source;
     }
@@ -200,6 +204,24 @@ class SourceService
         return $this->context->hasAcl('syncjobs') || $this->context->isAdmin();
     }
 
+    /**
+     * The connection of a source only changes when the caller may access every mailbox whose sync jobs use it,
+     * their stored passwords would go to the new server.
+     *
+     * @throws UserError imapsync_source_used_by_others
+     */
+    private function requireAllJobsAccessible(Source $source): void
+    {
+        if ($this->context->hasAdminRights()) {
+            return;
+        }
+        foreach ($this->repo->jobMailboxes((int)$source->id) as $mailbox) {
+            if (!$this->context->canAccessMailbox($mailbox)) {
+                throw new UserError('imapsync_source_used_by_others', [htmlspecialchars($source->name)]);
+            }
+        }
+    }
+
     /** @throws AccessDenied when the caller cannot see or edit the source */
     private function editable(int $id): Source
     {
@@ -274,6 +296,7 @@ class SourceService
         $source->host = $host;
         $source->port = $port;
         $source->enc = $enc;
+        $source->verifyCert = isset($input['verify_cert']) ? (int)!empty($input['verify_cert']) : $source->verifyCert;
         $source->authType = $authType;
         $source->active = isset($input['active']) ? (int)$input['active'] : $source->active;
     }

data/web/inc/lib/mailcow/MailPolicy/Controller/MailPolicyApiController.php

@@ -0,0 +1,132 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Controller;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use Mailcow\MailPolicy\Service\PolicyEngine;
+use Mailcow\MailPolicy\Service\PolicyService;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php and edit.php (routes `mail-policy`), and the answer to the Postfix policy
+ * service (data/conf/rspamd/dynmaps/mailpolicy.php). Messages are logged as ['mail_policy', action, 'policy', data].
+ */
+class MailPolicyApiController
+{
+    public function __construct(private PolicyService $policies, private PolicyEngine $engine, private ApiResponse $response)
+    {
+    }
+
+    /** POST add/mail-policy */
+    public function addPolicy(array $attr): bool
+    {
+        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
+            $policy = $this->policies->create($attr);
+            return $this->response->success(['mail_policy_added', htmlspecialchars($policy->name)], $log);
+        });
+    }
+
+    /** POST edit/mail-policy: one message per id */
+    public function editPolicies(?array $items, array $attr): bool
+    {
+        $data = array_merge(['id' => $items], $attr);
+        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
+            $this->policies->requireEditor();
+            return $this->batch((array)($data['id'] ?? []), $log,
+                fn(int $id): array => ['mail_policy_modified', htmlspecialchars($this->policies->update($id, $attr)->name)]);
+        });
+    }
+
+    /** POST delete/mail-policy: one message per id */
+    public function deletePolicies(array $items): bool
+    {
+        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
+            $this->policies->requireEditor();
+            return $this->batch($items, $log, fn(int $id): array => ['mail_policy_removed', htmlspecialchars($this->policies->delete($id)->name)]);
+        });
+    }
+
+    /** GET mail-policy/all[/<domain,domain>] */
+    public function getPolicies(?string $domains = null): array
+    {
+        $list = array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
+        return array_values(array_filter(array_map(fn(int $id): array => $this->getPolicy($id), $this->policies->visibleIds($list))));
+    }
+
+    /** GET mail-policy/<id>, also edit.php */
+    public function getPolicy(mixed $id): array
+    {
+        $id = $this->toId($id);
+        $policy = ($id === null) ? null : $this->policies->get($id);
+        return ($policy === null) ? [] : $this->policies->toArray($policy);
+    }
+
+    /** GET mail-policy/test/<sender>/<recipient>[/login]: would the mail be accepted? Sender "-" = null sender; [] when not allowed to ask */
+    public function test(string $sender, string $recipient, bool $loggedIn): array
+    {
+        try {
+            return $this->policies->test(($sender === '-') ? '' : $sender, $recipient, $loggedIn);
+        } catch (UserError) {
+            return [];
+        }
+    }
+
+    /** functions.mailbox.inc.php: a mailbox or alias was renamed */
+    public function addressRenamed(string $old, string $new): void
+    {
+        $this->policies->addressRenamed($old, $new);
+    }
+
+    /** Answer for the Postfix policy service: DUNNO or the rejection */
+    public function check(string $sender, string $recipient, string $login, int $port, string $helo = ''): string
+    {
+        return $this->engine->check($sender, $recipient, $login, $port, $helo)->action();
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     */
+    private function handle(string $action, mixed $data, callable $work): bool
+    {
+        $log = ['mail_policy', $action, 'policy', $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+
+    /** One message per id: the success message returned by $work(id), or the UserError */
+    private function batch(array $ids, array $log, callable $work): bool
+    {
+        foreach ($ids as $id) {
+            try {
+                $this->response->success($work($this->requireId($id)), $log);
+            } catch (UserError $e) {
+                $this->response->error($e, $log);
+            }
+        }
+        return true;
+    }
+
+    /** Integer id from request input ("12", 12); null for anything else */
+    private function toId(mixed $id): ?int
+    {
+        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
+    }
+
+    /** @throws AccessDenied for ids that cannot exist */
+    private function requireId(mixed $id): int
+    {
+        return $this->toId($id) ?? throw new AccessDenied();
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/MailPolicyModule.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy;
+
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\MailPolicy\Controller\MailPolicyApiController;
+use Mailcow\MailPolicy\Repository\LookupRepository;
+use Mailcow\MailPolicy\Repository\PolicyRepository;
+use Mailcow\MailPolicy\Service\PolicyEngine;
+use Mailcow\MailPolicy\Service\PolicyService;
+use PDO;
+
+/**
+ * Composition root of the mail policies: whom mailboxes and groups may send to and receive from, checked by
+ * Postfix at RCPT TO through the policy service (mailcow-policy in master.cf, data/conf/rspamd/dynmaps/mailpolicy.php).
+ *
+ *   process_get_return(MailPolicyModule::fromGlobals()->api->getPolicies());
+ *   echo MailPolicyModule::create($pdo, Context::system())->api->check($sender, $recipient, $login, $port);  // policy service
+ */
+class MailPolicyModule
+{
+    public function __construct(
+        public readonly PolicyService $policies,
+        public readonly PolicyEngine $engine,
+        public readonly MailPolicyApiController $api
+    ) {
+    }
+
+    public static function create(PDO $pdo, Context $context): self
+    {
+        $engine = new PolicyEngine(new LookupRepository($pdo));
+        $policies = new PolicyService(new PolicyRepository($pdo), $engine, $context);
+        return new self($policies, $engine, new MailPolicyApiController($policies, $engine, new ApiResponse()));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Model/Decision.php

@@ -0,0 +1,56 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Model;
+
+/**
+ * Result of the policy check for one recipient: allowed, refused by the send or receive part of a policy, or refused
+ * because the recipient is written in a form the check cannot follow (routing through another address).
+ */
+class Decision
+{
+    private const TEXT = [
+        'send' => 'Sending to this recipient is not allowed by the mail policy of the sender',
+        'receive' => 'This recipient does not accept mail from this sender',
+        'address' => 'This form of the recipient address is not allowed by the mail policy',
+    ];
+
+    private function __construct(public readonly bool $allowed, public readonly string $side = '', public readonly ?Policy $policy = null)
+    {
+    }
+
+    public static function allow(): self
+    {
+        return new self(true);
+    }
+
+    /** $side send or receive; without $policy the default text is used */
+    public static function deny(string $side, ?Policy $policy = null): self
+    {
+        return new self(false, $side, $policy);
+    }
+
+    /** The recipient routes through another address (user%domain, domain!user, a second @) */
+    public static function routedAddress(): self
+    {
+        return new self(false, 'address');
+    }
+
+    /** Text of the rejection: the text of the policy or the default */
+    public function text(): string
+    {
+        return ($this->policy !== null && $this->policy->rejectText !== '') ? $this->policy->rejectText : self::defaultText($this->side);
+    }
+
+    public static function defaultText(string $side): string
+    {
+        return self::TEXT[$side] ?? '';
+    }
+
+    /** Answer for the Postfix policy protocol */
+    public function action(): string
+    {
+        return $this->allowed ? 'DUNNO' : '550 5.7.1 ' . $this->text();
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Model/Policy.php

@@ -0,0 +1,111 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Model;
+
+/**
+ * A mail policy (table `mail_policy`): whom its mailboxes and groups may send to and receive from.
+ *
+ * Each direction has a mode: `all` (only the blocked entries are refused), `internal` (the own domain and its
+ * alias domains, plus the allowed entries) or `list` (only the allowed entries). Blocked entries are always
+ * refused. Entries are addresses, domains or `group:<id>`. A sender counts as internal only when it is logged in
+ * to this mailcow (SASL) with a mailbox of the domain, never because of its address.
+ */
+class Policy
+{
+    public const MODES = ['all', 'internal', 'list'];
+
+    public ?int $id = null;
+    public string $domain = '';
+    public string $name = '';
+    public string $description = '';
+    public string $sendMode = 'all';
+    /** @var string[] */
+    public array $sendAllow = [];
+    /** @var string[] */
+    public array $sendBlock = [];
+    public string $receiveMode = 'all';
+    /** @var string[] */
+    public array $receiveAllow = [];
+    /** @var string[] */
+    public array $receiveBlock = [];
+    /** Text of the rejection, '' = default text */
+    public string $rejectText = '';
+    public int $active = 1;
+    /** @var string[] usernames the policy is assigned to */
+    public array $mailboxes = [];
+    /** @var int[] groups the policy is assigned to: their members and their address */
+    public array $groups = [];
+    public ?string $created = null;
+    public ?string $modified = null;
+
+    /** Row of `mail_policy`; assignments are set by the repository */
+    public static function fromRow(array $row): self
+    {
+        $policy = new self();
+        $policy->id = (int)$row['id'];
+        $policy->domain = (string)$row['domain'];
+        $policy->name = (string)$row['name'];
+        $policy->description = (string)($row['description'] ?? '');
+        $policy->sendMode = (string)$row['send_mode'];
+        $policy->sendAllow = self::list($row['send_allow'] ?? null);
+        $policy->sendBlock = self::list($row['send_block'] ?? null);
+        $policy->receiveMode = (string)$row['receive_mode'];
+        $policy->receiveAllow = self::list($row['receive_allow'] ?? null);
+        $policy->receiveBlock = self::list($row['receive_block'] ?? null);
+        $policy->rejectText = (string)($row['reject_text'] ?? '');
+        $policy->active = (int)$row['active'];
+        $policy->created = $row['created'] ?? null;
+        $policy->modified = $row['modified'] ?? null;
+        return $policy;
+    }
+
+    /** Column values for `mail_policy` */
+    public function toRow(): array
+    {
+        return [
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'description' => $this->description,
+            'send_mode' => $this->sendMode,
+            'send_allow' => json_encode($this->sendAllow),
+            'send_block' => json_encode($this->sendBlock),
+            'receive_mode' => $this->receiveMode,
+            'receive_allow' => json_encode($this->receiveAllow),
+            'receive_block' => json_encode($this->receiveBlock),
+            'reject_text' => $this->rejectText,
+            'active' => $this->active,
+        ];
+    }
+
+    /** Entry of the API and the edit page */
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->id,
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'description' => $this->description,
+            'send_mode' => $this->sendMode,
+            'send_allow' => $this->sendAllow,
+            'send_block' => $this->sendBlock,
+            'receive_mode' => $this->receiveMode,
+            'receive_allow' => $this->receiveAllow,
+            'receive_block' => $this->receiveBlock,
+            'reject_text' => $this->rejectText,
+            'active' => $this->active,
+            'mailboxes' => $this->mailboxes,
+            'groups' => $this->groups,
+            'created' => $this->created,
+            'modified' => $this->modified,
+        ];
+    }
+
+    /** @return string[] entries of a JSON list column */
+    private static function list(mixed $json): array
+    {
+        $list = is_string($json) ? json_decode($json, true) : null;
+        return is_array($list) ? array_values(array_filter($list, 'is_string')) : [];
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Repository/LookupRepository.php

@@ -0,0 +1,141 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Repository;
+
+use Mailcow\Core\Repository;
+use Mailcow\MailPolicy\Model\Policy;
+
+/**
+ * Read-only lookups of the policy check: mailboxes, aliases, alias domains, groups and the active policies of a
+ * mailbox or group. Addresses are passed in lower case.
+ */
+class LookupRepository extends Repository
+{
+    /** @return array{username: string, domain: string}|null a regular mailbox that is not disabled */
+    public function mailbox(string $username): ?array
+    {
+        $row = $this->query("SELECT `username`, `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = '' AND `active` != '0'",
+            ['username' => $username])->fetch();
+        return is_array($row) ? ['username' => strtolower((string)$row['username']), 'domain' => strtolower((string)$row['domain'])] : null;
+    }
+
+    /** Target domain of an active alias domain */
+    public function aliasDomainTarget(string $domain): ?string
+    {
+        $target = $this->query("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'",
+            ['domain' => $domain])->fetchColumn();
+        return is_string($target) ? strtolower($target) : null;
+    }
+
+    /** A domain or alias domain Postfix delivers itself (as mysql_virtual_domains_maps.cf: active, no backup MX) */
+    public function isLocalDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :d1 AND `active` = '1'
+          UNION SELECT 1 FROM `domain` WHERE `domain` = :d2 AND `active` = '1' AND `backupmx` = '0'",
+            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
+    }
+
+    /**
+     * Stored spelling of a mailbox, alias or temporary alias address. MySQL compares without case and accents, so
+     * Postfix delivers mail for chéf@ to chef@.
+     */
+    public function storedAddress(string $address): ?string
+    {
+        $stored = $this->query("SELECT `username` FROM `mailbox` WHERE `username` = :a1 UNION SELECT `address` FROM `alias` WHERE `address` = :a2
+          UNION SELECT `address` FROM `spamalias` WHERE `address` = :a3 LIMIT 1", ['a1' => $address, 'a2' => $address, 'a3' => $address])->fetchColumn();
+        return is_string($stored) ? strtolower($stored) : null;
+    }
+
+    /** Stored spelling of a domain or alias domain, see storedAddress() */
+    public function storedDomain(string $domain): ?string
+    {
+        $stored = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :d1 UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :d2 LIMIT 1",
+            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
+        return is_string($stored) ? strtolower($stored) : null;
+    }
+
+    /** Active group with this distribution address */
+    public function groupByAddress(string $address): ?int
+    {
+        $id = $this->query("SELECT `id` FROM `mailbox_group` WHERE `address` = :address AND `active` = '1'", ['address' => $address])->fetchColumn();
+        return ($id === false) ? null : (int)$id;
+    }
+
+    public function groupAddress(int $id): ?string
+    {
+        $address = $this->query("SELECT `address` FROM `mailbox_group` WHERE `id` = :id AND `active` = '1'", ['id' => $id])->fetchColumn();
+        return is_string($address) && $address !== '' ? strtolower($address) : null;
+    }
+
+    /** Member of an active group */
+    public function isGroupMember(int $groupId, string $username): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
+          WHERE `gm`.`group_id` = :id AND `gm`.`username` = :username AND `g`.`active` = '1'",
+            ['id' => $groupId, 'username' => $username])->fetchColumn();
+    }
+
+    /**
+     * What Postfix' virtual alias maps find for this key, in their order: an alias (also the own alias of a mailbox
+     * and the alias of a group address) or a temporary alias.
+     *
+     * @return array{kind: string, domain?: string, internal?: bool, owner?: string}|null
+     */
+    public function aliasTarget(string $address): ?array
+    {
+        $alias = $this->query("SELECT `domain`, `internal` FROM `alias` WHERE `address` = :address AND `active` IN ('1', '2')",
+            ['address' => $address])->fetch();
+        if (is_array($alias)) {
+            return ['kind' => 'alias', 'domain' => strtolower((string)$alias['domain']), 'internal' => $alias['internal'] == '1'];
+        }
+        $owner = $this->query("SELECT `goto` FROM `spamalias` WHERE `address` = :address AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0)",
+            ['address' => $address])->fetchColumn();
+        return is_string($owner) ? ['kind' => 'temporary', 'owner' => strtolower($owner)] : null;
+    }
+
+    /** An active catch-all alias of the domain has the internal flag (applies to the whole domain) */
+    public function internalCatchAll(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address AND `internal` = '1' AND `active` IN ('1', '2')",
+            ['address' => '@' . $domain])->fetchColumn();
+    }
+
+    /** An active policy with members restricts sending */
+    public function hasSendRules(): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `mail_policy` `p` JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
+          WHERE `p`.`active` = '1' AND (`p`.`send_mode` != 'all' OR `p`.`send_block` != '[]') LIMIT 1")->fetchColumn();
+    }
+
+    /**
+     * @return Policy[] active policies of the mailbox: assigned to it or to an active group it is a member of; only
+     *                  policies of its own domain, so a group with members of other domains cannot restrict them
+     */
+    public function policiesForMailbox(string $username): array
+    {
+        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
+          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
+          JOIN `mailbox` `m` ON `m`.`username` = :u1 AND `m`.`domain` = `p`.`domain`
+          WHERE `p`.`active` = '1' AND (`pm`.`username` = :u2 OR `pm`.`group_id` IN (
+            SELECT `gm`.`group_id` FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
+            WHERE `gm`.`username` = :u3 AND `g`.`active` = '1'))
+          ORDER BY `p`.`id`", ['u1' => $username, 'u2' => $username, 'u3' => $username]);
+    }
+
+    /** @return Policy[] active policies assigned to the group */
+    public function policiesForGroup(int $groupId): array
+    {
+        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
+          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
+          WHERE `p`.`active` = '1' AND `pm`.`group_id` = :id
+          ORDER BY `p`.`id`", ['id' => $groupId]);
+    }
+
+    /** @return Policy[] */
+    private function policies(string $sql, array $params): array
+    {
+        return array_map(fn(array $row): Policy => Policy::fromRow($row), $this->query($sql, $params)->fetchAll());
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Repository/PolicyRepository.php

@@ -0,0 +1,148 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Repository;
+
+use Mailcow\Core\Context;
+use Mailcow\Core\Repository;
+use Mailcow\MailPolicy\Model\Policy;
+use PDO;
+
+/**
+ * Tables `mail_policy` and `mail_policy_member` (the mailboxes and groups a policy is assigned to), plus the
+ * lookups the editing needs. Assignments go with their mailbox, group or domain (foreign keys, ON DELETE CASCADE).
+ */
+class PolicyRepository extends Repository
+{
+    public function find(int $id): ?Policy
+    {
+        $row = $this->query("SELECT * FROM `mail_policy` WHERE `id` = :id", ['id' => $id])->fetch();
+        return is_array($row) ? $this->withMembers(Policy::fromRow($row)) : null;
+    }
+
+    /**
+     * @param string[] $domains limit to these domains, [] = all visible
+     * @return int[] ids of the policies visible to the caller, ordered by domain and name
+     */
+    public function visibleIds(Context $context, array $domains = []): array
+    {
+        if ($context->hasAdminRights()) {
+            [$visible, $params] = ['1=1', []];
+        } elseif ($context->isDomainAdmin()) {
+            [$visible, $params] = ["`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
+                ['vis_da' => $context->username]];
+        } else {
+            return [];
+        }
+        $sql = "SELECT `id` FROM `mail_policy` WHERE $visible";
+        if (!empty($domains)) {
+            $names = [];
+            foreach (array_values($domains) as $i => $domain) {
+                $names[] = ':dom' . $i;
+                $params['dom' . $i] = $domain;
+            }
+            $sql .= " AND `domain` IN (" . implode(', ', $names) . ")";
+        }
+        return array_map('intval', $this->query($sql . " ORDER BY `domain`, `name`", $params)->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `mail_policy` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
+            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
+    }
+
+    /** Inserts or updates the policy with its assignments; returns its id */
+    public function save(Policy $policy): int
+    {
+        return $this->transaction(function () use ($policy): int {
+            if ($policy->id === null) {
+                $policy->id = $this->insertRow('mail_policy', $policy->toRow());
+            } else {
+                $this->updateRow('mail_policy', $policy->id, $policy->toRow());
+                $this->query("DELETE FROM `mail_policy_member` WHERE `policy_id` = :id", ['id' => $policy->id]);
+            }
+            foreach ($policy->mailboxes as $username) {
+                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'username' => $username]);
+            }
+            foreach ($policy->groups as $groupId) {
+                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'group_id' => $groupId]);
+            }
+            return $policy->id;
+        });
+    }
+
+    public function delete(int $id): void
+    {
+        $this->query("DELETE FROM `mail_policy` WHERE `id` = :id", ['id' => $id]);
+    }
+
+    /** Address entries of all lists follow a renamed mailbox or alias */
+    public function renameAddress(string $old, string $new): void
+    {
+        $this->transaction(function () use ($old, $new): void {
+            $columns = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
+            foreach ($this->query("SELECT `id`, `" . implode('`, `', $columns) . "` FROM `mail_policy`")->fetchAll() as $row) {
+                $changed = [];
+                foreach ($columns as $column) {
+                    $entries = json_decode((string)$row[$column], true);
+                    if (is_array($entries) && in_array($old, $entries, true)) {
+                        $changed[$column] = json_encode(array_values(array_unique(array_map(fn(string $entry): string => ($entry === $old) ? $new : $entry, $entries))));
+                    }
+                }
+                if ($changed !== []) {
+                    $this->updateRow('mail_policy', (int)$row['id'], $changed);
+                }
+            }
+        });
+    }
+
+    /** A primary domain (policies never belong to alias domains) */
+    public function isDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+    }
+
+    /** Domain of a regular mailbox, null if there is none */
+    public function mailboxDomain(string $username): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = ''", ['username' => $username])->fetchColumn();
+        return is_string($domain) ? $domain : null;
+    }
+
+    public function groupDomain(int $id): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox_group` WHERE `id` = :id", ['id' => $id])->fetchColumn();
+        return is_string($domain) ? $domain : null;
+    }
+
+    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
+    public function groupsIn(array $domains): array
+    {
+        if (empty($domains)) {
+            return [];
+        }
+        $names = [];
+        $params = [];
+        foreach (array_values($domains) as $i => $domain) {
+            $names[] = ':dom' . $i;
+            $params['dom' . $i] = $domain;
+        }
+        return $this->query("SELECT `id`, `name`, `domain`, `address` FROM `mailbox_group` WHERE `domain` IN (" . implode(', ', $names) . ")
+          ORDER BY `domain`, `name`", $params)->fetchAll();
+    }
+
+    private function withMembers(Policy $policy): Policy
+    {
+        foreach ($this->query("SELECT `username`, `group_id` FROM `mail_policy_member` WHERE `policy_id` = :id ORDER BY `username`, `group_id`",
+            ['id' => $policy->id])->fetchAll() as $row) {
+            if ($row['username'] !== null) {
+                $policy->mailboxes[] = (string)$row['username'];
+            } elseif ($row['group_id'] !== null) {
+                $policy->groups[] = (int)$row['group_id'];
+            }
+        }
+        return $policy;
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyEngine.php

@@ -0,0 +1,325 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Service;
+
+use Mailcow\MailPolicy\Model\Decision;
+use Mailcow\MailPolicy\Model\Policy;
+use Mailcow\MailPolicy\Repository\LookupRepository;
+
+/**
+ * Policy check of one recipient, asked by Postfix at RCPT TO (data/conf/rspamd/dynmaps/mailpolicy.php).
+ *
+ * Send part: the policies of the logged-in mailbox (assigned to it or to its groups) and of the mailbox the
+ * sender address belongs to. Receive part: what Postfix delivers the recipient to, looked up in the order of its
+ * virtual alias maps (address, address without +tag, mailbox of an alias domain): the policies of the mailbox, of
+ * the group address or of the owner of a temporary alias, and the `internal` flag of the alias (a catch-all alias
+ * with the flag covers its whole domain). Mail sent to an alias or group reaches its members without their own
+ * policies. Every policy must allow the mail.
+ *
+ * Internal: a recipient in the domain of the policy or one of its alias domains; a sender logged in with a
+ * mailbox of that domain. Entries of a local domain or address count for logged-in senders only, so they cannot
+ * be forged; block entries also match the envelope sender. External entries are compared with the envelope sender.
+ * Addresses are compared the way Postfix finds them: lower case, without trailing dot, in their stored spelling
+ * (MySQL ignores accents). Recipients routed through localhost (user%domain@localhost, domain!user@localhost,
+ * user@domain@localhost) are refused, Postfix would deliver them to the routed address.
+ *
+ * Dovecot submits Sieve redirects, vacation replies and notifications on port 588 without login, SOGo sends
+ * reminders with a service login: mail with the address of a mailbox counts as sent by that mailbox. Other mail of
+ * these services (bounces redirected with the null sender, Sieve rejects) has no known sender: it is checked like
+ * mail from outside and does not leave the mailcow while any policy restricts sending. Quota warnings pass.
+ */
+class PolicyEngine
+{
+    /** Ports of mailcow's own deliveries: quarantine release (590), BCC and archive copies (591) */
+    private const INTERNAL_PORTS = [590, 591];
+    /** Submission of SOGo and Dovecot, without login reachable from mailcow's own network only */
+    private const SERVICE_PORT = 588;
+    /** Domain of the Dovecot master login of mailcow's services (SOGo reminders) */
+    private const SERVICE_LOGIN_DOMAIN = 'mailcow.local';
+    /** HELO of the quota warning copies of Dovecot (quota_notify.py), configured by admins */
+    private const QUOTA_HELO = 'quotanotification';
+    /** Recipient without domain every server accepts (RFC 5321) */
+    private const POSTMASTER = 'postmaster';
+    /** Postfix delivers these destinations itself (mydestination), user%domain and domain!user route from there */
+    private const LOCAL_DESTINATIONS = ['localhost', 'localhost.localdomain'];
+
+    /** Lookups of the current check */
+    private array $cache = [];
+
+    public function __construct(private LookupRepository $lookup)
+    {
+    }
+
+    /** $login: SASL username, '' if not logged in; $port: smtpd port of the request, 0 if unknown; $helo: HELO name */
+    public function check(string $sender, string $recipient, string $login, int $port = 0, string $helo = ''): Decision
+    {
+        $this->cache = [];
+        $recipient = self::address($recipient);
+        if (in_array($port, self::INTERNAL_PORTS, true) || $recipient === '' || $recipient === self::POSTMASTER) {
+            return Decision::allow();
+        }
+        if (self::isRouted($recipient)) {
+            return Decision::routedAddress();
+        }
+        $recipient = $this->stored($recipient);
+        $sender = $this->stored(self::address($sender));
+        $login = self::address($login);
+        $serviceLogin = self::domainOf($login) === self::SERVICE_LOGIN_DOMAIN;
+        $account = null;
+        if ($login !== '' && !$serviceLogin) {
+            $account = $this->cached('mailbox', $login);
+        } elseif ($serviceLogin || $port === self::SERVICE_PORT) {
+            if ($port === self::SERVICE_PORT && !$serviceLogin && strtolower($helo) === self::QUOTA_HELO) {
+                return Decision::allow();
+            }
+            // without a mailbox sender the sending mailbox is unknown
+            $account = $this->senderMailbox($sender);
+            if ($account === null && !$this->cached('isLocalDomain', self::domainOf($recipient)) && $this->lookup->hasSendRules()) {
+                return Decision::deny('send');
+            }
+        }
+        foreach ($this->sendingPolicies($account, $sender) as $policy) {
+            if (!$this->allows($policy->sendMode, $policy->sendAllow, $policy->sendBlock, $this->isInternalRecipient($recipient, $policy->domain),
+                fn(string $entry): bool => $this->recipientMatches($entry, $recipient))) {
+                return Decision::deny('send', $policy);
+            }
+        }
+        foreach ($this->receivingPolicies($recipient) as $policy) {
+            if (!$this->allows($policy->receiveMode, $policy->receiveAllow, $policy->receiveBlock,
+                $account !== null && $account['domain'] === $policy->domain,
+                fn(string $entry, bool $block): bool => $this->senderMatches($entry, $sender, $account, $block))) {
+                return Decision::deny('receive', $policy);
+            }
+        }
+        return Decision::allow();
+    }
+
+    /** Blocked entries first; then the mode decides. $matches($entry, $block) */
+    private function allows(string $mode, array $allow, array $block, bool $internal, callable $matches): bool
+    {
+        foreach ($block as $entry) {
+            if ($matches($entry, true)) {
+                return false;
+            }
+        }
+        return match ($mode) {
+            'internal' => $internal || $this->any($allow, $matches),
+            'list' => $this->any($allow, $matches),
+            default => true,
+        };
+    }
+
+    private function any(array $entries, callable $matches): bool
+    {
+        foreach ($entries as $entry) {
+            if ($matches($entry, false)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /** @return array{username: string, domain: string}|null mailbox of the sender address */
+    private function senderMailbox(string $sender): ?array
+    {
+        return str_contains($sender, '@') ? $this->cached('mailbox', $this->resolve(self::untagged($sender))) : null;
+    }
+
+    /** @return Policy[] of the logged-in mailbox and of the mailbox it sends as */
+    private function sendingPolicies(?array $account, string $sender): array
+    {
+        if ($account === null) {
+            return [];
+        }
+        $policies = $this->lookup->policiesForMailbox($account['username']);
+        $sentAs = $this->senderMailbox($sender);
+        if ($sentAs !== null && $sentAs['username'] !== $account['username']) {
+            $policies = array_merge($policies, $this->lookup->policiesForMailbox($sentAs['username']));
+        }
+        return self::unique($policies);
+    }
+
+    /** @return Policy[] of what Postfix delivers the recipient to, see the class comment */
+    private function receivingPolicies(string $recipient): array
+    {
+        // the address, then without +tag; an alias domain maps to the mailbox in its target domain and Postfix
+        // looks up the result again with the +tag
+        [$address, $target] = $this->aliasTarget([$recipient, self::untagged($recipient)]);
+        $resolved = $this->resolve($recipient);
+        if ($target === null && $resolved !== $recipient) {
+            [$address, $target] = $this->aliasTarget([$resolved, self::untagged($resolved)]);
+        }
+        $address ??= self::untagged($resolved);
+        if (($target['kind'] ?? '') === 'temporary') {
+            $address = $target['owner'];
+        }
+        $policies = ($this->cached('mailbox', $address) !== null) ? $this->lookup->policiesForMailbox($address) : [];
+        $group = $this->lookup->groupByAddress($address);
+        if ($group !== null) {
+            $policies = array_merge($policies, $this->lookup->policiesForGroup($group));
+        }
+        // the `internal` flag of an alias or group: only senders of its domain
+        if (!empty($target['internal'])) {
+            $policies[] = $this->internalRule($target['domain']);
+        }
+        foreach (array_unique([self::domainOf($recipient), $this->resolveDomain(self::domainOf($recipient))]) as $domain) {
+            if ($this->lookup->internalCatchAll($domain)) {
+                $policies[] = $this->internalRule($domain);
+            }
+        }
+        return self::unique($policies);
+    }
+
+    /** @return array{0: ?string, 1: ?array} the first key Postfix' virtual alias maps find, with what they find */
+    private function aliasTarget(array $keys): array
+    {
+        foreach (array_unique($keys) as $key) {
+            $target = $this->lookup->aliasTarget($key);
+            if ($target !== null) {
+                return [$key, $target];
+            }
+        }
+        return [null, null];
+    }
+
+    /** Receive rule of the `internal` flag */
+    private function internalRule(string $domain): Policy
+    {
+        $policy = new Policy();
+        $policy->domain = $this->resolveDomain($domain);
+        $policy->receiveMode = 'internal';
+        return $policy;
+    }
+
+    private function isInternalRecipient(string $recipient, string $domain): bool
+    {
+        return $this->resolveDomain(self::domainOf($recipient)) === $domain;
+    }
+
+    private function recipientMatches(string $entry, string $recipient): bool
+    {
+        $address = $this->resolve(self::untagged($recipient));
+        if (str_starts_with($entry, 'group:')) {
+            $id = (int)substr($entry, 6);
+            return $this->cached('groupAddress', $id) === $address
+                || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address));
+        }
+        if (str_contains($entry, '@')) {
+            return $this->resolve($entry) === $address;
+        }
+        return $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry);
+    }
+
+    /** $block: a block entry, which also matches the envelope sender (refusing a forged sender is no risk) */
+    private function senderMatches(string $entry, string $sender, ?array $account, bool $block): bool
+    {
+        $address = $this->resolve(self::untagged($sender));
+        // the sender address of a local entry only counts when logged in: Postfix checked that the login may use it
+        $useAddress = $account !== null || $block;
+        if (str_starts_with($entry, 'group:')) {
+            $id = (int)substr($entry, 6);
+            return ($account !== null && $this->cached('isGroupMember', $id, $account['username']))
+                || ($useAddress && ($this->cached('groupAddress', $id) === $address
+                    || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address))));
+        }
+        if (str_contains($entry, '@')) {
+            return $this->cached('isLocalDomain', self::domainOf($entry))
+                ? ($account !== null && $account['username'] === $this->resolve($entry)) || ($useAddress && $address === $this->resolve($entry))
+                : self::untagged($sender) === $entry;
+        }
+        return $this->cached('isLocalDomain', $entry)
+            ? ($account !== null && $account['domain'] === $this->resolveDomain($entry))
+                || ($useAddress && $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry))
+            : self::domainOf($sender) === $entry;
+    }
+
+    /**
+     * Stored spelling of an address of this mailcow: Postfix finds chéf@ as chef@ (MySQL ignores accents). Only
+     * addresses with non-ASCII characters can differ, as they are already in lower case.
+     */
+    private function stored(string $address): string
+    {
+        if (!preg_match('/[\x80-\xff]/', $address) || !str_contains($address, '@')) {
+            return $address;
+        }
+        foreach (array_unique([$address, self::untagged($address)]) as $key) {
+            $stored = $this->cached('storedAddress', $key);
+            if ($stored !== null) {
+                return $stored;
+            }
+        }
+        $domain = $this->cached('storedDomain', self::domainOf($address));
+        return ($domain === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $domain;
+    }
+
+    /** Address in an alias domain as the address in its target domain */
+    private function resolve(string $address): string
+    {
+        $domain = self::domainOf($address);
+        $target = ($domain !== '') ? $this->cached('aliasDomainTarget', $domain) : null;
+        return ($target === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $target;
+    }
+
+    private function resolveDomain(string $domain): string
+    {
+        return ($domain === '') ? '' : ($this->cached('aliasDomainTarget', $domain) ?? $domain);
+    }
+
+    /** A lookup, done once per check: long lists ask the same domains and groups again */
+    private function cached(string $method, int|string ...$args): mixed
+    {
+        $key = $method . "\0" . implode("\0", $args);
+        if (!array_key_exists($key, $this->cache)) {
+            $this->cache[$key] = $this->lookup->$method(...$args);
+        }
+        return $this->cache[$key];
+    }
+
+    /** @param Policy[] $policies @return Policy[] without repeated policies */
+    private static function unique(array $policies): array
+    {
+        $unique = [];
+        foreach ($policies as $policy) {
+            $unique[$policy->id ?? 'internal:' . $policy->domain] = $policy;
+        }
+        return array_values($unique);
+    }
+
+    /** Lower case, without angle brackets, quotes and trailing dot of the domain, as Postfix resolves it; '' = null sender */
+    private static function address(string $value): string
+    {
+        $value = strtolower(trim($value, " \t\r\n<>"));
+        $at = strrpos($value, '@');
+        return ($at === false) ? $value : str_replace('"', '', substr($value, 0, $at)) . '@' . rtrim(substr($value, $at + 1), '.');
+    }
+
+    /**
+     * Postfix would deliver the recipient elsewhere: no domain, or routing through a local destination
+     * (localhost, the IP of the server) with %, ! or a second @ in the local part
+     */
+    private static function isRouted(string $address): bool
+    {
+        $at = strrpos($address, '@');
+        if ($at === false || $at === 0 || $at === strlen($address) - 1) {
+            return true;
+        }
+        $domain = substr($address, $at + 1);
+        return (in_array($domain, self::LOCAL_DESTINATIONS, true) || str_starts_with($domain, '['))
+            && strpbrk(substr($address, 0, $at), '@%!') !== false;
+    }
+
+    private static function untagged(string $address): string
+    {
+        $at = strrpos($address, '@');
+        $pos = ($at === false) ? false : strpos(substr($address, 0, $at), '+');
+        return ($pos === false || $pos === 0) ? $address : substr($address, 0, $pos) . substr($address, $at);
+    }
+
+    private static function domainOf(string $address): string
+    {
+        $at = strrpos($address, '@');
+        return ($at === false) ? '' : substr($address, $at + 1);
+    }
+}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyService.php

@@ -0,0 +1,351 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\MailPolicy\Service;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Mailcow\MailPolicy\Model\Decision;
+use Mailcow\MailPolicy\Model\Policy;
+use Mailcow\MailPolicy\Repository\PolicyRepository;
+
+/**
+ * Use cases of mail policies.
+ *
+ * Read: admins all policies (also read-only API keys); domain admins with the `mail_policies` ACL the policies
+ *       of their domains.
+ * Edit: admins; domain admins with the ACL for policies of their domains. A policy is assigned to mailboxes and
+ *       groups of its own domain; `group:<id>` entries name groups of that domain. Groups need the `mailbox_groups`
+ *       ACL as well: without it a domain admin sees no groups and keeps the group entries and assignments.
+ */
+class PolicyService
+{
+    private const MAX_ENTRIES = 500;
+    /** The reject text goes into the SMTP reply: printable ASCII */
+    private const MAX_REJECT_TEXT = 200;
+    private const LISTS = ['send_allow' => 'sendAllow', 'send_block' => 'sendBlock', 'receive_allow' => 'receiveAllow', 'receive_block' => 'receiveBlock'];
+
+    public function __construct(private PolicyRepository $repo, private PolicyEngine $engine, private Context $context)
+    {
+    }
+
+    public function canRead(): bool
+    {
+        return $this->context->hasAdminRights()
+            || ($this->context->isDomainAdmin() && $this->context->hasAcl('mail_policies'));
+    }
+
+    /** @throws AccessDenied unless the caller may edit policies at all */
+    public function requireEditor(): void
+    {
+        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
+            throw new AccessDenied();
+        }
+    }
+
+    public function get(int $id): ?Policy
+    {
+        $policy = $this->canRead() ? $this->repo->find($id) : null;
+        return ($policy !== null && $this->visible($policy)) ? $policy : null;
+    }
+
+    /** @return int[] */
+    public function visibleIds(array $domains = []): array
+    {
+        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
+    }
+
+    public function canEdit(Policy $policy): bool
+    {
+        return $this->canRead() && !$this->context->isReadOnlyApi() && $this->visible($policy);
+    }
+
+    public function toArray(Policy $policy): array
+    {
+        return $policy->toArray() + ['can_edit' => (int)$this->canEdit($policy)];
+    }
+
+    /** @throws UserError */
+    public function create(array $input): Policy
+    {
+        $this->requireEditor();
+        $policy = new Policy();
+        $policy->domain = $this->domainFor($input['domain'] ?? null);
+        $this->apply($policy, $input);
+        $this->repo->save($policy);
+        return $policy;
+    }
+
+    /** Attributes missing in $input keep their value @throws UserError */
+    public function update(int $id, array $input): Policy
+    {
+        $policy = $this->editable($id);
+        $this->apply($policy, $input);
+        $this->repo->save($policy);
+        return $policy;
+    }
+
+    /** @throws UserError */
+    public function delete(int $id): Policy
+    {
+        $policy = $this->editable($id);
+        $this->repo->delete($id);
+        return $policy;
+    }
+
+    /** Address entries of the lists follow a renamed mailbox or alias; called after the rename, without permission check */
+    public function addressRenamed(string $old, string $new): void
+    {
+        $this->repo->renameAddress(strtolower($old), strtolower($new));
+    }
+
+    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
+    public function groupChoices(array $domains): array
+    {
+        return ($this->canRead() && $this->canUseGroups()) ? $this->repo->groupsIn($domains) : [];
+    }
+
+    private function canUseGroups(): bool
+    {
+        return $this->context->hasAdminRights() || $this->context->hasAcl('mailbox_groups');
+    }
+
+    /**
+     * "Test a rule": would the mail be accepted? A sender that is a mailbox counts as logged in when $loggedIn.
+     *
+     * @return array{allowed: int, side: string, policy: ?string, internal_alias: int, text: string}
+     * @throws UserError
+     */
+    public function test(mixed $sender, mixed $recipient, bool $loggedIn): array
+    {
+        if (!$this->canRead()) {
+            throw new AccessDenied();
+        }
+        $sender = strtolower((string)Input::text($sender));
+        $recipient = strtolower((string)Input::text($recipient));
+        if (($sender !== '' && !filter_var($sender, FILTER_VALIDATE_EMAIL)) || !filter_var($recipient, FILTER_VALIDATE_EMAIL)) {
+            throw new UserError('mail_policy_test_invalid');
+        }
+        // domain admins test mail of their domains; a sender of another domain counts as not logged in
+        $ownSender = $this->context->hasAdminRights() || $this->context->canAccessDomain(self::domainOf($sender));
+        if (!$ownSender && !$this->context->canAccessDomain(self::domainOf($recipient))) {
+            throw new AccessDenied();
+        }
+        $login = ($loggedIn && $ownSender && $this->repo->mailboxDomain($sender) !== null) ? $sender : '';
+        $decision = $this->engine->check($sender, $recipient, $login);
+        $policy = $decision->policy;
+        $hidden = $policy !== null && $policy->id !== null && !$this->visible($policy);
+        return [
+            'allowed' => (int)$decision->allowed,
+            'side' => $decision->side,
+            'policy' => ($policy !== null && $policy->id !== null && !$hidden) ? $policy->name : null,
+            'internal_alias' => (int)($policy !== null && $policy->id === null),
+            'logged_in' => (int)($login !== ''),
+            // the text of a policy of another domain stays hidden
+            'text' => $decision->allowed ? '' : ($hidden ? Decision::defaultText($decision->side) : $decision->text()),
+        ];
+    }
+
+    private function visible(Policy $policy): bool
+    {
+        return $this->context->hasAdminRights() || $this->context->canAccessDomain($policy->domain);
+    }
+
+    /** @throws UserError */
+    private function editable(int $id): Policy
+    {
+        $this->requireEditor();
+        $policy = $this->get($id);
+        if ($policy === null || !$this->canEdit($policy)) {
+            throw new AccessDenied();
+        }
+        return $policy;
+    }
+
+    /** @throws UserError */
+    private function domainFor(mixed $value): string
+    {
+        $domain = strtolower((string)Input::text($value));
+        if ($domain === '' || !$this->repo->isDomain($domain)) {
+            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
+        }
+        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
+            throw new AccessDenied();
+        }
+        return $domain;
+    }
+
+    /** @throws UserError */
+    private function apply(Policy $policy, array $input): void
+    {
+        if (array_key_exists('name', $input) || $policy->id === null) {
+            $name = Input::text($input['name'] ?? null);
+            if ($name === null || $name === '' || mb_strlen($name) > 100 || preg_match('/[\x00-\x1f\x7f]/', $name)) {
+                throw new UserError('mail_policy_name_invalid');
+            }
+            if ($this->repo->nameExists($policy->domain, $name, $policy->id)) {
+                throw new UserError('object_exists', [htmlspecialchars($name)]);
+            }
+            $policy->name = $name;
+        }
+        if (array_key_exists('description', $input)) {
+            $description = Input::text($input['description']);
+            if ($description === null || mb_strlen($description) > 255 || preg_match('/[\x00-\x1f\x7f]/', $description)) {
+                throw new UserError('mail_policy_description_invalid');
+            }
+            $policy->description = $description;
+        }
+        foreach (['send_mode' => 'sendMode', 'receive_mode' => 'receiveMode'] as $key => $property) {
+            if (array_key_exists($key, $input)) {
+                $mode = Input::text($input[$key]);
+                $policy->$property = in_array($mode, Policy::MODES, true) ? $mode : throw new UserError('mail_policy_mode_invalid');
+            }
+        }
+        $groupAccess = $this->canUseGroups();
+        foreach (self::LISTS as $key => $property) {
+            // the UI sends the groups of a list separately (<list>_groups); the API may name them as group:<id>
+            if (!array_key_exists($key, $input) && !array_key_exists($key . '_groups', $input)) {
+                continue;
+            }
+            $entries = array_key_exists($key, $input) ? self::values($input[$key]) : self::withoutGroups($policy->$property);
+            $kept = [];
+            if (!$groupAccess) {
+                // without access to groups the group entries stay (also when sent back unchanged); the form shows no group choices
+                $kept = array_values(array_diff($policy->$property, self::withoutGroups($policy->$property)));
+                $entries = array_values(array_diff(array_map('strtolower', $entries), $kept));
+            } elseif (array_key_exists($key . '_groups', $input)) {
+                $entries = array_merge(self::withoutGroups($entries), array_map(fn(string $id): string => 'group:' . $id, self::values($input[$key . '_groups'])));
+            }
+            $entries = array_values(array_unique(array_merge($this->entries($entries, $policy->domain, $groupAccess), $kept)));
+            if (count($entries) > self::MAX_ENTRIES) {
+                throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
+            }
+            $policy->$property = $entries;
+        }
+        if (array_key_exists('reject_text', $input)) {
+            $text = Input::text($input['reject_text']);
+            if ($text === null || strlen($text) > self::MAX_REJECT_TEXT || preg_match('/[^\x20-\x7e]/', $text)) {
+                throw new UserError('mail_policy_reject_text_invalid');
+            }
+            $policy->rejectText = $text;
+        }
+        if (array_key_exists('active', $input)) {
+            $policy->active = (int)self::flag($input['active']);
+        }
+        if (array_key_exists('mailboxes', $input)) {
+            $policy->mailboxes = [];
+            foreach (self::limited($input['mailboxes']) as $username) {
+                $username = strtolower($username);
+                if ($this->repo->mailboxDomain($username) !== $policy->domain) {
+                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($username)]);
+                }
+                $policy->mailboxes[$username] = $username;
+            }
+            $policy->mailboxes = array_values($policy->mailboxes);
+        }
+        // without access to groups the assignments stay; only sending them back unchanged is accepted
+        if (array_key_exists('groups', $input) && !$groupAccess && self::values($input['groups']) !== []) {
+            $sent = array_map('intval', self::values($input['groups']));
+            $stored = $policy->groups;
+            sort($sent);
+            sort($stored);
+            if (array_values(array_unique($sent)) !== $stored) {
+                throw new AccessDenied();
+            }
+        }
+        if (array_key_exists('groups', $input) && $groupAccess) {
+            $policy->groups = [];
+            foreach (self::limited($input['groups']) as $value) {
+                $id = Input::intInRange($value, 1, PHP_INT_MAX);
+                if ($id === null || $this->repo->groupDomain($id) !== $policy->domain) {
+                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($value)]);
+                }
+                $policy->groups[$id] = $id;
+            }
+            $policy->groups = array_values($policy->groups);
+        }
+    }
+
+    /**
+     * Entries of a list: addresses, domains (a leading @ is dropped) and group:<id> of a group of the policy's
+     * domain, lower case, without repeats.
+     *
+     * @param string[] $values
+     * @return string[]
+     * @throws UserError
+     */
+    private function entries(array $values, string $domain, bool $groupAccess): array
+    {
+        $entries = [];
+        foreach (self::limited(array_map('strtolower', $values)) as $entry) {
+            if (str_starts_with($entry, 'group:')) {
+                $id = Input::intInRange(substr($entry, 6), 1, PHP_INT_MAX);
+                if (!$groupAccess || $id === null || $this->repo->groupDomain($id) !== $domain) {
+                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
+                }
+                $entry = 'group:' . $id;
+            } elseif (str_contains(ltrim($entry, '@'), '@')) {
+                if (!filter_var($entry, FILTER_VALIDATE_EMAIL)) {
+                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
+                }
+            } else {
+                $entry = ltrim($entry, '@');
+                if (!str_contains($entry, '.') || !filter_var($entry, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME)) {
+                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
+                }
+            }
+            $entries[$entry] = $entry;
+        }
+        return array_values($entries);
+    }
+
+    /**
+     * Values of a list field without repeats, at most MAX_ENTRIES (checked before anything is looked up)
+     *
+     * @return string[]
+     * @throws UserError
+     */
+    private static function limited(mixed $value): array
+    {
+        $values = array_values(array_unique(self::values($value)));
+        if (count($values) > self::MAX_ENTRIES) {
+            throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
+        }
+        return $values;
+    }
+
+    /** @return string[] entries without group:<id> */
+    private static function withoutGroups(array $entries): array
+    {
+        return array_values(array_filter($entries, fn(string $entry): bool => !str_starts_with(strtolower($entry), 'group:')));
+    }
+
+    /**
+     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
+     *
+     * @return string[]
+     */
+    private static function values(mixed $value): array
+    {
+        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
+        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
+    }
+
+    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
+    private static function flag(mixed $value): bool
+    {
+        if (is_array($value)) {
+            $value = end($value);
+        }
+        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
+    }
+
+    private static function domainOf(string $address): string
+    {
+        $at = strrpos($address, '@');
+        return ($at === false) ? '' : substr($address, $at + 1);
+    }
+}

data/web/inc/lib/mailcow/MailboxGroup/Model/Group.php

@@ -42,6 +42,8 @@ class Group
     public array $sendAs = [];
     /** @var array[] {owner, rights, subscribe, state, last_error}: members may open these mailboxes (ACL group=mcg_<id>) */
     public array $imapGrants = [];
+    /** @var string[] owners of removed IMAP grants whose ACL Dovecot still has, not part of the grants to write */
+    public array $removingImapOwners = [];
     /** @var array[] {owner, type, folder, access, subscribe, state, last_error}: SOGo folders shared with the members */
     public array $sogoGrants = [];
     /** @var array[] warnings of the last write (msg values), not stored */

data/web/inc/lib/mailcow/MailboxGroup/Repository/GroupRepository.php

@@ -501,6 +501,8 @@ class GroupRepository extends Repository
         $group->imapGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
             $this->query("SELECT `owner`, `rights`, `subscribe`, `state`, `last_error` FROM `mailbox_group_imap_grant`
               WHERE `group_id` = :id AND `state` != 'removing' ORDER BY `owner`", $id)->fetchAll(PDO::FETCH_ASSOC));
+        $group->removingImapOwners = $this->query("SELECT `owner` FROM `mailbox_group_imap_grant` WHERE `group_id` = :id AND `state` = 'removing'",
+            $id)->fetchAll(PDO::FETCH_COLUMN);
         $group->sogoGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
             $this->query("SELECT `owner`, `folder_type` AS `type`, `folder`, `access`, `subscribe`, `state`, `last_error`
               FROM `mailbox_group_sogo_grant` WHERE `group_id` = :id ORDER BY `owner`, `folder_type`, `folder`", $id)->fetchAll(PDO::FETCH_ASSOC));

data/web/inc/lib/mailcow/MailboxGroup/Service/GroupService.php

@@ -79,6 +79,12 @@ class GroupService
                 return false;
             }
         }
+        // A removed grant still opens the mailbox until Dovecot dropped its ACL
+        foreach ($group->removingImapOwners as $owner) {
+            if (!$this->context->canAccessMailbox((string)$owner)) {
+                return false;
+            }
+        }
         return true;
     }
 
@@ -113,7 +119,7 @@ class GroupService
         $current = $this->editable($id);
         $group = clone $current;
         $this->apply($group, $input, $current);
-        $snapshot = $this->effects->snapshot(array_merge($current->usernames(), $group->usernames()));
+        $snapshot = $this->effects->snapshot($this->inScope(array_merge($current->usernames(), $group->usernames())));
         $sogo = $this->repo->update($group);
         $group->warnings = array_merge($this->sizeWarnings($group), $this->effects->apply($sogo, $snapshot),
             $this->sharing->applyImap((int)$group->id), $this->sharing->applySogo((int)$group->id));
@@ -132,7 +138,7 @@ class GroupService
     {
         $this->requireEditor();
         $group = $this->editable($id);
-        $snapshot = $this->effects->snapshot($group->usernames());
+        $snapshot = $this->effects->snapshot($this->inScope($group->usernames()));
         $this->sharing->removeImap($group);
         $sogo = $this->repo->delete($id);
         $group->warnings = $this->effects->apply($sogo, $snapshot);
@@ -150,13 +156,30 @@ class GroupService
         if (!$own && !($this->canRead() && $this->context->canAccessMailbox($username))) {
             return [];
         }
+        $groups = $this->lookup->membershipsOf($username);
+        // Domain admins see the mailbox's groups in their domains, not those of other domains
+        if (!$own && !$this->context->hasAdminRights()) {
+            $groups = array_values(array_filter($groups, fn(array $group): bool => $this->context->canAccessDomain((string)$group['domain'])));
+        }
         return [
-            'groups' => $this->lookup->membershipsOf($username),
+            'groups' => $groups,
             'template' => $this->templates->effective([$username])[$username] ?? null,
             'send_as' => $this->lookup->sendAsOf($username),
         ];
     }
 
+    /**
+     * Members whose templates a change of the caller may apply: a domain admin never changes mailboxes outside
+     * his domains, their new effective template applies with the next admin action or IdP login.
+     */
+    private function inScope(array $usernames): array
+    {
+        if ($this->context->hasAdminRights()) {
+            return $usernames;
+        }
+        return array_values(array_filter($usernames, fn(string $username): bool => $this->context->canAccessMailbox($username)));
+    }
+
     /** @throws AccessDenied when the caller cannot see or edit the group */
     private function editable(int $id): Group
     {

data/web/inc/lib/mailcow/Retention/Controller/RetentionApiController.php

@@ -0,0 +1,168 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention\Controller;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\UserError;
+use Mailcow\Retention\Service\Planner;
+use Mailcow\Retention\Service\RuleService;
+use PDOException;
+
+/**
+ * HTTP API adapter for json_api.php, edit.php and user.php (routes `retention-rule`, `logs/retention`), and the
+ * plan and results of the nightly run (data/conf/rspamd/dynmaps/retention.php). Messages are logged as
+ * ['retention', action, 'rule', data].
+ */
+class RetentionApiController
+{
+    public function __construct(private RuleService $rules, private ApiResponse $response)
+    {
+    }
+
+    /** POST add/retention-rule */
+    public function addRule(array $attr): bool
+    {
+        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
+            $rule = $this->rules->create($attr);
+            return $this->response->success(['retention_rule_added', htmlspecialchars($rule->name)], $log);
+        });
+    }
+
+    /** POST edit/retention-rule: one message per id */
+    public function editRules(?array $items, array $attr): bool
+    {
+        $data = array_merge(['id' => $items], $attr);
+        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
+            $this->rules->requireEditor();
+            return $this->batch((array)($data['id'] ?? []), $log,
+                fn(int $id): array => ['retention_rule_modified', htmlspecialchars($this->rules->update($id, $attr)->name)]);
+        });
+    }
+
+    /** POST delete/retention-rule: one message per id */
+    public function deleteRules(array $items): bool
+    {
+        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
+            $this->rules->requireEditor();
+            return $this->batch($items, $log, fn(int $id): array => ['retention_rule_removed', htmlspecialchars($this->rules->delete($id)->name)]);
+        });
+    }
+
+    /** GET retention-rule/all[/<domain,domain>]: with the summary of the last nightly run */
+    public function getRules(?string $domains = null): array
+    {
+        $ids = $this->rules->visibleIds(self::domains($domains));
+        $runs = $this->rules->lastRuns($ids);
+        $rules = [];
+        foreach ($ids as $id) {
+            $rule = $this->rules->get($id);
+            if ($rule !== null) {
+                $rules[] = $this->rules->toArray($rule, $runs[$id] ?? null);
+            }
+        }
+        return $rules;
+    }
+
+    /** GET retention-rule/<id>, also edit.php */
+    public function getRule(mixed $id): array
+    {
+        $id = $this->toId($id);
+        $rule = ($id === null) ? null : $this->rules->get($id);
+        return ($rule === null) ? [] : $this->rules->toArray($rule);
+    }
+
+    /**
+     * GET retention-rule/effective/<username>: what the rules delete in the mailbox; [] when not allowed to ask.
+     * $describe: for people (user.php), see Planner::describe()
+     */
+    public function getEffective(mixed $username, bool $describe = false): array
+    {
+        try {
+            $actions = $this->rules->effective($username);
+            return $describe ? Planner::describe($actions) : $actions;
+        } catch (UserError) {
+            return [];
+        }
+    }
+
+    /** GET logs/retention[/<count>]: mailboxes with found mail or errors in the nightly runs, newest first */
+    public function getLog(int $limit = 0): array
+    {
+        return $this->rules->log($limit);
+    }
+
+    /** A page of the plan of the nightly run (retention.php): mailboxes with their actions, next: after for the next page */
+    public function plan(string $after = '', int $limit = 0): array
+    {
+        $page = $this->rules->plan($after, $limit);
+        $plan = [];
+        foreach ($page['mailboxes'] as $username => $actions) {
+            $plan[] = ['username' => (string)$username, 'actions' => $actions];
+        }
+        return ['mailboxes' => $plan, 'next' => $page['next']];
+    }
+
+    /** Results of the nightly run (retention.php): mailboxes with found mail or errors, a summary per rule, the run */
+    public function report(array $results, array $summary, ?array $run = null): int
+    {
+        return $this->rules->report($results, $summary, $run);
+    }
+
+    /** The domain was deleted (functions.mailbox.inc.php) */
+    public function domainDeleted(string $domain): void
+    {
+        $this->rules->domainDeleted($domain);
+    }
+
+    /**
+     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
+     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
+     */
+    private function handle(string $action, mixed $data, callable $work): bool
+    {
+        $log = ['retention', $action, 'rule', $data];
+        try {
+            $result = $work($log);
+        } catch (UserError $e) {
+            $result = $this->response->error($e, $log);
+        } catch (PDOException $e) {
+            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
+        }
+        $this->response->flush();
+        return $result;
+    }
+
+    /** One message per id: the success message returned by $work(id), or the UserError */
+    private function batch(array $ids, array $log, callable $work): bool
+    {
+        foreach ($ids as $id) {
+            try {
+                $this->response->success($work($this->requireId($id)), $log);
+            } catch (UserError $e) {
+                $this->response->error($e, $log);
+            }
+        }
+        return true;
+    }
+
+    /** @return string[] domains of a comma separated filter */
+    private static function domains(?string $domains): array
+    {
+        return array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
+    }
+
+    /** Integer id from request input ("12", 12); null for anything else */
+    private function toId(mixed $id): ?int
+    {
+        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
+    }
+
+    /** @throws AccessDenied for ids that cannot exist */
+    private function requireId(mixed $id): int
+    {
+        return $this->toId($id) ?? throw new AccessDenied();
+    }
+}

data/web/inc/lib/mailcow/Retention/Model/Rule.php

@@ -0,0 +1,96 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention\Model;
+
+/**
+ * A retention rule (table `retention_rule`): mail is deleted from its folders once it was saved there `days` days
+ * ago. Folders are special folders with all their names (trash: Trash, Papierkorb, Deleted Items, ...), the INBOX,
+ * all folders, or own folder names (`folder:<name>`, wildcard *). A rule applies to all mailboxes of its domain or
+ * to mailboxes and groups of it. In mode `simulate` the nightly run only counts what it would delete.
+ */
+class Rule
+{
+    /** Special folders, the INBOX and all folders; own folders are folder:<name> */
+    public const FOLDERS = ['trash', 'junk', 'sent', 'drafts', 'archive', 'inbox', 'all'];
+    public const MODES = ['simulate', 'delete'];
+    public const OWN_FOLDER = 'folder:';
+
+    public ?int $id = null;
+    public string $domain = '';
+    public string $name = '';
+    public string $description = '';
+    /** @var string[] keys of FOLDERS and folder:<name> */
+    public array $folders = [];
+    public int $days = 30;
+    /** Flagged mail stays */
+    public int $keepFlagged = 0;
+    public string $mode = 'simulate';
+    /** Applies to all mailboxes of the domain */
+    public int $allMailboxes = 0;
+    public int $active = 1;
+    /** @var string[] usernames the rule is assigned to */
+    public array $mailboxes = [];
+    /** @var int[] groups the rule is assigned to (their members of the rule's domain) */
+    public array $groups = [];
+    public ?string $created = null;
+    public ?string $modified = null;
+
+    /** Row of `retention_rule`; assignments are set by the repository */
+    public static function fromRow(array $row): self
+    {
+        $rule = new self();
+        $rule->id = (int)$row['id'];
+        $rule->domain = (string)$row['domain'];
+        $rule->name = (string)$row['name'];
+        $rule->description = (string)($row['description'] ?? '');
+        $folders = is_string($row['folders'] ?? null) ? json_decode($row['folders'], true) : null;
+        $rule->folders = is_array($folders) ? array_values(array_filter($folders, 'is_string')) : [];
+        $rule->days = (int)$row['days'];
+        $rule->keepFlagged = (int)$row['keep_flagged'];
+        $rule->mode = (string)$row['mode'];
+        $rule->allMailboxes = (int)$row['all_mailboxes'];
+        $rule->active = (int)$row['active'];
+        $rule->created = $row['created'] ?? null;
+        $rule->modified = $row['modified'] ?? null;
+        return $rule;
+    }
+
+    /** Column values for `retention_rule` */
+    public function toRow(): array
+    {
+        return [
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'description' => $this->description,
+            'folders' => json_encode($this->folders),
+            'days' => $this->days,
+            'keep_flagged' => $this->keepFlagged,
+            'mode' => $this->mode,
+            'all_mailboxes' => $this->allMailboxes,
+            'active' => $this->active,
+        ];
+    }
+
+    /** Entry of the API and the edit page */
+    public function toArray(): array
+    {
+        return [
+            'id' => $this->id,
+            'domain' => $this->domain,
+            'name' => $this->name,
+            'description' => $this->description,
+            'folders' => $this->folders,
+            'days' => $this->days,
+            'keep_flagged' => $this->keepFlagged,
+            'mode' => $this->mode,
+            'all_mailboxes' => $this->allMailboxes,
+            'active' => $this->active,
+            'mailboxes' => $this->mailboxes,
+            'groups' => $this->groups,
+            'created' => $this->created,
+            'modified' => $this->modified,
+        ];
+    }
+}

data/web/inc/lib/mailcow/Retention/Repository/RuleRepository.php

@@ -0,0 +1,351 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention\Repository;
+
+use Mailcow\Core\Context;
+use Mailcow\Core\Repository;
+use Mailcow\Retention\Model\Rule;
+use Mailcow\Retention\Service\Planner;
+use PDO;
+
+/**
+ * Tables `retention_rule`, `retention_rule_member` (the mailboxes and groups a rule is assigned to) and
+ * `retention_log` (results of the nightly run), plus the lookups the editing needs. Assignments go with their
+ * mailbox, group or domain (foreign keys, ON DELETE CASCADE).
+ */
+class RuleRepository extends Repository
+{
+    /** Results of the nightly run are kept this long */
+    private const LOG_DAYS = 90;
+
+    public function find(int $id): ?Rule
+    {
+        $row = $this->query("SELECT * FROM `retention_rule` WHERE `id` = :id", ['id' => $id])->fetch();
+        return is_array($row) ? $this->withMembers(Rule::fromRow($row)) : null;
+    }
+
+    /**
+     * @param string[] $domains limit to these domains, [] = all visible
+     * @return int[] ids of the rules visible to the caller, ordered by domain and name
+     */
+    public function visibleIds(Context $context, array $domains = []): array
+    {
+        [$where, $params] = $this->visibleDomains($context, $domains);
+        if ($where === null) {
+            return [];
+        }
+        return array_map('intval', $this->query("SELECT `id` FROM `retention_rule` WHERE $where ORDER BY `domain`, `name`", $params)
+            ->fetchAll(PDO::FETCH_COLUMN));
+    }
+
+    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `retention_rule` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
+            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
+    }
+
+    /** Inserts or updates the rule with its assignments; returns its id */
+    public function save(Rule $rule): int
+    {
+        return $this->transaction(function () use ($rule): int {
+            if ($rule->id === null) {
+                $rule->id = $this->insertRow('retention_rule', $rule->toRow());
+            } else {
+                $this->updateRow('retention_rule', $rule->id, $rule->toRow());
+                $this->query("DELETE FROM `retention_rule_member` WHERE `rule_id` = :id", ['id' => $rule->id]);
+            }
+            foreach ($rule->mailboxes as $username) {
+                $this->insertRow('retention_rule_member', ['rule_id' => $rule->id, 'username' => $username]);
+            }
+            foreach ($rule->groups as $groupId) {
+                $this->insertRow('retention_rule_member', ['rule_id' => $rule->id, 'group_id' => $groupId]);
+            }
+            return $rule->id;
+        });
+    }
+
+    public function delete(int $id): void
+    {
+        $this->query("DELETE FROM `retention_rule` WHERE `id` = :id", ['id' => $id]);
+    }
+
+    /** A primary domain (rules never belong to alias domains) */
+    public function isDomain(string $domain): bool
+    {
+        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
+    }
+
+    /** Domain of a regular mailbox, null if there is none */
+    public function mailboxDomain(string $username): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = ''", ['username' => $username])->fetchColumn();
+        return is_string($domain) ? $domain : null;
+    }
+
+    public function groupDomain(int $id): ?string
+    {
+        $domain = $this->query("SELECT `domain` FROM `mailbox_group` WHERE `id` = :id", ['id' => $id])->fetchColumn();
+        return is_string($domain) ? $domain : null;
+    }
+
+    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
+    public function groupsIn(array $domains): array
+    {
+        if (empty($domains)) {
+            return [];
+        }
+        [$in, $params] = self::in('dom', $domains);
+        return $this->query("SELECT `id`, `name`, `domain`, `address` FROM `mailbox_group` WHERE `domain` IN ($in) ORDER BY `domain`, `name`", $params)
+            ->fetchAll();
+    }
+
+    /**
+     * A page of the mailboxes the nightly run looks at: Dovecot knows them (regular, not disabled), their domain is
+     * active and has active rules.
+     *
+     * @return string[] usernames after $after, in order
+     */
+    public function mailboxesWithRules(string $after, int $limit): array
+    {
+        return $this->query("SELECT `m`.`username` FROM `mailbox` `m` JOIN `domain` `d` ON `d`.`domain` = `m`.`domain` AND `d`.`active` = '1'
+          WHERE `m`.`kind` = '' AND `m`.`active` IN ('1', '2') AND `m`.`username` > :after
+            AND EXISTS (SELECT 1 FROM `retention_rule` `r` WHERE `r`.`domain` = `m`.`domain` AND `r`.`active` = '1')
+          ORDER BY `m`.`username` LIMIT " . max(1, $limit), ['after' => $after])->fetchAll(PDO::FETCH_COLUMN);
+    }
+
+    /**
+     * Active rules of these mailboxes with the level of their assignment: the whole domain, a group of the mailbox
+     * (members of the rule's domain only) or the mailbox itself. Only mailboxes Dovecot knows (regular, not
+     * disabled) in active domains.
+     *
+     * @param string[] $usernames
+     * @return array<string, array<array{rule: Rule, level: int}>> username => candidates
+     */
+    public function candidates(array $usernames): array
+    {
+        if (empty($usernames)) {
+            return [];
+        }
+        $params = [];
+        $mailbox = [];
+        foreach ([1, 2, 3] as $part) {
+            [$in, $inParams] = self::in('u' . $part . '_', $usernames);
+            $params += $inParams;
+            $mailbox[$part] = "`m`.`kind` = '' AND `m`.`active` IN ('1', '2') AND `m`.`username` IN ($in)
+              AND `m`.`domain` IN (SELECT `domain` FROM `domain` WHERE `active` = '1')";
+        }
+        $rows = $this->query("SELECT `r`.*, `m`.`username` AS `member`, " . Planner::LEVEL_DOMAIN . " AS `level` FROM `retention_rule` `r`
+            JOIN `mailbox` `m` ON `m`.`domain` = `r`.`domain` AND {$mailbox[1]}
+            WHERE `r`.`active` = '1' AND `r`.`all_mailboxes` = '1'
+          UNION ALL SELECT `r`.*, `m`.`username`, " . Planner::LEVEL_GROUP . " FROM `retention_rule` `r`
+            JOIN `retention_rule_member` `rm` ON `rm`.`rule_id` = `r`.`id`
+            JOIN `mailbox_group` `g` ON `g`.`id` = `rm`.`group_id` AND `g`.`active` = '1'
+            JOIN `mailbox_group_member` `gm` ON `gm`.`group_id` = `g`.`id`
+            JOIN `mailbox` `m` ON `m`.`username` = `gm`.`username` AND `m`.`domain` = `r`.`domain` AND {$mailbox[2]}
+            WHERE `r`.`active` = '1'
+          UNION ALL SELECT `r`.*, `m`.`username`, " . Planner::LEVEL_MAILBOX . " FROM `retention_rule` `r`
+            JOIN `retention_rule_member` `rm` ON `rm`.`rule_id` = `r`.`id`
+            JOIN `mailbox` `m` ON `m`.`username` = `rm`.`username` AND `m`.`domain` = `r`.`domain` AND {$mailbox[3]}
+            WHERE `r`.`active` = '1'
+          ORDER BY `member`, `id`", $params)->fetchAll();
+        $candidates = [];
+        foreach ($rows as $row) {
+            $candidates[strtolower((string)$row['member'])][] = ['rule' => Rule::fromRow($row), 'level' => (int)$row['level']];
+        }
+        return $candidates;
+    }
+
+    /**
+     * Stores results of the nightly run. Domain and rule name come from the database, not from the runner; results
+     * of a rule from another domain than the mailbox are dropped. With the summary of a rule, the results its
+     * earlier runs found while simulating are dropped: only the last simulation is kept. After a run without error,
+     * the active rules that decided about no folder get an empty summary.
+     *
+     * @param array[] $results username, rule_id, folder (names), days, matched, deleted, simulated, error: mailboxes
+     *                         with found mail or an error
+     * @param array[] $summary rule_id, mailboxes, matched, deleted, errors, simulated: one per rule and run
+     * @param array|null $run  started (unix time), mailboxes, matched, deleted, errors, error: the whole run, sent at its end
+     */
+    public function addLog(array $results, array $summary, ?array $run = null): int
+    {
+        $stored = $this->transaction(function () use ($results, $summary, $run): int {
+            $stored = 0;
+            foreach ($results as $result) {
+                $domain = $this->mailboxDomain((string)$result['username']);
+                $rule = $this->ruleName((int)$result['rule_id']);
+                if ($domain === null || $rule === null || $rule['domain'] !== $domain) {
+                    continue;
+                }
+                $this->insertRow('retention_log', [
+                    'domain' => $domain,
+                    'username' => (string)$result['username'],
+                    'rule_id' => (int)$result['rule_id'],
+                    'rule_name' => $rule['name'],
+                    'folder' => mb_substr((string)$result['folder'], 0, 10000),
+                    'days' => (int)($result['days'] ?? 0),
+                    'matched' => (int)($result['matched'] ?? 0),
+                    'deleted' => (int)($result['deleted'] ?? 0),
+                    'errors' => (int)(($result['error'] ?? '') !== ''),
+                    'simulated' => (int)(bool)($result['simulated'] ?? false),
+                    'error' => ($result['error'] ?? '') === '' ? null : mb_substr((string)$result['error'], 0, 1000),
+                ]);
+                $stored++;
+            }
+            $summarized = [];
+            foreach ($summary as $runOfRule) {
+                $rule = $this->ruleName((int)$runOfRule['rule_id']);
+                if ($rule === null || isset($summarized[(int)$runOfRule['rule_id']])) {
+                    continue;
+                }
+                $summarized[(int)$runOfRule['rule_id']] = true;
+                $this->addSummary((int)$runOfRule['rule_id'], $rule, $runOfRule);
+                $stored++;
+            }
+            if ($run !== null && ($run['error'] ?? '') === '') {
+                // rules created during the run may have missed its first pages
+                foreach ($this->query("SELECT `r`.`id`, `r`.`domain`, `r`.`name`, `r`.`mode` FROM `retention_rule` `r`
+                  JOIN `domain` `d` ON `d`.`domain` = `r`.`domain` AND `d`.`active` = '1'
+                  WHERE `r`.`active` = '1' AND `r`.`created` < FROM_UNIXTIME(:started)", ['started' => (int)($run['started'] ?? 0)])->fetchAll() as $row) {
+                    if (!isset($summarized[(int)$row['id']])) {
+                        $this->addSummary((int)$row['id'], ['domain' => (string)$row['domain'], 'name' => (string)$row['name']],
+                            ['simulated' => $row['mode'] !== 'delete']);
+                        $stored++;
+                    }
+                }
+            }
+            if ($run !== null) {
+                $this->insertRow('retention_log', [
+                    'domain' => '',
+                    'mailboxes' => (int)($run['mailboxes'] ?? 0),
+                    'matched' => (int)($run['matched'] ?? 0),
+                    'deleted' => (int)($run['deleted'] ?? 0),
+                    'errors' => (int)($run['errors'] ?? 0),
+                    'error' => ($run['error'] ?? '') === '' ? null : mb_substr((string)$run['error'], 0, 1000),
+                ]);
+                $stored++;
+            }
+            return $stored;
+        });
+        if ($run !== null) {
+            $this->prune();
+        }
+        return $stored;
+    }
+
+    /** Summary of a rule's run; drops what its earlier runs found while simulating */
+    private function addSummary(int $ruleId, array $rule, array $values): void
+    {
+        $previous = $this->query("SELECT MAX(`id`) FROM `retention_log` WHERE `rule_id` = :id AND `username` = ''", ['id' => $ruleId])->fetchColumn();
+        if ($previous !== null && $previous !== false) {
+            $this->query("DELETE FROM `retention_log` WHERE `rule_id` = :id AND `username` != '' AND `simulated` = '1' AND `id` < :previous",
+                ['id' => $ruleId, 'previous' => (int)$previous]);
+        }
+        $this->insertRow('retention_log', [
+            'domain' => $rule['domain'],
+            'rule_id' => $ruleId,
+            'rule_name' => $rule['name'],
+            'mailboxes' => (int)($values['mailboxes'] ?? 0),
+            'matched' => (int)($values['matched'] ?? 0),
+            'deleted' => (int)($values['deleted'] ?? 0),
+            'errors' => (int)($values['errors'] ?? 0),
+            'simulated' => (int)(bool)($values['simulated'] ?? false),
+        ]);
+    }
+
+    /** Drops the results of the runs in a domain (the domain was deleted) */
+    public function deleteLog(string $domain): void
+    {
+        $this->query("DELETE FROM `retention_log` WHERE `domain` = :domain", ['domain' => $domain]);
+    }
+
+    /** @return array<int, array> rule id => the summary of its last run (created, mailboxes, matched, deleted, errors, simulated) */
+    public function lastRuns(array $ruleIds): array
+    {
+        if (empty($ruleIds)) {
+            return [];
+        }
+        [$in, $params] = self::in('rule', array_map('intval', $ruleIds));
+        $runs = [];
+        foreach ($this->query("SELECT `l`.`rule_id`, `l`.`created`, `l`.`mailboxes`, `l`.`matched`, `l`.`deleted`, `l`.`errors`, `l`.`simulated`
+          FROM `retention_log` `l` WHERE `l`.`username` = '' AND `l`.`rule_id` IN ($in) AND `l`.`id` = (SELECT MAX(`id`) FROM `retention_log`
+            WHERE `rule_id` = `l`.`rule_id` AND `username` = '')", $params)->fetchAll() as $row) {
+            $runs[(int)$row['rule_id']] = $row;
+        }
+        return $runs;
+    }
+
+    /**
+     * @return array[] mailboxes with found mail or errors in the nightly runs visible to the caller, and for admins the
+     *                 whole runs (username '', rule_id null), newest first
+     */
+    public function log(Context $context, array $domains, int $limit): array
+    {
+        [$where, $params] = $this->visibleDomains($context, $domains);
+        if ($where === null) {
+            return [];
+        }
+        return $this->query("SELECT UNIX_TIMESTAMP(`created`) AS `time`, `domain`, `username`, `rule_id`, `rule_name`, `folder`, `days`, `mailboxes`, `matched`, `deleted`, `errors`, `simulated`, `error`
+          FROM `retention_log` WHERE (`username` != '' OR `rule_id` IS NULL) AND $where ORDER BY `created` DESC, `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll();
+    }
+
+    /** Drops results older than LOG_DAYS, in parts so a large log does not hold the table */
+    private function prune(): void
+    {
+        do {
+            $deleted = $this->query("DELETE FROM `retention_log` WHERE `created` < NOW() - INTERVAL " . self::LOG_DAYS . " DAY LIMIT 10000")->rowCount();
+        } while ($deleted >= 10000);
+    }
+
+    /** @return array{domain: string, name: string}|null */
+    private function ruleName(int $id): ?array
+    {
+        $row = $this->query("SELECT `domain`, `name` FROM `retention_rule` WHERE `id` = :id", ['id' => $id])->fetch();
+        return is_array($row) ? ['domain' => (string)$row['domain'], 'name' => (string)$row['name']] : null;
+    }
+
+    /** @return array{0: ?string, 1: array} condition on `domain` for the caller and the domain filter, null = nothing visible */
+    private function visibleDomains(Context $context, array $domains): array
+    {
+        if ($context->hasAdminRights()) {
+            [$where, $params] = ['1=1', []];
+        } elseif ($context->isDomainAdmin()) {
+            [$where, $params] = ["`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
+                ['vis_da' => $context->username]];
+        } else {
+            return [null, []];
+        }
+        if (!empty($domains)) {
+            [$in, $inParams] = self::in('dom', $domains);
+            $where .= " AND `domain` IN ($in)";
+            $params += $inParams;
+        }
+        return [$where, $params];
+    }
+
+    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
+    private static function in(string $prefix, array $values): array
+    {
+        $names = [];
+        $params = [];
+        foreach (array_values($values) as $i => $value) {
+            $names[] = ':' . $prefix . $i;
+            $params[$prefix . $i] = $value;
+        }
+        return [implode(', ', $names), $params];
+    }
+
+    private function withMembers(Rule $rule): Rule
+    {
+        foreach ($this->query("SELECT `username`, `group_id` FROM `retention_rule_member` WHERE `rule_id` = :id ORDER BY `username`, `group_id`",
+            ['id' => $rule->id])->fetchAll() as $row) {
+            if ($row['username'] !== null) {
+                $rule->mailboxes[] = strtolower((string)$row['username']);
+            } elseif ($row['group_id'] !== null) {
+                $rule->groups[] = (int)$row['group_id'];
+            }
+        }
+        return $rule;
+    }
+}

data/web/inc/lib/mailcow/Retention/RetentionModule.php

@@ -0,0 +1,40 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention;
+
+use Mailcow\Core\ApiResponse;
+use Mailcow\Core\Context;
+use Mailcow\Retention\Controller\RetentionApiController;
+use Mailcow\Retention\Repository\RuleRepository;
+use Mailcow\Retention\Service\RuleService;
+use PDO;
+
+/**
+ * Composition root of the retention rules: which mail is deleted from which folders after how many days. The
+ * nightly run in the dovecot container (retention_runner.py) gets its plan from data/conf/rspamd/dynmaps/retention.php.
+ *
+ *   process_get_return(RetentionModule::fromGlobals()->api->getRules());
+ *   echo json_encode(RetentionModule::create($pdo, Context::system())->api->plan());  // nightly run
+ */
+class RetentionModule
+{
+    public function __construct(
+        public readonly RuleService $rules,
+        public readonly RetentionApiController $api
+    ) {
+    }
+
+    public static function create(PDO $pdo, Context $context): self
+    {
+        $rules = new RuleService(new RuleRepository($pdo), $context);
+        return new self($rules, new RetentionApiController($rules, new ApiResponse()));
+    }
+
+    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION */
+    public static function fromGlobals(?Context $context = null): self
+    {
+        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
+    }
+}

data/web/inc/lib/mailcow/Retention/Service/Planner.php

@@ -0,0 +1,112 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention\Service;
+
+use Mailcow\Retention\Model\Rule;
+
+/**
+ * What the rules of one mailbox delete: the most specific rule wins, per folder of the mailbox. The folders of the
+ * rules become entries in order: a rule assigned to the mailbox before one of its groups before one of the whole
+ * domain; then by what an entry matches (MATCHES, the folder itself first); then the longer period, then the lower
+ * id. The runner gives every folder to the first entry that matches it.
+ */
+class Planner
+{
+    public const LEVEL_DOMAIN = 1;
+    public const LEVEL_GROUP = 2;
+    public const LEVEL_MAILBOX = 3;
+
+    /**
+     * What an entry matches, most specific first: the folder itself (a special folder, the inbox, an own folder
+     * without *), an own folder name with *, the subfolders of a special folder or the inbox, all folders
+     */
+    public const MATCHES = ['folder' => 3, 'pattern' => 2, 'subfolders' => 1, 'all' => 0];
+
+    /**
+     * @param array<array{rule: Rule, level: int}> $candidates active rules of one mailbox with their assignment level
+     * @return array[] in order: folder (key), match, days, keep_flagged, mode, rule_id, rule_name. Entries that never
+     *                 win are left out: one matching the same folders as an entry before and everything after all folders.
+     */
+    public static function actions(array $candidates): array
+    {
+        $entries = [];
+        foreach ($candidates as $candidate) {
+            foreach ($candidate['rule']->folders as $folder) {
+                foreach (self::matches($folder) as $match) {
+                    // the lower id decides between equal rules, so the result does not depend on the order
+                    $entries[] = [
+                        'score' => [$candidate['level'], self::MATCHES[$match], $candidate['rule']->days, -(int)$candidate['rule']->id],
+                        'folder' => $folder,
+                        'match' => $match,
+                        'rule' => $candidate['rule'],
+                    ];
+                }
+            }
+        }
+        usort($entries, fn(array $a, array $b): int => $b['score'] <=> $a['score']);
+        $actions = [];
+        foreach ($entries as $entry) {
+            if (isset($actions[$entry['folder'] . "\n" . $entry['match']])) {
+                continue;
+            }
+            $actions[$entry['folder'] . "\n" . $entry['match']] = self::action($entry['folder'], $entry['match'], $entry['rule']);
+            if ($entry['match'] === 'all') {
+                break;
+            }
+        }
+        return array_values($actions);
+    }
+
+    /**
+     * The plan for people: the subfolders of a special folder or the inbox go with the folder (subfolders 1) when the
+     * same rule decides both and no entry with * comes between them.
+     *
+     * @return array[] the entries of actions() with subfolders 0 or 1
+     */
+    public static function describe(array $actions): array
+    {
+        $described = [];
+        $folders = [];
+        foreach ($actions as $action) {
+            $key = $action['folder'] . "\n" . $action['rule_id'];
+            if ($action['match'] === 'subfolders' && isset($folders[$key])) {
+                $described[$folders[$key]]['subfolders'] = 1;
+                continue;
+            }
+            if ($action['match'] === 'pattern') {
+                $folders = [];
+            } elseif ($action['match'] === 'folder') {
+                $folders[$key] = count($described);
+            }
+            $described[] = $action + ['subfolders' => (int)($action['match'] === 'subfolders')];
+        }
+        return $described;
+    }
+
+    /** @return string[] what the folder key of a rule matches */
+    private static function matches(string $folder): array
+    {
+        if ($folder === 'all') {
+            return ['all'];
+        }
+        if (str_starts_with($folder, Rule::OWN_FOLDER)) {
+            return [str_contains($folder, '*') ? 'pattern' : 'folder'];
+        }
+        return ['folder', 'subfolders'];
+    }
+
+    private static function action(string $folder, string $match, Rule $rule): array
+    {
+        return [
+            'folder' => $folder,
+            'match' => $match,
+            'days' => $rule->days,
+            'keep_flagged' => $rule->keepFlagged,
+            'mode' => $rule->mode,
+            'rule_id' => $rule->id,
+            'rule_name' => $rule->name,
+        ];
+    }
+}

data/web/inc/lib/mailcow/Retention/Service/RuleService.php

@@ -0,0 +1,359 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mailcow\Retention\Service;
+
+use Mailcow\Core\AccessDenied;
+use Mailcow\Core\Context;
+use Mailcow\Core\Input;
+use Mailcow\Core\UserError;
+use Mailcow\Retention\Model\Rule;
+use Mailcow\Retention\Repository\RuleRepository;
+
+/**
+ * Use cases of retention rules.
+ *
+ * Read: admins all rules (also read-only API keys); domain admins with the `retention` ACL the rules of their
+ *       domains. Mailbox users see the rules that apply to their mailbox (effective()).
+ * Edit: admins; domain admins with the ACL for rules of their domains. A rule is assigned to all mailboxes of its
+ *       domain or to mailboxes and groups of it. Groups need the `mailbox_groups` ACL as well: without it a domain
+ *       admin sees no groups and keeps the group assignments.
+ */
+class RuleService
+{
+    private const MAX_FOLDERS = 50;
+    private const MAX_MEMBERS = 500;
+    private const MAX_DAYS = 36500;
+    private const LOG_LIMIT = 1000;
+    /** Mailboxes per page of the plan, and results per report of the runner */
+    private const PLAN_PAGE = 500;
+    private const MAX_RESULTS = 1000;
+
+    public function __construct(private RuleRepository $repo, private Context $context)
+    {
+    }
+
+    public function canRead(): bool
+    {
+        return $this->context->hasAdminRights()
+            || ($this->context->isDomainAdmin() && $this->context->hasAcl('retention'));
+    }
+
+    /** @throws AccessDenied unless the caller may edit rules at all */
+    public function requireEditor(): void
+    {
+        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
+            throw new AccessDenied();
+        }
+    }
+
+    public function get(int $id): ?Rule
+    {
+        $rule = $this->canRead() ? $this->repo->find($id) : null;
+        return ($rule !== null && $this->visible($rule)) ? $rule : null;
+    }
+
+    /** @return int[] */
+    public function visibleIds(array $domains = []): array
+    {
+        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
+    }
+
+    public function canEdit(Rule $rule): bool
+    {
+        return $this->canRead() && !$this->context->isReadOnlyApi() && $this->visible($rule);
+    }
+
+    /** $lastRun: summary of the last nightly run, null = not run yet */
+    public function toArray(Rule $rule, ?array $lastRun = null): array
+    {
+        return $rule->toArray() + ['can_edit' => (int)$this->canEdit($rule), 'last_run' => $lastRun];
+    }
+
+    /** @return array<int, array> rule id => summary of its last nightly run */
+    public function lastRuns(array $ruleIds): array
+    {
+        return $this->canRead() ? $this->repo->lastRuns($ruleIds) : [];
+    }
+
+    /** @throws UserError */
+    public function create(array $input): Rule
+    {
+        $this->requireEditor();
+        $rule = new Rule();
+        $rule->domain = $this->domainFor($input['domain'] ?? null);
+        $this->apply($rule, $input);
+        $this->repo->save($rule);
+        return $rule;
+    }
+
+    /** Attributes missing in $input keep their value @throws UserError */
+    public function update(int $id, array $input): Rule
+    {
+        $rule = $this->editable($id);
+        $this->apply($rule, $input);
+        $this->repo->save($rule);
+        return $rule;
+    }
+
+    /** @throws UserError */
+    public function delete(int $id): Rule
+    {
+        $rule = $this->editable($id);
+        $this->repo->delete($id);
+        return $rule;
+    }
+
+    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
+    public function groupChoices(array $domains): array
+    {
+        return ($this->canRead() && $this->canUseGroups()) ? $this->repo->groupsIn($domains) : [];
+    }
+
+    /**
+     * What the rules delete in a mailbox, per folder (see Planner): for the mailbox user, or for those who may read
+     * the rules and access the mailbox.
+     *
+     * @return array[] actions of Planner::actions()
+     * @throws AccessDenied
+     */
+    public function effective(mixed $username): array
+    {
+        $username = strtolower((string)Input::text($username));
+        $allowed = $this->context->isUser() ? $username === strtolower((string)$this->context->username)
+            : ($this->canRead() && ($this->context->hasAdminRights() || $this->context->canAccessMailbox($username)));
+        if ($username === '' || !$allowed) {
+            throw new AccessDenied();
+        }
+        return Planner::actions($this->repo->candidates([$username])[$username] ?? []);
+    }
+
+    /** @return array[] results of the nightly runs in the caller's domains, newest first; $limit 0 = LOG_LIMIT */
+    public function log(int $limit = 0): array
+    {
+        return $this->canRead() ? $this->repo->log($this->context, [], ($limit > 0) ? min($limit, self::LOG_LIMIT * 10) : self::LOG_LIMIT) : [];
+    }
+
+    /**
+     * A page of the plan of the nightly run (system context), mailboxes after $after in order
+     *
+     * @return array{mailboxes: array<string, array[]>, next: ?string} username => actions of Planner::actions() for
+     *         the mailboxes with rules; next: the $after of the next page, null after the last
+     */
+    public function plan(string $after = '', int $limit = 0): array
+    {
+        $limit = ($limit > 0) ? min($limit, self::PLAN_PAGE) : self::PLAN_PAGE;
+        $usernames = $this->repo->mailboxesWithRules($after, $limit);
+        $plan = array_filter(array_map(fn(array $candidates): array => Planner::actions($candidates), $this->repo->candidates($usernames)));
+        return ['mailboxes' => $plan, 'next' => (count($usernames) < $limit) ? null : (string)end($usernames)];
+    }
+
+    /**
+     * Results of the nightly run (system context): mailboxes with found mail or errors, a summary per rule and at
+     * the end the whole run
+     *
+     * @throws UserError for more than MAX_RESULTS results
+     */
+    public function report(array $results, array $summary, ?array $run = null): int
+    {
+        if (count($results) > self::MAX_RESULTS || count($summary) > self::MAX_RESULTS * 10) {
+            throw new UserError('retention_entries_exceeded', [self::MAX_RESULTS]);
+        }
+        $results = array_filter($results, fn(mixed $result): bool => is_array($result) && is_string($result['username'] ?? null)
+            && is_numeric($result['rule_id'] ?? null) && is_string($result['folder'] ?? null));
+        $summary = array_filter($summary, fn(mixed $runOfRule): bool => is_array($runOfRule) && is_numeric($runOfRule['rule_id'] ?? null));
+        return $this->repo->addLog(array_values($results), array_values($summary), $run);
+    }
+
+    /** The domain was deleted: its rules went with it (foreign key), the results of its runs go now */
+    public function domainDeleted(string $domain): void
+    {
+        $this->repo->deleteLog($domain);
+    }
+
+    private function visible(Rule $rule): bool
+    {
+        return $this->context->hasAdminRights() || $this->context->canAccessDomain($rule->domain);
+    }
+
+    private function canUseGroups(): bool
+    {
+        return $this->context->hasAdminRights() || $this->context->hasAcl('mailbox_groups');
+    }
+
+    /** @throws UserError */
+    private function editable(int $id): Rule
+    {
+        $this->requireEditor();
+        $rule = $this->get($id);
+        if ($rule === null || !$this->canEdit($rule)) {
+            throw new AccessDenied();
+        }
+        return $rule;
+    }
+
+    /** @throws UserError */
+    private function domainFor(mixed $value): string
+    {
+        $domain = strtolower((string)Input::text($value));
+        if ($domain === '' || !$this->repo->isDomain($domain)) {
+            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
+        }
+        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
+            throw new AccessDenied();
+        }
+        return $domain;
+    }
+
+    /** @throws UserError */
+    private function apply(Rule $rule, array $input): void
+    {
+        if (array_key_exists('name', $input) || $rule->id === null) {
+            $name = Input::text($input['name'] ?? null);
+            if ($name === null || $name === '' || mb_strlen($name) > 100 || preg_match('/[\x00-\x1f\x7f]/', $name)) {
+                throw new UserError('retention_name_invalid');
+            }
+            if ($this->repo->nameExists($rule->domain, $name, $rule->id)) {
+                throw new UserError('object_exists', [htmlspecialchars($name)]);
+            }
+            $rule->name = $name;
+        }
+        if (array_key_exists('description', $input)) {
+            $description = Input::text($input['description']);
+            if ($description === null || mb_strlen($description) > 255 || preg_match('/[\x00-\x1f\x7f]/', $description)) {
+                throw new UserError('retention_description_invalid');
+            }
+            $rule->description = $description;
+        }
+        // the UI sends the special folders and the own folder names apart (own_folders, one per line or a list)
+        if (array_key_exists('folders', $input) || array_key_exists('own_folders', $input)) {
+            $folders = array_key_exists('folders', $input) ? self::values($input['folders'])
+                : array_filter($rule->folders, fn(string $folder): bool => in_array($folder, Rule::FOLDERS, true));
+            if (array_key_exists('own_folders', $input)) {
+                $folders = array_filter($folders, fn(string $folder): bool => !str_starts_with($folder, Rule::OWN_FOLDER));
+                $names = is_array($input['own_folders']) ? $input['own_folders'] : preg_split('/\r?\n/', (string)Input::text($input['own_folders']));
+                foreach ($names as $name) {
+                    if (!is_string($name)) {
+                        throw new UserError('retention_folder_invalid', ['']);
+                    }
+                    if (trim($name) !== '') {
+                        $folders[] = Rule::OWN_FOLDER . trim($name);
+                    }
+                }
+            }
+            $rule->folders = self::folders($folders);
+        }
+        if ($rule->folders === []) {
+            throw new UserError('retention_folders_invalid');
+        }
+        if (array_key_exists('days', $input) || $rule->id === null) {
+            $days = Input::intInRange($input['days'] ?? null, 1, self::MAX_DAYS);
+            $rule->days = $days ?? throw new UserError('retention_days_invalid', [self::MAX_DAYS]);
+        }
+        if (array_key_exists('mode', $input)) {
+            $mode = Input::text($input['mode']);
+            $rule->mode = in_array($mode, Rule::MODES, true) ? $mode : throw new UserError('retention_mode_invalid');
+        }
+        foreach (['keep_flagged' => 'keepFlagged', 'all_mailboxes' => 'allMailboxes', 'active' => 'active'] as $key => $property) {
+            if (array_key_exists($key, $input)) {
+                $rule->$property = (int)self::flag($input[$key]);
+            }
+        }
+        if (array_key_exists('mailboxes', $input)) {
+            $rule->mailboxes = [];
+            foreach (self::limited($input['mailboxes'], self::MAX_MEMBERS) as $username) {
+                $username = strtolower($username);
+                if ($this->repo->mailboxDomain($username) !== $rule->domain) {
+                    throw new UserError('retention_member_invalid', [htmlspecialchars($username)]);
+                }
+                $rule->mailboxes[$username] = $username;
+            }
+            $rule->mailboxes = array_values($rule->mailboxes);
+        }
+        // without access to groups the assignments stay; only sending them back unchanged is accepted
+        $groupAccess = $this->canUseGroups();
+        if (array_key_exists('groups', $input) && !$groupAccess && self::values($input['groups']) !== []) {
+            $sent = array_values(array_unique(array_map('intval', self::values($input['groups']))));
+            $stored = $rule->groups;
+            sort($sent);
+            sort($stored);
+            if ($sent !== $stored) {
+                throw new AccessDenied();
+            }
+        }
+        if (array_key_exists('groups', $input) && $groupAccess) {
+            $rule->groups = [];
+            foreach (self::limited($input['groups'], self::MAX_MEMBERS) as $value) {
+                $id = Input::intInRange($value, 1, PHP_INT_MAX);
+                if ($id === null || $this->repo->groupDomain($id) !== $rule->domain) {
+                    throw new UserError('retention_member_invalid', [htmlspecialchars($value)]);
+                }
+                $rule->groups[$id] = $id;
+            }
+            $rule->groups = array_values($rule->groups);
+        }
+    }
+
+    /**
+     * Folders of a rule: keys of Rule::FOLDERS and folder:<name>, without repeats. Names are Dovecot folder names
+     * (separator /, wildcard *), printable and at most 255 characters; INBOX is written in capitals like Dovecot does.
+     *
+     * @param string[] $values
+     * @return string[]
+     * @throws UserError
+     */
+    private static function folders(array $values): array
+    {
+        $folders = [];
+        foreach (self::limited($values, self::MAX_FOLDERS) as $folder) {
+            if (str_starts_with($folder, Rule::OWN_FOLDER)) {
+                $name = trim(substr($folder, strlen(Rule::OWN_FOLDER)), " \t/");
+                if ($name === '' || mb_strlen($name) > 255 || preg_match('/[\x00-\x1f\x7f]/u', $name) || !mb_check_encoding($name, 'UTF-8')) {
+                    throw new UserError('retention_folder_invalid', [htmlspecialchars($folder)]);
+                }
+                $name = preg_replace('~^inbox(?=/|$)~i', 'INBOX', $name);
+                $folder = Rule::OWN_FOLDER . $name;
+            } elseif (!in_array($folder, Rule::FOLDERS, true)) {
+                throw new UserError('retention_folder_invalid', [htmlspecialchars($folder)]);
+            }
+            $folders[$folder] = $folder;
+        }
+        return array_values($folders);
+    }
+
+    /**
+     * Values of a list field without repeats, at most $max (checked before anything is looked up)
+     *
+     * @return string[]
+     * @throws UserError
+     */
+    private static function limited(mixed $value, int $max): array
+    {
+        $values = array_values(array_unique(self::values($value)));
+        if (count($values) > $max) {
+            throw new UserError('retention_entries_exceeded', [$max]);
+        }
+        return $values;
+    }
+
+    /**
+     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks or commas.
+     *
+     * @return string[]
+     */
+    private static function values(mixed $value): array
+    {
+        $values = is_array($value) ? $value : preg_split('/[\r\n,;]+/', (string)Input::text($value));
+        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
+    }
+
+    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
+    private static function flag(mixed $value): bool
+    {
+        if (is_array($value)) {
+            $value = end($value);
+        }
+        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
+    }
+}

data/web/inc/restore.inc.php

@@ -0,0 +1,18 @@
+<?php
+// E-Mail > Restore of admins and domain admins (admin/restore.php, domainadmin/restore.php set $role after protect_route):
+// restores from the snapshots of the backup, also of deleted mailboxes (and domains for admins)
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/header.inc.php';
+$_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
+
+$js_minifier->add('/web/js/site/backup.js');
+
+$template = 'restore.twig';
+$template_data = [
+  'acl' => $_SESSION['acl'],
+  'role' => $role,
+  'lang_datatables' => json_encode($lang['datatables']),
+  'backup_configured' => \Mailcow\Backup\BackupModule::fromGlobals()->settings->load()->configured(),
+  'backup_timezone' => getenv('TZ') ?: 'UTC',
+];
+
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/inc/triggers.user.inc.php

@@ -19,8 +19,10 @@ if ($iam_provider){
       header('Location: ' . $redirect_uri);
       die();
     }
-  } elseif ($_GET['code'] && $_GET['state'] === $_SESSION['oauth2state']) {
+  } elseif (!empty($_GET['code']) && !empty($_SESSION['oauth2state']) && is_string($_GET['state'] ?? null)
-    // Check given state against previously stored one to mitigate CSRF attack
+    && hash_equals((string)$_SESSION['oauth2state'], $_GET['state'])) {
+    // Check given state against previously stored one to mitigate CSRF attack, a state is used once
+    unset($_SESSION['oauth2state']);
     // Received access token in $_GET['code']
     // extract info and verify user
     identity_provider('verify-sso');

data/web/inc/vars.inc.php

@@ -169,6 +169,12 @@ $SESSION_NAME = "MCSESSID";
 // Label for OTP devices
 $OTP_LABEL = "mailcow UI";
 
+// Failed second factor logins (TOTP, Yubico OTP, WebAuthn) of an account before its second factor is locked
+$TFA_MAX_FAILS = 10;
+
+// Time (in s) from the first failed second factor login until the counter and the lock are reset
+$TFA_FAIL_WINDOW = 900;
+
 // How long to wait (in s) for cURL Docker requests
 $DOCKER_TIMEOUT = 60;
 
@@ -291,13 +297,13 @@ $RSPAMD_MAPS = array(
 //   true   = option requires a non-empty value
 //   false  = pure flag, must NOT have a value
 //   string = PCRE (no delimiters) the whole non-empty value must match
-// Anything not listed here is rejected when a job is saved (the runner passes stored options as they are).
+// Anything not listed here is rejected when a job is saved; the runner passes stored options as they are, except the blocked ones below.
 // Override in vars.local.inc.php per option, e.g. $IMAPSYNC_OPTIONS['someoption'] = true; or unset($IMAPSYNC_OPTIONS['debug']);
 // The old format $IMAPSYNC_OPTIONS['whitelist'] (list of names) is no longer supported and ignored (logged).
 // NEVER add (RCE): 'delete2foldersonly', 'delete2foldersbutnot', 'regexflag', 'regexmess', 'pipemess', 'regextrans2', 'maxlinelengthcmd', 'oauthrefreshcmd1', 'oauthrefreshcmd2'
 // NEVER add options taking local file paths (e.g. 'logfile', 'logdir', 'pidfile', 'passfile1', 'sslargs1' with SSL_ca_file)
+// These, the tests* options (print the environment), 'showpasswords' and 'abort' (stops other runs) are also blocked in code
 $IMAPSYNC_OPTIONS_DEFAULT = array(
-  'abort'                  => false,
   'authmd51'               => false,
   'authuser1'              => true,
   'debug'                  => false,
@@ -373,9 +379,6 @@ $IMAPSYNC_OPTIONS_DEFAULT = array(
   'syncduplicates'         => false,
   'syncinternaldates'      => false,
   'synclabels'             => false,
-  'tests'                  => false,
-  'testslive'              => false,
-  'testslive6'             => false,
   'truncmess'              => true,
   'usecache'               => false,
   'useheader'              => true,

data/web/js/site/dashboard.js

@@ -875,6 +875,93 @@ jQuery(function($){
       hideTableExpandCollapseBtn('#tab-cron-logs', '#cron_log');
     });
   }
+  // results of the nightly retention runs: mailboxes with found mail or errors, and the runs (username '', no rule)
+  function draw_retention_logs() {
+    // just recalc width if instance already exists
+    if ($.fn.DataTable.isDataTable('#retention_log') ) {
+      $('#retention_log').DataTable().columns.adjust().responsive.recalc();
+      return;
+    }
+    var L = function(key) { return (lang.retention && lang.retention[key]) || ''; };
+    $('#retention_log').DataTable({
+      responsive: true,
+      processing: true,
+      serverSide: false,
+      stateSave: true,
+      pageLength: log_pagination_size,
+      dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" +
+           "tr" +
+           "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>",
+      language: lang_datatables,
+      order: [[0, 'desc']],
+      ajax: {
+        type: "GET",
+        url: "/api/v1/get/logs/retention",
+        dataSrc: function(data){
+          return Array.isArray(data) ? data : [];
+        }
+      },
+      columns: [
+        {
+          title: lang.time,
+          data: 'time',
+          defaultContent: '',
+          createdCell: function(td, cellData) {
+            createSortableDate(td, cellData)
+          }
+        },
+        {
+          title: L('mailbox'),
+          data: 'username',
+          defaultContent: '',
+          render: function(data, type, row) {
+            return row.rule_id === null ? '<i>' + escapeHtml(L('run')) + '</i>' : escapeHtml(data);
+          }
+        },
+        {
+          title: L('rule'),
+          data: 'rule_name',
+          defaultContent: '',
+          render: function(data) {
+            return escapeHtml(data);
+          }
+        },
+        {
+          title: L('folder'),
+          data: 'folder',
+          defaultContent: '',
+          className: 'text-break',
+          render: function(data, type, row) {
+            if (row.rule_id === null) return escapeHtml(L('run_mailboxes').replace('%s', row.mailboxes));
+            return (data || '').split('\n').map(escapeHtml).join('<br>') + ' <span class="text-muted small">(' + escapeHtml(L('days_short').replace('%s', row.days)) + ')</span>';
+          }
+        },
+        {
+          title: L('found'),
+          data: 'matched',
+          defaultContent: ''
+        },
+        {
+          title: L('deleted'),
+          data: 'deleted',
+          defaultContent: '',
+          render: function(data, type, row) {
+            return row.simulated == 1 ? '<span class="badge bg-secondary">' + escapeHtml(L('simulated')) + '</span>' : escapeHtml(data);
+          }
+        },
+        {
+          title: L('error'),
+          data: 'error',
+          defaultContent: '',
+          className: 'dtr-col-md text-break',
+          render: function(data, type, row) {
+            if (!data && row.rule_id === null && row.errors > 0) data = L('errors').replace('%s', row.errors);
+            return data ? '<span class="text-danger">' + escapeHtml(data) + '</span>' : '';
+          }
+        }
+      ]
+    });
+  }
   function rspamd_pie_graph() {
     $.ajax({
       url: '/api/v1/get/rspamd/actions',
@@ -1295,6 +1382,7 @@ jQuery(function($){
   onVisible("[id^=sasl_logs]", () => draw_sasl_logs());
   onVisible("[id^=netfilter_log]", () => draw_netfilter_logs());
   onVisible("[id^=cron_log]", () => draw_cron_logs());
+  onVisible("[id^=retention_log]", () => draw_retention_logs());
   onVisible("[id^=rspamd_history]", () => draw_rspamd_history());
   onVisible("[id^=rspamd_donut]", () => rspamd_pie_graph());