update postscreen_access.cidr

_modules/scripts/backup.sh

@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# _modules/scripts/backup.sh
-# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
-# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
-
-# Backup with restic before the update when it is set up (target, confirmed password, initialized repository):
-# interactive: asked for; --backup: without asking; --force: none (with --backup: without asking, a failure or a backup
-# that cannot run stops the update; only "not set up" continues)
-backup_failed() {
-  echo -e "\e[31m${1}\e[0m"
-  if [ "$FORCE" ]; then
-    echo "The update was started with --force --backup, it stops here. Nothing was changed."
-    exit 1
-  fi
-  read -r -p "Continue the update without a backup? [y/N] " backup_response
-  if [[ ! "${backup_response}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-    echo "OK, exiting."
-    exit 1
-  fi
-}
-
-# Whether to back up (asked with the other questions); the backup itself runs before the first change of the checkout
-backup_decide() {
-  local output rc
-  RUN_BACKUP=
-  if [ "$FORCE" ] && [ ! "$BACKUP" ]; then
-    return 0
-  fi
-  if [[ -z "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
-    output="the backup container does not run"
-    rc=1
-  else
-    # exit code 0: set up, 2: not set up, others: error (e.g. credentials this container cannot read)
-    output=$($COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --configured 2>&1 < /dev/null)
-    rc=$?
-  fi
-  if [[ ${rc} -eq 2 ]]; then
-    [ "$BACKUP" ] && echo -e "\e[33mNo backup before the update: ${output}.\e[0m"
-    return 0
-  elif [[ ${rc} -ne 0 ]]; then
-    if [ "$BACKUP" ]; then
-      backup_failed "No backup before the update possible: ${output}"
-    elif [[ -n "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
-      echo -e "\e[33mNo backup before the update: ${output}\e[0m"
-    fi
-    return 0
-  fi
-  if [ ! "$BACKUP" ]; then
-    read -r -p "Create a backup before the update? [Y/n] " backup_response
-    if [[ "${backup_response}" =~ ^([nN][oO]|[nN])$ ]]; then
-      echo "OK, no backup."
-      return 0
-    fi
-  fi
-  RUN_BACKUP=y
-}
-
-# The backup chosen by backup_decide, once
-backup_run() {
-  if [ ! "$RUN_BACKUP" ]; then
-    return 0
-  fi
-  RUN_BACKUP=
-  if [[ "${RESTIC_BACKUP_MODE}" =~ ^(data|config)$ ]]; then
-    echo -e "\e[33mRESTIC_BACKUP_MODE=${RESTIC_BACKUP_MODE}: the backup holds no complete mailcow directory, a restore does not bring back this mailcow version.\e[0m"
-  fi
-  echo -e "\e[32mCreating a backup before the update...\e[0m"
-  if ! $COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --backup < /dev/null; then
-    backup_failed "The backup before the update failed, see the output above and the backup jobs in the mailcow UI."
-  fi
-}

_modules/scripts/core.sh

@@ -192,7 +192,6 @@ detect_major_update() {
       "2025-02"
       "2025-03"
       "2025-09"
-      "2026-11"
     )
 
     current_version=""

_modules/scripts/new_options.sh

@@ -60,9 +60,6 @@ adapt_new_options() {
   "ACME_DNS_CHALLENGE"
   "ACME_DNS_PROVIDER"
   "ACME_ACCOUNT_EMAIL"
-  "NGINX_MAX_BODY_SIZE"
-  "RESTIC_LOCAL_PATH"
-  "RESTIC_BACKUP_MODE"
   )
 
   sed -i --follow-symlinks '$a\' mailcow.conf
@@ -312,24 +309,6 @@ adapt_new_options() {
             echo '# Account email for ACME DNS-01 challenge registration' >> mailcow.conf
             echo 'ACME_ACCOUNT_EMAIL=me@example.com' >> mailcow.conf
             ;;
-        NGINX_MAX_BODY_SIZE)
-            echo '# Maximum request size for SOGo webmail, CalDAV/CardDAV and ActiveSync, e.g. 128m or 1g - 0 means unlimited' >> mailcow.conf
-            echo '# Keep it above the message size limit of Postfix (100 MB by default). The mailcow UI and API stay limited to 64m.' >> mailcow.conf
-            echo 'NGINX_MAX_BODY_SIZE=128m' >> mailcow.conf
-            ;;
-        RESTIC_LOCAL_PATH)
-            echo '# Folder on the host for backups to the target "local" (backup settings in the mailcow UI), as an absolute path,' >> mailcow.conf
-            echo '# e.g. a NAS mounted from another disk. The repository is created in a subfolder of it.' >> mailcow.conf
-            echo '# Empty: the Docker volume restic-repo-vol-1 of mailcow, on the same disk as the mailcow data.' >> mailcow.conf
-            echo 'RESTIC_LOCAL_PATH=' >> mailcow.conf
-            ;;
-        RESTIC_BACKUP_MODE)
-            echo '# What the backups of the mailcow UI hold:' >> mailcow.conf
-            echo '# - all (default),' >> mailcow.conf
-            echo '# - data (vmail, vmail-index, database, redis, mail crypt keys and mailcow.conf)' >> mailcow.conf
-            echo '# - config (database, redis, mail crypt keys, mailcow.conf and data/conf).' >> mailcow.conf
-            echo 'RESTIC_BACKUP_MODE=all' >> mailcow.conf
-            ;;
         *)
             echo "${option}=" >> mailcow.conf
             ;;

data/Dockerfiles/dockerapi/main.py

@@ -159,9 +159,6 @@ async def post_containers(container_id : str, post_action : str, request: Reques
       api_call_method = getattr(dockerapi, api_call_method_name, lambda container_id: Response(content=json.dumps({'type': 'danger', 'msg':'container_post - unknown api call' }, indent=4), media_type="application/json"))
 
       dockerapi.logger.info("api call: %s, container_id: %s" % (api_call_method_name, container_id))
-      if getattr(api_call_method, 'long_running', False):
-        # runs for minutes (e.g. sogo-tool of a backup restore): in a worker thread, the other api calls go on
-        return await asyncio.to_thread(api_call_method, request_json, container_id=container_id)
       return api_call_method(request_json, container_id=container_id)
     except Exception as e:
       dockerapi.logger.error("error - container_post: %s" % str(e))

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -6,15 +6,9 @@ import time
 import json
 import asyncio
 import platform
-import docker
 from datetime import datetime
 from fastapi import FastAPI, Response, Request
 
-def long_running(method):
-  """Marks an api call that runs for minutes: main.py runs it in a worker thread, the other api calls go on"""
-  method.long_running = True
-  return method
-
 class DockerApi:
   def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
     self.redis_client = redis_client
@@ -324,7 +318,7 @@ class DockerApi:
 
     if 'username' in request_json and 'script_name' in request_json:
       for container in self.sync_docker_client.containers.list(filters=filters):
-        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' -- '" + request_json['script_name'].replace("'", "'\\''") + "'"]
+        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]
         sieve_return = container.exec_run(cmd)
         return self.exec_run_handler('utf8_text_only', sieve_return)
   # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
@@ -451,35 +445,6 @@ class DockerApi:
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
       init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
       return self.exec_run_handler('generic', init_return)
-  # api call: container_post - post_action: exec - cmd: sogo - task: restore
-  # Resets calendars, address books and preferences of a user to the sogo-tool backup file <dir>/<username> in a
-  # restore batch of the backup (/restore/<job>/b<batch>/sogo_backup, only the sogo user can read it); values are
-  # checked and passed as arguments, never through the shell. Runs in a worker thread (main.py) with its own client.
-  @long_running
-  def container_post__exec__sogo__restore(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    directory = str(request_json.get('dir', ''))
-    username = str(request_json.get('username', ''))
-    if (not re.fullmatch(r'/restore/[0-9]+/b[0-9]+/sogo_backup', directory)
-      or not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+', username)):
-      return Response(content=json.dumps({ 'type': 'danger', 'msg': 'dir and username required' }, indent=4), media_type="application/json")
-    # the folder: no symlink, uid 999, mode 0700; the file: plain, uid 999, mode 0600. sogo-tool restores folders and
-    # preferences in two runs, both stopped before the runner gives up waiting (30 minutes)
-    script = ('[ -d "$1" ] && [ ! -L "$1" ] && [ "$(realpath -e -- "$1")" = "$1" ] && [ "$(stat -c "%u %a" -- "$1")" = "999 700" ] '
-      '&& [ -f "$1/$2" ] && [ ! -L "$1/$2" ] && [ "$(stat -c "%u %a" -- "$1/$2")" = "999 600" ] '
-      '|| { echo "invalid directory or file"; exit 1; }; '
-      'timeout -k 30 850 sogo-tool restore -f ALL "$1" "$2" && exec timeout -k 30 850 sogo-tool restore -p "$1" "$2"')
-    client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version='auto', timeout=1800)
-    try:
-      for container in client.containers.list(filters=filters):
-        restore_return = container.exec_run(["/bin/bash", "-c", script, "restore", directory, username], user='sogo')
-        return self.exec_run_handler('generic', restore_return)
-    finally:
-      client.close()
   # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
   # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
   def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
@@ -526,7 +491,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' -- '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -545,7 +510,7 @@ class DockerApi:
           if mailbox in mailbox_seen:
             continue
 
-          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' -- '{mailbox}'"])
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' '{mailbox}'"])
           acls = acls.output.decode('utf-8').strip().splitlines()
           if len(acls) >= 2:
             for acl in acls[1:]:
@@ -569,7 +534,7 @@ class DockerApi:
       id = request_json['id'].replace("'", "'\\''")
 
       if user and mailbox and id:
-        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}'"])
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' '{self.acl_identifier(id)}'"])
         return self.exec_run_handler('generic', acl_delete_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
   def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
@@ -603,7 +568,7 @@ class DockerApi:
           rights += right + " "
 
       if user and mailbox and id and rights:
-        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' -- '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' '{self.acl_identifier(id)}' {rights}"])
         return self.exec_run_handler('generic', acl_set_return)
   # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl_all
   # Sets the rights of an identifier on every folder of a mailbox, no rights = delete; values are passed as
@@ -620,7 +585,7 @@ class DockerApi:
       rights = [right.lower() for right in request_json.get('rights', []) if right.lower() in self.acl_rights]
       if not user or id in ('user=', 'group='):
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
-      action = 'doveadm acl set -u "$ACL_USER" -- "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" -- "$folder" "$ACL_ID"'
+      action = 'doveadm acl set -u "$ACL_USER" "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" "$folder" "$ACL_ID"'
       script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
         'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
       acl_return = container.exec_run(["/bin/bash", "-c", script],
@@ -636,7 +601,7 @@ class DockerApi:
 
     for container in self.sync_docker_client.containers.list(filters=filters):
       users = [str(user) for user in request_json.get('users', []) if str(user)]
-      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush", "--"] + users)
+      flush_return = container.exec_run(["doveadm", "auth", "cache", "flush"] + users)
       return self.exec_run_handler('generic', flush_return)
 
   # Rights doveadm acl set accepts from the API

data/Dockerfiles/dovecot/Dockerfile

@@ -150,8 +150,6 @@ COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
 COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
-COPY retention_runner.py /usr/local/bin/retention_runner.py
-COPY restore_runner.py /usr/local/bin/restore_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -281,8 +281,6 @@ chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
   /usr/local/bin/imapsync_runner.py \
-  /usr/local/bin/retention_runner.py \
-  /usr/local/bin/restore_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \
@@ -308,20 +306,13 @@ IMAPSYNC_TABLE=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DB
 echo "$(envsubst < /usr/local/bin/maildir_gc.sh)" > /usr/local/bin/maildir_gc.sh
 
 # Wait for updated schema (DB_VERSION from docker-compose.yml), the userdb query reads tables the init_db of php-fpm creates on updates
-# No error while the versions table is missing (fresh install, set -e)
+DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; do
-DBV_WAIT=0
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
   echo "Waiting for schema update..."
   sleep 5
-  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null || true)
 done
-if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
-  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
-else
 echo "DB schema is ${DBV_NOW}"
-fi
 
 # GUID generation
 while [[ ${VERSIONS_OK} != 'OK' ]]; do

data/Dockerfiles/dovecot/imapsync_runner.py

@@ -1,12 +1,10 @@
 #!/usr/bin/python3
 
 import fcntl
-import ipaddress
 import json
 import os
 import re
 import signal
-import socket
 import subprocess
 import sys
 import tempfile
@@ -20,15 +18,6 @@ max_output = 16 * 1024 * 1024
 lock_file = run_dir + '/imapsync_busy.lock'
 # An OAuth run needs a token valid this long at start (half the lifetime for short-lived tokens)
 token_min_valid = 900
-# Custom params never passed, also when stored before (same list as CustomParams::BLOCKED in the web UI)
-blocked_params = {'abort', 'delete2foldersbutnot', 'delete2foldersonly', 'logdir', 'logfile', 'maxlinelengthcmd',
-  'oauthrefreshcmd1', 'oauthrefreshcmd2', 'passfile1', 'passfile2', 'pidfile', 'pipemess', 'regexflag', 'regexmess',
-  'regextrans2', 'showpasswords', 'skipmess', 'tests', 'testsdebug', 'testslive', 'testslive6', 'testsunit', 'tmpdir'}
-# imapsync only gets these variables, not the secrets of the container environment
-job_env_keys = ('PATH', 'LANG', 'LC_ALL', 'TZ')
-# A source must not point to mailcow itself (internal Docker network), host1 is checked before each run
-internal_nets = [ipaddress.ip_network(os.environ.get('IPV4_NETWORK', '172.22.1') + '.0/24', strict=False),
-  ipaddress.ip_network(os.environ.get('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'), strict=False)]
 # Refreshed tokens are written to the token files of running jobs this often, imapsync re-reads them on reconnect
 token_sync_interval = 60
 
@@ -36,7 +25,7 @@ token_sync_interval = 60
 next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
   i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
   i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
-  s.host1, s.port1, s.enc1, s.verify_cert, s.auth_type, s.oauth_flow
+  s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow
     FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
       WHERE i.active = 1
         AND s.active = 1
@@ -117,28 +106,6 @@ def defer_job(job_id, text, exit_status):
     (text, exit_status, job_id))
   deferred.add(job_id)
 
-# True when host resolves to loopback, link-local or the internal network; unknown hosts are left to imapsync
-def internal_host(host):
-  try:
-    addrs = {a[4][0] for a in socket.getaddrinfo(host, None)}
-  except (socket.gaierror, UnicodeError):
-    return False
-  for addr in addrs:
-    ip = ipaddress.ip_address(addr.split('%')[0])
-    if ip.version == 6 and ip.ipv4_mapped:
-      ip = ip.ipv4_mapped
-    if ip.is_loopback or ip.is_unspecified or ip.is_link_local or any(ip in net for net in internal_nets):
-      return True
-  return False
-
-# SNI takes host names only
-def ip_literal(host):
-  try:
-    ipaddress.ip_address(host)
-    return True
-  except ValueError:
-    return False
-
 def oauth_token(j):
   if j['oauth_flow'] == 'authorization_code':
     return query(user_token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
@@ -183,11 +150,6 @@ def prepare_job(j, global_bw, files):
       f.write(content)
     return path
 
-  if internal_host(j['host1']):
-    finish_job(j['id'], 'Host1 %s is an address of mailcow itself, the sync job does not connect to it\n' % j['host1'], 0,
-      'EXIT_CONNECTION_FAILURE_HOST1')
-    return None
-
   auth_type = j['auth_type'] or 'PLAIN'
   oauth = None
   if auth_type == 'XOAUTH2':
@@ -212,11 +174,7 @@ def prepare_job(j, global_bw, files):
   except ValueError:
     pairs = []
   for p in pairs if isinstance(pairs, list) else []:
-    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']) \
+    if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
-        or p['o'] in blocked_params:
-      continue
-    # The certificate check protects the OAuth token of the source, a job must not turn it off
-    if oauth and p['o'] == 'sslargs1':
       continue
     value = '' if p.get('v') is None else str(p['v'])
     custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
@@ -247,13 +205,7 @@ def prepare_job(j, global_bw, files):
       cmd.append(opt)
   enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
   if enc1:
-    # Mail::IMAPClient starts TLS without a host name: check the certificate against host1, not its address
+    cmd.append(enc1)
-    cmd += [enc1, '--sslargs1', 'SSL_verifycn_scheme=imap', '--sslargs1', 'SSL_verifycn_name=' + j['host1']]
-    if not ip_literal(j['host1']):
-      cmd += ['--sslargs1', 'SSL_hostname=' + j['host1']]
-    # imapsync does not check certificates by default; custom sslargs1 of password jobs can still change it
-    if j['verify_cert'] == 1:
-      cmd += ['--sslargs1', 'SSL_verify_mode=1']
   cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
   cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
   if j['dry'] == 1:
@@ -264,10 +216,8 @@ def prepare_job(j, global_bw, files):
 
 # Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
 def start_job(job):
-  env = {k: os.environ[k] for k in job_env_keys if k in os.environ}
-  env['HOME'] = run_dir
   with open(job['outfile'], 'wb') as out:
-    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, env=env)
+    return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
 
 # Store the result of a finished worker and remove its temp files
 def reap_job(job):

data/Dockerfiles/dovecot/restore_runner.py

@@ -1,712 +0,0 @@
-#!/usr/bin/python3
-# Mail part of restores (ofelia, dovecot container, as vmail): the restic container restores a batch of mailboxes
-# into /restore/<job id>/b<n> and sets the job to step 'mails'. Per mailbox this imports the mails folder by folder
-# (into "Restored <snapshot time>" or only the missing ones into the original folders), adds IMAP ACLs and missing
-# Sieve scripts, merges the results into the job and sets step 'mails_done'; the restic runner then removes the staging
-# and goes on with the next batch. Nothing that exists is changed or deleted.
-
-import base64
-import fcntl
-import json
-import os
-import re
-import smtplib
-import stat
-import subprocess
-import sys
-import threading
-import time
-from email.message import EmailMessage
-from email.utils import formatdate, make_msgid
-import MySQLdb
-import MySQLdb.cursors
-
-lock_file = '/tmp/restore_runner.lock'
-restore_dir = '/restore'
-max_log = 1000000
-# import, moves and the removal of the temporary folder ignore the quota (a restore may need more than is free),
-# the quota is recalculated afterwards
-no_quota = ['-o', 'plugin/quota=']
-# uid ranges per import (one argument stays far below the 128 KiB limit), folders per fetch and status
-uid_ranges = 2000
-folder_batch = 200
-# the restic runner waits for the batch while this heartbeat is fresh
-heartbeat_interval = 30
-# seconds to wait for the next batch of a restore
-batch_wait = 120
-rights_by_letter = {'l': 'lookup', 'r': 'read', 'w': 'write', 's': 'write-seen', 't': 'write-deleted', 'i': 'insert',
-  'p': 'post', 'e': 'expunge', 'k': 'create', 'x': 'delete', 'a': 'admin'}
-components_here = {'mails', 'imap_acl', 'filters', 'sieve_activate'}
-
-conn = None
-
-
-def connect():
-  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
-    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
-
-# Reconnect when the server dropped the connection (imports can outlast wait_timeout)
-def query(sql, args=None):
-  global conn
-  try:
-    conn.ping()
-  except (AttributeError, MySQLdb.Error):
-    conn = connect()
-  cur = conn.cursor(MySQLdb.cursors.DictCursor)
-  cur.execute(sql, args)
-  return cur
-
-# state_restore_seen while a batch runs, in its own connection
-class Heartbeat(threading.Thread):
-  def __init__(self):
-    super().__init__(daemon=True)
-    self.done = threading.Event()
-
-  def run(self):
-    db = None
-    while not self.done.is_set():
-      try:
-        if db is None:
-          db = connect()
-        db.cursor().execute("""INSERT INTO backup_setting (name, value) VALUES ('state_restore_seen', %s)
-          ON DUPLICATE KEY UPDATE value = VALUES(value)""", (str(int(time.time())),))
-      except MySQLdb.Error:
-        db = None
-      self.done.wait(heartbeat_interval)
-
-def log(job_id, text):
-  print('restore job %d: %s' % (job_id, text), flush=True)
-  line = '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), text)
-  query("UPDATE backup_job SET log = RIGHT(CONCAT(IFNULL(log, ''), %s), %s) WHERE id = %s", (line, max_log, job_id))
-
-def save(job_id, result):
-  query("UPDATE backup_job SET result = %s WHERE id = %s", (json.dumps(result), job_id))
-
-def doveadm(*args, data=None, check=True, partial=False):
-  result = subprocess.run(['doveadm', *args], input=data, stdin=None if data is not None else subprocess.DEVNULL,
-    capture_output=True, timeout=86400)
-  # mailbox status fails for folders that cannot be opened (implicit parents); the others are listed
-  if check and result.returncode != 0 and not (partial and result.stdout.strip()):
-    # mail_log writes an info line per message, the errors are the other lines
-    lines = [l for l in result.stderr.decode('utf-8', 'replace').splitlines() if ': Info: ' not in l]
-    raise RuntimeError('\n'.join(lines)[-500:] or 'doveadm exit code %d' % result.returncode)
-  return result.stdout.decode('utf-8', 'replace')
-
-def doveadm_json(*args, partial=False):
-  out = doveadm('-f', 'json', *args, partial=partial)
-  return json.loads(out) if out.strip() else []
-
-# free disk space that must stay after the import: max(2 GiB, 5 % of the filesystem), a full disk stops mail
-# (disk_reserve, mib and count_errors as in restic_runner.py of the backup image; tests/python/backup compares them)
-def disk_reserve(total):
-  return max(2 << 30, total // 20)
-
-def import_space_ok(need, free, total):
-  return need * 11 // 10 + disk_reserve(total) <= free
-
-def mib(size):
-  return '%.1f MiB' % (size / 1048576)
-
-def component(errors=None):
-  return {'added': 0, 'skipped': 0, 'errors': errors or []}
-
-# Folder names on disk: modified UTF-7, listescape writes . / \ as \xx
-def mutf7_decode(text):
-  def utf16(m):
-    if not m.group(1):
-      return '&'
-    data = m.group(1).replace(',', '/')
-    return base64.b64decode(data + '=' * (-len(data) % 4)).decode('utf-16-be')
-  return re.sub(r'&([A-Za-z0-9+,]*)-', utf16, text)
-
-def unescape(part):
-  return mutf7_decode(re.sub(r'\\([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1), 16)), part))
-
-# Maildir++: the maildir is INBOX, ".A.B" is A/B
-def folder_name(entry):
-  return '/'.join(unescape(part) for part in entry[1:].split('.')) if entry else 'INBOX'
-
-def is_dir(path):
-  try:
-    return stat.S_ISDIR(os.lstat(path).st_mode)
-  except OSError:
-    return False
-
-def read_file(path):
-  fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
-  with os.fdopen(fd, 'rb') as f:
-    return f.read()
-
-# Prefixes of the shared and public namespaces, their folders belong to others
-def shared_prefixes():
-  prefixes, namespace, depth = {'Shared/'}, None, 0
-  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
-  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
-    stripped = line.strip()
-    if stripped.endswith('{'):
-      depth += 1
-      if depth == 1 and stripped.startswith('namespace'):
-        namespace = {}
-    elif stripped == '}':
-      if depth == 1 and namespace is not None:
-        if namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
-          prefixes.add(namespace['prefix'].split('%')[0])
-        namespace = None
-      depth = max(depth - 1, 0)
-    elif depth == 1 and namespace is not None and '=' in stripped:
-      key, value = (part.strip() for part in stripped.split('=', 1))
-      namespace[key] = value
-  return tuple(prefix for prefix in prefixes if prefix)
-
-# name => guid of the user's own folders (implicit parents have none); the names are looked up literally
-def own_folders(user, prefixes):
-  names = [n for n in doveadm('mailbox', 'list', '-u', user).splitlines() if n and not n.startswith(prefixes)]
-  wanted, guids = set(names), {}
-  for i in range(0, len(names), folder_batch):
-    for row in doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', *names[i:i + folder_batch], partial=True):
-      if row.get('mailbox') in wanted and re.fullmatch(r'[0-9a-f]{32}', row.get('guid') or ''):
-        guids[row['mailbox']] = row['guid']
-  return guids
-
-def masked(user):
-  return '*' in user or '?' in user
-
-def free_name(base, existing):
-  name, count = base, 1
-  while name in existing or any(folder.startswith(name + '/') for folder in existing):
-    count += 1
-    name = '%s (%d)' % (base, count)
-  return name
-
-# Path of the maildir now (/var/vmail/<domain>/<local>/Maildir), None for unknown or inactive mailboxes
-def mail_location(user):
-  result = subprocess.run(['doveadm', 'user', '-f', 'mail', '--', user], capture_output=True, text=True)
-  m = re.match(r'maildir:([^:]+)', result.stdout.strip()) if result.returncode == 0 else None
-  return m.group(1).rstrip('/') if m else None
-
-# The restored maildir of a mailbox (may hold only ACL files), None if the snapshot has nothing of it
-def staged_maildir(staging, home, location):
-  for path in (staging + location, home + '/' + os.environ.get('MAILDIR_SUB', 'Maildir'), home):
-    if is_dir(path):
-      return path
-  return None
-
-def uid_chunks(uids):
-  ranges, start, prev = [], None, None
-  for uid in sorted(uids):
-    if start is None:
-      start = prev = uid
-    elif uid == prev + 1:
-      prev = uid
-    else:
-      ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
-      start = prev = uid
-  if start is not None:
-    ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
-  return [','.join(ranges[i:i + uid_ranges]) for i in range(0, len(ranges), uid_ranges)]
-
-# Same message: same Message-ID, without one the same Date and size
-def message_key(row):
-  message_id = re.sub(r'\s+', '', row.get('hdr.message-id') or '').strip('<>')
-  if message_id:
-    return 'id:' + message_id
-  return 'date:%s:%s' % (row.get('date.sent'), row.get('size.physical'))
-
-def present_keys(user, guids):
-  keys = set()
-  guids = list(guids)
-  for i in range(0, len(guids), folder_batch):
-    search = []
-    for guid in guids[i:i + folder_batch]:
-      search += (['OR'] if search else []) + ['mailbox-guid', guid]
-    keys.update(message_key(row) for row in doveadm_json('fetch', '-u', user, 'hdr.message-id date.sent size.physical',
-      *search))
-  return keys
-
-def location(path):
-  if ':' in path:
-    raise RuntimeError('unsupported path')
-  return 'maildir:%s:INDEX=MEMORY' % path
-
-# The folders of a restored maildir as standalone maildirs (their messages are the INBOX there): the subfolders are
-# moved out of the maildir, so every folder is imported without the name rules of the source (listescape)
-def split_folders(maildir, home):
-  folders, failed = [('INBOX', maildir)], []
-  out = home + '/_restore_folders'
-  for entry in sorted(os.listdir(maildir)):
-    path = os.path.join(maildir, entry)
-    if not entry.startswith('.') or entry in ('.', '..') or not is_dir(path) or not is_dir(path + '/cur'):
-      continue
-    try:
-      name = folder_name(entry)
-    except (ValueError, UnicodeDecodeError):
-      failed.append(entry)
-      continue
-    if not os.path.isdir(out):
-      os.mkdir(out, 0o700)
-    target = '%s/f%d' % (out, len(folders))
-    os.rename(path, target)
-    folders.append((name, target))
-  return folders, failed
-
-def source_messages(user, root):
-  return doveadm_json('-o', 'mail_location=' + location(root), 'fetch', '-u', user,
-    'uid hdr.message-id date.sent size.physical', 'mailbox', 'INBOX')
-
-def remove_tree(user, name):
-  doveadm(*no_quota, 'mailbox', 'delete', '-u', user, '-r', '-s', '--', name, check=False)
-
-def folder_guid(user, name):
-  rows = doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', name)
-  for row in rows:
-    if row.get('mailbox') == name:
-      return row['guid']
-  raise RuntimeError('folder not found after the import')
-
-def quota_room(user):
-  # free bytes of the storage quota, None without a limit
-  for row in doveadm_json('quota', 'get', '-u', user):
-    if row.get('type') == 'STORAGE' and str(row.get('limit', '-')).isdigit():
-      return (int(row['limit']) - int(row.get('value') or 0)) * 1024
-  return None
-
-# Subscriptions of the backup (subscriptions file: "V\t2", then names with \t between the levels)
-def backup_subscriptions(maildir):
-  try:
-    lines = read_file(maildir + '/subscriptions').decode('utf-8', 'replace').splitlines()
-  except OSError:
-    return []
-  if lines and lines[0].startswith('V\t'):
-    return ['/'.join(unescape(part) for part in line.split('\t')) for line in lines[1:] if line.strip()]
-  # version 1: storage names with the Maildir++ separator
-  return [folder_name('.' + line.strip()) for line in lines if line.strip()]
-
-def restore_subscriptions(user, wanted, prefixes):
-  if not wanted:
-    return 0
-  existing = set(own_folders(user, prefixes)) | {'INBOX'}
-  subscribed = set(doveadm('mailbox', 'list', '-u', user, '-s').splitlines())
-  missing = [name for name in dict.fromkeys(wanted) if name in existing and name not in subscribed]
-  if missing:
-    doveadm('mailbox', 'subscribe', '-u', user, '--', *missing)
-  return len(missing)
-
-# Mails folder by folder: original = only messages missing anywhere in the mailbox, into their original folders;
-# folder = all messages below "Restored <snapshot time>"
-def restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota):
-  result = component()
-  result.update(folders=0, folders_failed=0)
-  if not maildir or not is_dir(maildir + '/cur'):
-    # a key the UI translates
-    result['note'] = 'no_mails'
-    return result
-  tmp = 'Restoring %d' % job_id
-  # a leftover of an interrupted run goes first, it is never counted as present
-  remove_tree(user, tmp)
-  subscriptions = backup_subscriptions(maildir)
-  folders, broken = split_folders(maildir, home)
-  for entry in broken:
-    result['errors'].append('folder %s: name not readable' % entry)
-    result['folders_failed'] += 1
-  live = own_folders(user, prefixes)
-  present = present_keys(user, live.values()) if mode == 'original' else set()
-  plan, need = [], 0
-  for name, root in folders:
-    try:
-      rows = source_messages(user, root)
-    except RuntimeError as e:
-      result['errors'].append('%s: %s' % (name, e))
-      result['folders_failed'] += 1
-      continue
-    wanted = [row for row in rows if mode != 'original' or message_key(row) not in present]
-    result['skipped'] += len(rows) - len(wanted)
-    if wanted:
-      size = sum(int(row.get('size.physical') or 0) for row in wanted)
-      plan.append((name, root, [int(row['uid']) for row in wanted], size))
-      need += size
-  if not plan:
-    if mode == 'original':
-      result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
-    return result
-  vfs = os.statvfs('/var/vmail')
-  free, total = vfs.f_bavail * vfs.f_frsize, vfs.f_blocks * vfs.f_frsize
-  if not import_space_ok(need, free, total):
-    raise RuntimeError('not enough free disk space for the mails: %s needed plus %s reserve, %s free' % (
-      mib(need * 11 // 10), mib(disk_reserve(total)), mib(free)))
-  if check_quota:
-    room = quota_room(user)
-    if room is not None and need > room:
-      raise RuntimeError('the mails do not fit into the quota of the mailbox: %s needed, %s free' % (
-        mib(need), mib(max(room, 0))))
-  names = set(live) | {'INBOX'}
-  dest = None
-  if mode != 'original':
-    dest = free_name('Restored ' + snapshot_time, names)
-    doveadm('mailbox', 'create', '-u', user, '-s', '--', dest)
-    names.add(dest)
-    result['folder'] = dest
-  created = []
-  try:
-    for number, (name, root, uids, size) in enumerate(plan):
-      target = dest + '/' + name if dest else name
-      stage = '%s/f%d' % (tmp, number)
-      try:
-        for chunk in uid_chunks(uids):
-          doveadm(*no_quota, 'import', '-u', user, '-s', location(root), stage, 'mailbox', 'INBOX', 'uid', chunk)
-        guid = folder_guid(user, stage + '/INBOX')
-        if target not in names:
-          doveadm('mailbox', 'create', '-u', user, '-s', '--', target)
-          names.add(target)
-          created.append(target)
-        doveadm(*no_quota, 'move', '-u', user, '--', target, 'mailbox-guid', guid, 'all')
-        result['added'] += len(uids)
-        result['folders'] += 1
-      except RuntimeError as e:
-        result['errors'].append('%s: %s' % (name, e))
-        result['folders_failed'] += 1
-  finally:
-    remove_tree(user, tmp)
-  result['folders_created'] = len(created)
-  if mode == 'original':
-    result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
-  return result
-
-def acl_entries(path):
-  entries = []
-  try:
-    lines = read_file(path).decode('utf-8', 'replace').splitlines()
-  except OSError:
-    return entries
-  for line in lines:
-    parts = line.split()
-    # negative rights would take rights away
-    if len(parts) < 2 or parts[0].startswith('-'):
-      continue
-    rights = [rights_by_letter[c] for c in parts[1] if c in rights_by_letter] + [p for p in parts[2:] if p.startswith(':')]
-    if rights:
-      entries.append((parts[0], rights))
-  return entries
-
-# (folder, entries) of the dovecot-acl files of a restored maildir
-def maildir_acls(maildir):
-  acls = []
-  if not maildir or not is_dir(maildir):
-    return acls
-  for entry in [''] + sorted(e for e in os.listdir(maildir) if e.startswith('.') and e not in ('.', '..')):
-    path = os.path.join(maildir, entry, 'dovecot-acl') if entry else os.path.join(maildir, 'dovecot-acl')
-    if entry and not is_dir(os.path.join(maildir, entry)):
-      continue
-    try:
-      if stat.S_ISREG(os.lstat(path).st_mode):
-        acls.append((folder_name(entry), acl_entries(path)))
-    except (OSError, ValueError, UnicodeDecodeError):
-      continue
-  return acls
-
-# owner => (folder, entries) of every mailbox in the ACL staging (domain admins: only their domains)
-def staged_acls(acl_dir, allowed):
-  acls = {}
-  base = (acl_dir or '') + '/var/vmail'
-  sub = os.environ.get('MAILDIR_SUB', '')
-  if not acl_dir or not is_dir(base):
-    return acls
-  for domain in os.listdir(base):
-    if '.' not in domain or not is_dir(os.path.join(base, domain)) or (allowed is not None and domain.lower() not in allowed):
-      continue
-    for local in os.listdir(os.path.join(base, domain)):
-      home = os.path.join(base, domain, local)
-      maildir = os.path.join(home, sub) if sub and is_dir(os.path.join(home, sub)) else home
-      found = maildir_acls(maildir)
-      if found:
-        acls['%s@%s' % (local, domain)] = found
-  return acls
-
-class AclRestore:
-  def __init__(self, prefixes, allowed):
-    self.prefixes = prefixes
-    self.allowed = allowed
-    self.users, self.folders = {}, {}
-    # anyone/authenticated only when the configuration allows them
-    self.anyone = os.environ.get('ACL_ANYONE', 'disallow') == 'allow'
-
-  def in_scope(self, address):
-    return self.allowed is None or address.rpartition('@')[2].lower() in self.allowed
-
-  def user_exists(self, user):
-    if user not in self.users:
-      self.users[user] = query("SELECT 1 FROM mailbox WHERE username = %s", (user,)).fetchone() is not None
-    return self.users[user]
-
-  def identifier_ok(self, identifier):
-    if identifier.startswith('user='):
-      return self.user_exists(identifier[5:].lower())
-    # group=mcg_<id> belongs to the mailbox groups, the group grants cron keeps them
-    if identifier.startswith('group=mcg_'):
-      return False
-    if identifier in ('anyone', 'authenticated'):
-      return self.anyone
-    return True
-
-  def shown(self, identifier):
-    # domain admins do not see addresses of other domains
-    if identifier.startswith('user=') and not self.in_scope(identifier[5:]):
-      return 'user=(other domain)'
-    return identifier
-
-  def add(self, owner, folder, identifier, rights, result):
-    if not self.in_scope(owner) or masked(owner):
-      result['skipped'] += 1
-      return
-    try:
-      if owner not in self.folders:
-        self.folders[owner] = set(own_folders(owner, self.prefixes)) | {'INBOX'}
-      if folder not in self.folders[owner] or not self.identifier_ok(identifier):
-        result['skipped'] += 1
-        return
-      have = set()
-      for row in doveadm_json('acl', 'get', '-u', owner, '--', folder):
-        if row.get('id') == identifier:
-          have.update((row.get('rights') or '').split())
-      missing = [right for right in rights if right not in have]
-      if not missing:
-        result['skipped'] += 1
-        return
-      doveadm('acl', 'add', '-u', owner, '--', folder, identifier, *missing)
-      result['added'] += 1
-    except RuntimeError as e:
-      result['errors'].append('%s %s %s: %s' % (owner, folder, self.shown(identifier), e))
-
-# Rights the mailbox granted on its folders and rights other mailboxes granted to it; added, never removed
-def restore_acl(acl, user, own, granted):
-  result = component()
-  for folder, entries in own:
-    for identifier, rights in entries:
-      acl.add(user, folder, identifier, rights, result)
-  if acl.user_exists(user):
-    for owner, folders in granted.items():
-      if owner == user or not acl.in_scope(owner) or not acl.user_exists(owner):
-        continue
-      for folder, entries in folders:
-        for identifier, rights in entries:
-          if identifier.lower() == 'user=' + user:
-            acl.add(owner, folder, identifier, rights, result)
-  return result
-
-# Own Sieve scripts (incl. the SOGo vacation script): missing names are added; with activate the active script of the
-# backup is activated if none is active now
-def restore_sieve(user, home, active_link, activate):
-  result = component()
-  folder = home + '/sieve'
-  scripts = []
-  if is_dir(folder):
-    scripts = sorted(f[:-6] for f in os.listdir(folder) if f.endswith('.sieve') and stat.S_ISREG(os.lstat(os.path.join(folder, f)).st_mode))
-  current, active = set(), None
-  for line in doveadm('sieve', 'list', '-u', user).splitlines():
-    name, _, flag = line.rpartition(' ')
-    if name:
-      current.add(name)
-      if flag == 'ACTIVE':
-        active = name
-  for name in scripts:
-    if name in current:
-      result['skipped'] += 1
-      continue
-    try:
-      doveadm('sieve', 'put', '-u', user, '--', name, data=read_file(os.path.join(folder, name + '.sieve')))
-      current.add(name)
-      result['added'] += 1
-    except (OSError, RuntimeError) as e:
-      result['errors'].append('%s: %s' % (name, e))
-  if not activate:
-    return result
-  target = os.readlink(active_link) if os.path.islink(active_link) else ''
-  backup_active = os.path.basename(target)[:-6] if target.endswith('.sieve') else None
-  if backup_active and not active and backup_active in current:
-    try:
-      doveadm('sieve', 'activate', '-u', user, '--', backup_active)
-      result['activated'] = backup_active
-    except RuntimeError as e:
-      result['errors'].append('%s: %s' % (backup_active, e))
-  return result
-
-# filters: backup.php reported its part already, the counts are added up
-def merge(results, key, part):
-  old = results.get(key)
-  if not isinstance(old, dict):
-    results[key] = part
-    return
-  for count in ('added', 'skipped'):
-    old[count] = (old.get(count) or 0) + part.get(count, 0)
-  old['errors'] = (old.get('errors') or []) + part.get('errors', [])
-  for name, value in part.items():
-    if name not in ('added', 'skipped', 'errors'):
-      old[name] = value
-
-def count_errors(result):
-  count = sum(len(result.get(key, {}).get('errors') or []) for key in ('domain', 'aliases') if isinstance(result.get(key), dict))
-  for entry in (result.get('mailboxes') or {}).values():
-    count += int(bool(entry.get('error')))
-    count += sum(len(value.get('errors') or []) for value in entry.values() if isinstance(value, dict))
-  return count
-
-def restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, result):
-  local, _, domain = user.rpartition('@')
-  home = '%s/var/vmail/%s/%s' % (staging, domain, local)
-  if masked(user):
-    for key in components & {'mails', 'imap_acl', 'filters'}:
-      merge(result, key, component(['username with * or ?, doveadm reads it as a mask']))
-    return
-  location = mail_location(user)
-  if not location:
-    for key in components & {'mails', 'imap_acl', 'filters'}:
-      merge(result, key, component(['the mailbox does not exist now or is inactive']))
-    return
-  maildir = staged_maildir(staging, home, location)
-  # the ACL files are read before the folders of the maildir are moved for the import
-  own = maildir_acls(maildir) if 'imap_acl' in components else []
-  if 'mails' in components:
-    try:
-      result['mails'] = restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota)
-    except Exception as e:
-      result['mails'] = component([str(e)[:1000]])
-    try:
-      doveadm('quota', 'recalc', '-u', user)
-    except RuntimeError as e:
-      result['mails']['errors'].append('quota recalc: %s' % e)
-    # new folders
-    acl.folders.pop(user, None)
-  if 'imap_acl' in components:
-    try:
-      result['imap_acl'] = restore_acl(acl, user, own, granted)
-    except Exception as e:
-      result['imap_acl'] = component([str(e)[:1000]])
-  if 'filters' in components:
-    try:
-      part = restore_sieve(user, home, '%s/var/vmail/sieve/%s.sieve' % (staging, user), 'sieve_activate' in components)
-    except Exception as e:
-      part = component([str(e)[:1000]])
-    merge(result, 'filters', part)
-
-def fail(job_id, error):
-  query("""UPDATE backup_job SET state = IF(state = 'cancelled', state, 'failed'), error = %s, finished = NOW()
-    WHERE id = %s""", (error, job_id))
-  log(job_id, 'restore job failed: ' + error)
-  notify(job_id, error)
-
-def notify(job_id, error):
-  hostname = os.environ.get('MAILCOW_HOSTNAME', '')
-  row = query("SELECT value FROM backup_setting WHERE name = 'notify_email'").fetchone()
-  rcpts = [r.strip() for r in (row['value'] if row else '').split(',') if re.fullmatch(r'[^@\s<>,]+@[^@\s<>,]+', r.strip())]
-  if not rcpts:
-    return
-  msg = EmailMessage()
-  msg['From'] = 'backup@' + hostname
-  msg['To'] = ', '.join(rcpts)
-  msg['Subject'] = 'mailcow backup on %s: restore job %d failed' % (hostname, job_id)
-  msg['Date'] = formatdate(localtime=True)
-  msg['Message-ID'] = make_msgid(domain=hostname or None)
-  msg.set_content('The restore job %d failed:\n\n%s\n\nThe job log is in the backup section of the mailcow UI.\n'
-    % (job_id, error))
-  try:
-    with smtplib.SMTP('postfix', 588, hostname or None, timeout=60) as smtp:
-      smtp.send_message(msg)
-  except (OSError, smtplib.SMTPException) as e:
-    print('restore job %d: the notification could not be sent: %s' % (job_id, e), file=sys.stderr)
-
-def staging_dir(job_id, path):
-  # the batch folder of this job, never a symlink
-  if not isinstance(path, str) or not re.fullmatch(r'/restore/%d/[A-Za-z0-9_-]+' % job_id, path) or \
-      os.path.realpath(path) != path or not is_dir(path):
-    return None
-  return path
-
-def run_batch(job):
-  job_id = job['id']
-  params = json.loads(job['params'] or '{}')
-  result = json.loads(job['result'] or '{}')
-  batch = result.get('batch') or {}
-  staging = staging_dir(job_id, batch.get('dir'))
-  acl_dir = staging_dir(job_id, batch.get('acl_dir')) if batch.get('acl_dir') else None
-  mailboxes = [m for m in batch.get('mailboxes') or [] if isinstance(m, str)]
-  components = {c for c in params.get('components') or [] if c in components_here}
-  mode = 'original' if params.get('mail_target') == 'original' else 'folder'
-  allowed = {d.lower() for d in params['allowed_domains']} if isinstance(params.get('allowed_domains'), list) else None
-  check_quota = params.get('requested_role', 'admin') != 'admin'
-  snapshot_time = (result.get('snapshot') or {}).get('time') or time.strftime('%Y-%m-%d %H:%M')
-  results = result.setdefault('mailboxes', {})
-  prefixes = shared_prefixes()
-  acl = AclRestore(prefixes, allowed)
-  granted = staged_acls(acl_dir, allowed) if 'imap_acl' in components else {}
-  log(job_id, 'restoring %s for %d mailboxes%s' % (', '.join(sorted(components)), len(mailboxes),
-    ' (mails: %s)' % mode if 'mails' in components else ''))
-  for user in mailboxes:
-    entry = results.setdefault(user, {})
-    if entry.get('error'):
-      continue
-    started = time.monotonic()
-    if not staging:
-      for key in components & {'mails', 'imap_acl', 'filters'}:
-        merge(entry, key, component(['the restored files are missing']))
-    else:
-      restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, entry)
-    summary = ', '.join('%s +%d' % (key, entry[key].get('added', 0)) for key in sorted(components) if isinstance(entry.get(key), dict))
-    errors = sum(len(entry[key].get('errors') or []) for key in components if isinstance(entry.get(key), dict))
-    log(job_id, '%s: %s%s (%d s)' % (user, summary, ', %d errors' % errors if errors else '', time.monotonic() - started))
-    result['errors'] = count_errors(result)
-    save(job_id, result)
-  result['errors'] = count_errors(result)
-  save(job_id, result)
-  # the restic runner removes the staging and goes on
-  query("UPDATE backup_job SET step = 'mails_done' WHERE id = %s AND state = 'running' AND step = 'mails_running'", (job_id,))
-
-def main():
-  global conn
-  if os.geteuid() == 0:
-    # as vmail like the ofelia job: a run as root would leave a lock file the runs as vmail cannot open
-    os.execv('/usr/local/bin/gosu', ['gosu', 'vmail', sys.executable, os.path.abspath(__file__)] + sys.argv[1:])
-  os.umask(0o077)
-  # released by the kernel when the process ends, a crashed run leaves no stale lock
-  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
-  try:
-    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except OSError:
-    sys.exit("restore: can't lock " + lock_file)
-  conn = connect()
-  heartbeat = Heartbeat()
-  heartbeat.start()
-  try:
-    # under the lock: a batch in mails_running was left by a crashed or stopped run, its temporary folders go
-    for row in query("""SELECT id, result FROM backup_job WHERE type = 'restore' AND state = 'running'
-        AND step = 'mails_running'""").fetchall():
-      try:
-        for user in (json.loads(row['result'] or '{}').get('batch') or {}).get('mailboxes') or []:
-          remove_tree(user, 'Restoring %d' % row['id'])
-          doveadm('quota', 'recalc', '-u', user, check=False)
-      except (ValueError, AttributeError, RuntimeError):
-        pass
-      fail(row['id'], 'interrupted: the restore of the mails was stopped')
-    last, waited = None, 0
-    while True:
-      job = query("""SELECT id, params, result FROM backup_job WHERE type = 'restore' AND state = 'running'
-        AND step = 'mails' ORDER BY id LIMIT 1""").fetchone()
-      if not job:
-        # the next batch of a restore follows within seconds: wait for it instead of the next ofelia run
-        row = query("SELECT state FROM backup_job WHERE id = %s", (last,)).fetchone() if last else None
-        if row and row['state'] == 'running' and waited < batch_wait:
-          time.sleep(2)
-          waited += 2
-          continue
-        break
-      last, waited = job['id'], 0
-      if query("UPDATE backup_job SET step = 'mails_running' WHERE id = %s AND state = 'running' AND step = 'mails'",
-          (job['id'],)).rowcount != 1:
-        continue
-      try:
-        run_batch(job)
-      except Exception as e:
-        fail(job['id'], str(e)[:1000])
-  finally:
-    heartbeat.done.set()
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/dovecot/retention_runner.py

@@ -1,249 +0,0 @@
-#!/usr/bin/python3
-# Nightly run of the retention rules (ofelia, dovecot container): gets the plan page by page from the mailcow UI
-# (nginx:8081/retention.php), gives every folder of a mailbox to the first entry of its plan that matches it, counts
-# the mail saved in the folders before the period of the rule with doveadm, deletes it unless the rule only
-# simulates, and reports the results back.
-
-import fcntl
-import hashlib
-import hmac
-import json
-import os
-import re
-import subprocess
-import sys
-import time
-import urllib.parse
-import urllib.request
-
-plan_url = 'http://nginx:8081/retention.php'
-lock_file = '/tmp/retention_runner.lock'
-# folder keys of the rules and the special-use flag of their folders in the Dovecot configuration
-special_use = {'trash': '\\Trash', 'junk': '\\Junk', 'sent': '\\Sent', 'drafts': '\\Drafts', 'archive': '\\Archive'}
-own_folder = 'folder:'
-# mailboxes per page of the plan and results per report, a large instance has many mailboxes
-plan_page = 200
-report_batch = 200
-# a request is tried this often, waiting longer each time
-tries = 3
-
-token = hmac.new(os.environ.get('DBPASS', '').encode(), b'retention', hashlib.sha256).hexdigest()
-
-def request(url, body=None):
-  data = None if body is None else json.dumps(body).encode()
-  req = urllib.request.Request(url, data=data, headers={'Content-Type': 'application/json', 'X-Retention-Token': token},
-    method='GET' if body is None else 'POST')
-  with urllib.request.urlopen(req, timeout=300) as response:
-    return json.load(response)
-
-def retried(url, body=None):
-  for attempt in range(1, tries + 1):
-    try:
-      return request(url, body)
-    except Exception as e:
-      if attempt == tries:
-        raise
-      print('retention: request failed, trying again: %s' % e, file=sys.stderr)
-      time.sleep(30 * attempt)
-
-def plan():
-  # one page after the other, so memory stays small; rule changes apply to the pages fetched after them
-  after = ''
-  while after is not None:
-    page = retried(plan_url + '?' + urllib.parse.urlencode({'after': after, 'limit': plan_page}))
-    yield from page['mailboxes']
-    after = page['next']
-
-def post(body):
-  # False if the report could not be sent
-  try:
-    retried(plan_url, body)
-    return True
-  except Exception as e:
-    print('retention: report failed: %s' % e, file=sys.stderr)
-    return False
-
-class Reports:
-  # results go back in parts; once a part cannot be sent, the rest waits for the end of the run
-  def __init__(self):
-    self.pending, self.failed = [], False
-
-  def add(self, result):
-    self.pending.append(result)
-    if len(self.pending) >= report_batch and not self.failed:
-      self.failed = not self.send()
-
-  def send(self, summary=None, run=None):
-    # full parts, at the end all results with the summary and the run in the last part
-    final = summary is not None
-    while len(self.pending) >= report_batch or final:
-      body = {'results': self.pending[:report_batch]}
-      if final and len(self.pending) <= report_batch:
-        body.update(summary=summary, run=run)
-      if not post(body):
-        return False
-      del self.pending[:report_batch]
-      if 'summary' in body:
-        break
-    return True
-
-def doveadm(*args, partial=False):
-  result = subprocess.run(['doveadm', *args], capture_output=True, text=True, timeout=3600)
-  # mailbox status also fails for single folders that cannot be opened; the others are still listed
-  if result.returncode != 0 and not (partial and result.stdout):
-    raise RuntimeError((result.stderr or result.stdout).strip()[:500] or 'doveadm exit %d' % result.returncode)
-  return result.stdout
-
-def configuration():
-  # names of the special folders per key, and the prefixes of the shared and public namespaces (not the user's)
-  names = {key: set() for key in special_use}
-  prefixes = ['Shared/']
-  namespace, mailbox = None, None
-  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
-  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
-    stripped = line.strip()
-    if line.startswith('namespace ') and line.endswith('{'):
-      namespace = {}
-    elif line == '}':
-      if namespace and namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
-        prefixes.append(namespace['prefix'].split('%')[0])
-      namespace = None
-    elif namespace is None:
-      continue
-    elif re.match(r'mailbox .+ \{$', stripped):
-      mailbox = stripped[len('mailbox '):-2].strip().strip('"')
-    elif stripped == '}':
-      mailbox = None
-    elif '=' in stripped:
-      key, value = (part.strip() for part in stripped.split('=', 1))
-      if mailbox is not None and key == 'special_use':
-        for folder, flag in special_use.items():
-          if flag in value.split():
-            names[folder].add(mailbox)
-      elif mailbox is None and key in ('type', 'prefix'):
-        namespace[key] = value
-  if not names['trash']:
-    raise RuntimeError('no special folders in the output of doveconf -n')
-  return names, [prefix for prefix in prefixes if prefix]
-
-def own_folders(user, prefixes):
-  # folder name => guid of the user's own folders; listed first, so folders others share are not opened
-  names = set(name for name in doveadm('-f', 'tab', 'mailbox', 'list', '-u', user).splitlines()[1:]
-    if name and not any((name + '/').startswith(prefix) for prefix in prefixes))
-  folders = {}
-  if not names:
-    return folders
-  for line in doveadm('-f', 'tab', 'mailbox', 'status', '-u', user, 'guid', *sorted(names), partial=True).splitlines()[1:]:
-    name, _, guid = line.rpartition('\t')
-    if name in names and re.fullmatch(r'[0-9a-f]{32}', guid):
-      folders[name] = guid
-  return folders
-
-def canonical(name):
-  # Dovecot writes INBOX in capitals, also as the parent of folders
-  return 'INBOX' + name[5:] if name[:5].upper() == 'INBOX' and name[5:6] in ('', '/') else name
-
-def matches(action, name, names):
-  # match: the folder itself, an own folder name with *, the subfolders of a special folder or the inbox, all folders
-  key, match = action['folder'], action['match']
-  if match == 'all':
-    return True
-  if key.startswith(own_folder):
-    # * is the only wildcard of own folder names
-    pattern = '.*'.join(re.escape(part) for part in canonical(key[len(own_folder):]).split('*'))
-    return re.fullmatch(pattern, name) is not None
-  folders = ['INBOX'] if key == 'inbox' else names.get(key, ())
-  if match == 'subfolders':
-    return any(name.startswith(folder + '/') for folder in folders)
-  return name in folders
-
-def decide(actions, folders, names):
-  # rule id => (its action, the folders it decides about): every folder goes to the first action that matches it
-  decided = {}
-  for name in sorted(folders):
-    for action in actions:
-      if matches(action, canonical(name), names):
-        decided.setdefault(action['rule_id'], (action, []))[1].append(name)
-        break
-  return decided
-
-def found(user, query):
-  return len(doveadm('search', '-u', user, *query).splitlines())
-
-def clean(user, action, selected, folders, result):
-  # name and guid together: doveadm then only opens these folders
-  query = []
-  for name in selected:
-    query += (['OR'] if query else []) + ['(', 'mailbox', name, 'mailbox-guid', folders[name], ')']
-  query = ['('] + query + [')', 'savedbefore', '%dd' % int(action['days'])]
-  if action['keep_flagged']:
-    query += ['NOT', 'FLAGGED']
-  result['matched'] = found(user, query)
-  if not result['matched'] or action['mode'] != 'delete':
-    return
-  try:
-    doveadm('expunge', '-u', user, *query)
-    result['deleted'] = result['matched']
-  except Exception as e:
-    # part of the mail may be gone: what is left is counted
-    result['error'] = str(e)[:1000]
-    try:
-      result['deleted'] = max(0, result['matched'] - found(user, query))
-    except Exception:
-      pass
-
-def main():
-  # released by the kernel when the process ends, a crashed run leaves no stale lock
-  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
-  try:
-    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except OSError:
-    sys.exit("retention: can't lock " + lock_file)
-  reports = Reports()
-  run = {'started': int(time.time()), 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0, 'error': ''}
-  summary = {}
-  try:
-    names, prefixes = configuration()
-    for mailbox in plan():
-      user = mailbox['username']
-      run['mailboxes'] += 1
-      try:
-        folders, error = own_folders(user, prefixes), ''
-        decided = decide(mailbox['actions'], folders, names)
-      except Exception as e:
-        # the folders are unknown: an error for every rule of the mailbox
-        folders, error, decided = {}, str(e)[:1000], {}
-        for action in mailbox['actions']:
-          decided.setdefault(action['rule_id'], (action, []))
-      for rule_id, (action, selected) in decided.items():
-        rule = summary.setdefault(rule_id, {'rule_id': rule_id, 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0,
-          'simulated': int(action['mode'] != 'delete')})
-        result = {'username': user, 'rule_id': rule_id, 'folder': '\n'.join(selected), 'days': action['days'], 'matched': 0,
-          'deleted': 0, 'simulated': rule['simulated'], 'error': error}
-        if not error:
-          try:
-            clean(user, action, selected, folders, result)
-          except Exception as e:
-            result['error'] = str(e)[:1000]
-        for counter in (rule, run):
-          counter['matched'] += result['matched']
-          counter['deleted'] += result['deleted']
-          counter['errors'] += int(result['error'] != '')
-        rule['mailboxes'] += 1
-        # only mailboxes with found mail or an error are logged, every rule gets a summary
-        if result['matched'] or result['error']:
-          reports.add(result)
-  except Exception as e:
-    run['error'] = str(e)[:1000]
-  reported = reports.send(list(summary.values()), run)
-  print('retention: %d mailboxes, %d mails deleted, %d found by simulating rules, %d errors' % (run['mailboxes'],
-    run['deleted'], sum(rule['matched'] for rule in summary.values() if rule['simulated']), run['errors']))
-  if run['error']:
-    print('retention: %s' % run['error'], file=sys.stderr)
-  if not reported:
-    print('retention: the results could not be reported', file=sys.stderr)
-  if run['error'] or not reported:
-    sys.exit(1)
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -7,6 +7,9 @@ class BanManager:
   and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
   (chains, SNAT), because the backends must not run in parallel."""
 
+  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
+  EXCUSE_TIME = 60
+
   def __init__(self, r, tables, options, logger, lock, clock=time.time):
     self.r = r
     self.tables = tables
@@ -18,6 +21,16 @@ class BanManager:
     self.banned = set()
     self.denied = {}
     self.allowlist = []
+    self.excused = {}
+
+  def excuse(self, ip):
+    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
+    with self.lock:
+      now = self.clock()
+      if self.excused.get(ip, 0) <= now:
+        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
+          ip, self.EXCUSE_TIME))
+      self.excused[ip] = now + self.EXCUSE_TIME
 
   def attempt(self, ip):
     """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
@@ -30,6 +43,9 @@ class BanManager:
       if allowed is not None:
         self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
         return
+      if self.excused.get(ip, 0) > now:
+        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
+        return
       if net in self.banned:
         # log lines of connections from before the ban
         return
@@ -192,6 +208,8 @@ class BanManager:
       for net in [net for net, counter in self.counters.items()
                   if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
         del self.counters[net]
+      for ip in [ip for ip, until in self.excused.items() if until <= now]:
+        del self.excused[ip]
 
   def apply(self, nets):
     for net in self.tables.ban(nets):

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -109,6 +109,12 @@ class Netfilter:
   def handle_line(self, line):
     # a broken line must not end the watcher
     try:
+      excused = Regexes.temporary_failure(line)
+      if excused is not None:
+        ip = Regexes.parse_ip(excused)
+        if self.external(ip):
+          self.bans.excuse(ip)
+        return
       result = self.regexes.match(line)
       if result is None:
         return

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -6,43 +6,20 @@ import re
 class Regexes:
   """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
 
-  # Anchored to the line start: text clients control in other lines (login names, SMTP commands) must not match.
-  # Failed IMAP, POP3, Sieve and SMTP logins are reported by mailcowauth.php (rule 3), not by Dovecot or Postfix.
-  # Only the master passdb is checked by Dovecot itself (rule 8); the last address is the one Dovecot added.
   DEFAULTS = {
-    1: r'^mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)$',
+    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
-    2: r'^Rspamd UI: Invalid password by ([0-9a-f\.:]+)$',
+    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
-    3: r'^MAILCOWAUTH: Login failed for user .+ with service \w* from IP ([0-9a-f\.:]+)$',
+    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
-    4: r'^warning: non-SMTP command from [^\[\]\s]*\[([0-9a-f\.:]+)\]:.+',
+    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
-    5: r'^NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
-    6: r'^(?:\[\d+\]: )?SOGo\w* Login from \'(?:[0-9a-f\.:]+, )*([0-9a-f\.:]+)\' for user .+ might not have worked',
+    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
-    7: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
-    8: r'^auth: passwd-file\(.*,([0-9a-f\.:]+),master(?:,<[^<>]*>)?\): (?:Password mismatch|unknown user)',
+    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
+    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
   }
-  # Former default rules (with any rule id) => rule id of their replacement, None: the rule is removed.
+  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
-  # Rules the admin changed are kept.
+  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
-  FORMER_DEFAULTS = {
+  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
-    r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)': 1,
-    r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)': 2,
-    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed': 3,
-    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+': 3,
-    r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+': 4,
-    r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+': 5,
-    r"SOGo.+ Login from '([0-9a-f\.:]+)' for user .+ might not have worked": 6,
-    r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked': 6,
-    r'-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
-    r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
-    r'-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
-    r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
-    r'-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
-    r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
-    r'auth: static\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
-    r'auth: \w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
-    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': 8,
-    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)': None,
-  }
-  # Lines of the web UI are not sanitized like syslog lines: a newline in a login name must not hide the address
-  CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f]')
 
   def __init__(self, logger):
     self.logger = logger
@@ -65,10 +42,6 @@ class Regexes:
     if not isinstance(stored, dict):
       self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
       return
-    migrated = self.migrate(stored)
-    if migrated != stored:
-      self.store_migrated(r)
-      stored = migrated
 
     rules = []
     for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
@@ -81,7 +54,6 @@ class Regexes:
 
   def match(self, line):
     """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
-    line = self.CONTROL_CHARS.sub('_', line)
     for rule_id, regex in self.rules:
       result = regex.search(line)
       if result and result.lastindex and result.group(1):
@@ -89,35 +61,10 @@ class Regexes:
     return None
 
   @classmethod
-  def migrate(cls, stored):
+  def temporary_failure(cls, line):
-    """The stored rules with former default rules replaced by the current ones (or removed)"""
+    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
-    migrated = {}
+    result = cls.TEMPORARY_FAILURE.search(line)
-    for rule_id, regex in stored.items():
+    return result.group(1) if result else None
-      if isinstance(regex, str) and regex in cls.FORMER_DEFAULTS:
-        replacement = cls.FORMER_DEFAULTS[regex]
-        if replacement is None:
-          continue
-        regex = cls.DEFAULTS[replacement]
-      migrated[rule_id] = regex
-    return migrated
-
-  def store_migrated(self, r):
-    # WATCH keeps a save of the UI between reading and writing
-    def replace(pipe):
-      current = json.loads(pipe.get('F2B_REGEX') or '{}')
-      if not isinstance(current, dict):
-        return
-      migrated = self.migrate(current)
-      if migrated == current:
-        return
-      pipe.multi()
-      pipe.set('F2B_REGEX', json.dumps(migrated, ensure_ascii=False))
-
-    try:
-      r.transaction(replace, 'F2B_REGEX')
-      self.logger.logInfo('Replaced former default rules in F2B_REGEX')
-    except Exception as ex:
-      self.logger.logWarn('Could not replace former default rules in F2B_REGEX: %s' % ex)
 
   @staticmethod
   def parse_ip(text):

data/Dockerfiles/nginx/bootstrap.py

@@ -1,5 +1,4 @@
 import os
-import re
 import subprocess
 from jinja2 import Environment, FileSystemLoader
 
@@ -45,11 +44,6 @@ def prepare_template_vars():
   ipv4_network = os.getenv("IPV4_NETWORK", "172.22.1")
   additional_server_names = os.getenv("ADDITIONAL_SERVER_NAMES", "")
   trusted_proxies = os.getenv("TRUSTED_PROXIES", "")
-  # nginx size syntax, invalid values fall back to the default
-  max_body_size = os.getenv("NGINX_MAX_BODY_SIZE", "128m").strip()
-  if not re.fullmatch(r'[0-9]+[kKmMgG]?', max_body_size):
-    print(f"Invalid NGINX_MAX_BODY_SIZE '{max_body_size}', using 128m")
-    max_body_size = "128m"
 
   template_vars = {
     'IPV4_NETWORK': ipv4_network,
@@ -66,7 +60,6 @@ def prepare_template_vars():
     'PHPFPMHOST': os.getenv("PHPFPMHOST", "php-fpm-mailcow"),
     'ENABLE_IPV6': os.getenv("ENABLE_IPV6", "true").lower() != "false",
     'HTTP_REDIRECT': os.getenv("HTTP_REDIRECT", "n").lower() in ("y", "yes"),
-    'NGINX_MAX_BODY_SIZE': max_body_size,
   }
 
   ssl_dir = '/etc/ssl/mail/'

data/Dockerfiles/postfix/Dockerfile

@@ -26,7 +26,6 @@ RUN groupadd -g 102 postfix \
 	postfix \
 	postfix-mysql \
 	postfix-pcre \
-	python3 \
 	redis-tools \
 	sasl2-bin \
 	sudo \
@@ -47,7 +46,6 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
-COPY mailcow-policy.py /usr/local/bin/mailcow-policy.py
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
@@ -55,7 +53,6 @@ RUN chmod +x /opt/postfix.sh \
   /usr/local/bin/rspamd-pipe-ham \
   /usr/local/bin/rspamd-pipe-spam \
   /usr/local/bin/whitelist_forwardinghosts.sh \
-  /usr/local/bin/mailcow-policy.py \
   /usr/local/sbin/stop-supervisor.sh
 RUN rm -rf /tmp/* /var/tmp/*
 

data/Dockerfiles/postfix/mailcow-policy.py

@@ -1,54 +0,0 @@
-#!/usr/bin/python3
-# Policy service of the mail policies (mailcow-policy in master.cf, spawned by Postfix): reads the policy requests
-# of smtpd and asks the mailcow UI for the action (nginx:8081/mailpolicy.php). Without an answer the recipient is
-# deferred; smtpd only asks while policies or internal aliases exist (mysql_mail_policy_maps.cf).
-
-import http.client
-import re
-import socket
-import sys
-from urllib.parse import urlencode
-
-# one line: DUNNO or a rejection with code and text
-valid_action = re.compile(rb'DUNNO|[45]\d\d [45]\.\d{1,3}\.\d{1,3} [\x20-\x7e]{1,200}')
-unavailable = b'DEFER_IF_PERMIT Mail policy check temporarily unavailable'
-# address of nginx, resolved once per process (a lookup costs more than the check)
-nginx = None
-
-def check(request):
-  global nginx
-  body = urlencode(sorted(request.items())).encode()
-  try:
-    if nginx is None:
-      nginx = socket.gethostbyname('nginx')
-    # http.client sets TCP_NODELAY, so the body is not held back until the headers are acknowledged
-    # below the timeout of Postfix (main.cf), so an answer always arrives
-    conn = http.client.HTTPConnection(nginx, 8081, timeout=8)
-    try:
-      conn.request('POST', '/mailpolicy.php', body, {'Host': 'nginx', 'Content-Type': 'application/x-www-form-urlencoded'})
-      response = conn.getresponse()
-      action = response.read()
-    finally:
-      conn.close()
-  except (OSError, http.client.HTTPException):
-    # nginx may have a new address after a restart
-    nginx = None
-    return unavailable
-  return action if response.status == 200 and valid_action.fullmatch(action) else unavailable
-
-def main():
-  request = {}
-  # attributes as bytes: addresses may contain any 8-bit characters
-  for line in sys.stdin.buffer:
-    line = line.rstrip(b'\r\n')
-    if line:
-      name, separator, value = line.partition(b'=')
-      if separator:
-        request[name] = value
-      continue
-    sys.stdout.buffer.write(b'action=' + check(request) + b'\n\n')
-    sys.stdout.buffer.flush()
-    request = {}
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/postfix/postfix.sh

@@ -15,21 +15,6 @@ until dig +short mailcow.email > /dev/null; do
   sleep 1
 done
 
-# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates
-DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
-DBV_WAIT=0
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
-  echo "Waiting for schema update..."
-  sleep 5
-  DBV_WAIT=$((DBV_WAIT + 1))
-  DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
-done
-if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
-  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
-else
-  echo "DB schema is ${DBV_NOW}"
-fi
-
 cat <<EOF > /etc/aliases
 # Autogenerated by mailcow
 null: /dev/null
@@ -239,19 +224,6 @@ query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports
   LIMIT 1;
 EOF
 
-# Mail policies: while policies or internal aliases exist, smtpd asks the policy service (mailcow_mail_policy in main.cf)
-cat <<EOF > /opt/postfix/conf/sql/mysql_mail_policy_maps.cf
-# Autogenerated by mailcow
-user = ${DBUSER}
-password = ${DBPASS}
-hosts = unix:/var/run/mysqld/mysqld.sock
-dbname = ${DBNAME}
-query = SELECT 'mailcow_mail_policy' FROM DUAL WHERE '%s' != ''
-  AND (EXISTS (SELECT 1 FROM mail_policy_member JOIN mail_policy ON mail_policy.id = mail_policy_member.policy_id
-      WHERE mail_policy.active = '1')
-    OR EXISTS (SELECT 1 FROM alias WHERE internal = '1' AND active IN ('1', '2')));
-EOF
-
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}

data/Dockerfiles/restic/Dockerfile

@@ -1,28 +0,0 @@
-FROM alpine:3.24
-
-LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
-
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, the runner connects through the socket
-ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
-
-RUN apk add --no-cache --update \
-  ca-certificates \
-  git \
-  mariadb-client \
-  openssh-client \
-  py3-mysqlclient \
-  py3-pynacl \
-  python3 \
-  restic \
-  tini \
-  tzdata \
-  # dump, export, manifest and the mailcow directory are below /mailcow (paths of the snapshot), only root may read them
-  && mkdir -m 700 /mailcow
-
-COPY restic_runner.py /usr/local/bin/restic_runner.py
-COPY sogo_restore.py /usr/local/bin/sogo_restore.py
-
-ENTRYPOINT ["/sbin/tini", "--"]
-CMD ["/usr/bin/python3", "-u", "/usr/local/bin/restic_runner.py"]

data/Dockerfiles/restic/sogo_restore.py

@@ -1,118 +0,0 @@
-"""SOGo part of a mailbox restore (component sogo, overwrites), called by the restic runner for one mailbox.
-
-Runs `sogo-tool restore -f ALL <dir> <user>` and `sogo-tool restore -p <dir> <user>` in the sogo container through
-dockerapi on the user's file of the nightly sogo-tool backup: missing calendars, address books and entries come back,
-the names and shares of existing folders and all SOGo preferences are replaced by the backup's, entries changed since
-stay. Then the user's entries in SOGo's cache (memcached) are removed, so the change shows at once.
-"""
-
-import json
-import os
-import re
-import socket
-import ssl
-import urllib.request
-
-dockerapi_url = 'https://dockerapi:443'
-memcached_addr = ('memcached-mailcow', 11211)
-# dockerapi stops each of the two sogo-tool runs after 850 s
-restore_timeout = 1800
-# accepted by the dockerapi handler as well
-user_re = re.compile(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+')
-
-
-def _short(text, limit=300):
-  # sogo-tool may print database URLs: no credentials, one line, the end has the error
-  text = re.sub(r'([A-Za-z][A-Za-z0-9+.-]*://)[^\s/@]*@', r'\1***@', str(text))
-  text = ' '.join(text.split())
-  return text if len(text) <= limit else '...' + text[-limit:]
-
-
-def _api(path, body=None, timeout=60):
-  # dockerapi uses the mail certificate, its name does not match
-  ctx = ssl.create_default_context()
-  ctx.check_hostname = False
-  ctx.verify_mode = ssl.CERT_NONE
-  data = None if body is None else json.dumps(body).encode()
-  req = urllib.request.Request(dockerapi_url + path, data=data, headers={'Content-Type': 'application/json'},
-    method='GET' if body is None else 'POST')
-  with urllib.request.urlopen(req, timeout=timeout, context=ctx) as response:
-    return json.load(response)
-
-
-def _sogo_container():
-  project = os.environ.get('COMPOSE_PROJECT_NAME', '').lower()
-  by_name = None
-  for cid, info in _api('/containers/json').items():
-    if not cid.isalnum() or not isinstance(info, dict):
-      continue
-    labels = (info.get('Config') or {}).get('Labels') or {}
-    if labels.get('com.docker.compose.service') == 'sogo-mailcow':
-      if not project or labels.get('com.docker.compose.project', '').lower() == project:
-        return cid
-    elif by_name is None and 'sogo-mailcow' in str(info.get('Name', '')):
-      by_name = cid
-  if by_name:
-    return by_name
-  raise RuntimeError('sogo container not found')
-
-
-def _memcached_delete(keys):
-  keys = [k for k in keys if 0 < len(k.encode()) <= 250 and not re.search(r'[\s\x00-\x1f\x7f]', k)]
-  if not keys:
-    return
-  with socket.create_connection(memcached_addr, timeout=10) as s:
-    for key in keys:
-      s.sendall(b'delete ' + key.encode() + b'\r\n')
-      reply = b''
-      while not reply.endswith(b'\r\n'):
-        chunk = s.recv(64)
-        if not chunk:
-          raise RuntimeError('memcached closed the connection')
-        reply += chunk
-      if reply.strip() not in (b'DELETED', b'NOT_FOUND'):
-        raise RuntimeError('memcached: %s' % _short(reply.decode('ascii', 'replace')))
-
-
-def _cache_keys(db, username):
-  # preferences and the ACLs of the user's folders (<uid>/Calendar/personal+acl); never read the c_*location
-  # columns of sogo_folder_info (database credentials)
-  keys = ['%s+defaults' % username, '%s+settings' % username]
-  cur = db.cursor()
-  try:
-    cur.execute('SELECT c_path FROM sogo_folder_info WHERE c_path2 = %s', (username,))
-    keys += ['%s+acl' % row[0][len('/Users/'):] for row in cur.fetchall() if row[0].startswith('/Users/')]
-  finally:
-    cur.close()
-  return keys
-
-
-def restore(db, backup_dir, username, log):
-  """Restores the SOGo data of one user from <backup_dir>/<username> (staged readable for the sogo user only).
-  db: open MySQLdb connection; log: callable for short progress lines. Returns {"added": 0|1, "errors": [...]}."""
-  result = {'added': 0, 'errors': []}
-  if not user_re.fullmatch(username):
-    result['errors'].append('username not supported by the SOGo restore')
-    return result
-  path = os.path.join(backup_dir, username)
-  if not os.path.isfile(path) or os.path.islink(path):
-    # sogo-tool backup ALL skips users without SOGo folders
-    # a key the UI translates
-    result['note'] = 'no_sogo_data'
-    log('sogo: no SOGo data in the snapshot')
-    return result
-  try:
-    res = _api('/containers/%s/exec' % _sogo_container(), {'cmd': 'sogo', 'task': 'restore', 'dir': backup_dir,
-      'username': username}, timeout=restore_timeout)
-    if not isinstance(res, dict) or res.get('type') != 'success':
-      raise RuntimeError('sogo-tool restore failed: %s' % _short(res.get('msg', '') if isinstance(res, dict) else res))
-    result['added'] = 1
-    log('sogo: calendars, address books and preferences reset to the backup')
-  except Exception as e:
-    result['errors'].append(_short(e))
-  try:
-    # SOGo caches ACLs and preferences for 15 minutes
-    _memcached_delete(_cache_keys(db, username))
-  except Exception as e:
-    result['errors'].append('SOGo cache not cleared, changes may show up after 15 minutes: %s' % _short(e))
-  return result

data/Dockerfiles/sogo/Dockerfile

@@ -1,6 +1,6 @@
 # SOGo built from source to enable security patch application
 # Repository: https://github.com/Alinto/sogo
-# Version: SOGo-5.12.13
+# Version: SOGo-5.12.12
 #
 # Applied security patches:
 # -
@@ -12,8 +12,8 @@ FROM debian:bookworm
 LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG SOGO_VERSION=SOGo-5.12.13
+ARG SOGO_VERSION=SOGo-5.12.12
-ARG SOPE_VERSION=SOPE-5.12.13
+ARG SOPE_VERSION=SOPE-5.12.12
 # Security patches to apply (space-separated commit hashes)
 ARG SOGO_SECURITY_PATCHES=""
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$

data/Dockerfiles/watchdog/watchdog.sh

@@ -43,19 +43,6 @@ while ! mariadb-admin status --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u$
   sleep 2
 done
 
-# Wait for the schema update (DB_VERSION) at startup, do not restart them meanwhile.
-DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-DBV_WAIT=0
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
-  echo "Waiting for schema update..."
-  sleep 5
-  DBV_WAIT=$((DBV_WAIT + 1))
-  DBV_NOW=$(mariadb --skip-ssl --socket=/var/run/mysqld/mysqld.sock -u${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN)
-done
-if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
-  echo -e "\e[31m$(date) - DB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting checks anyway, check the php-fpm log (init_db)\e[0m"
-fi
-
 # Do not attempt to write to slave
 if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"

data/conf/dovecot/auth/mailcowauth.php

@@ -68,17 +68,13 @@ require_once 'functions.acl.inc.php';
 
 
 $isSOGoRequest = $post['real_rip'] == getenv('IPV4_NETWORK') . '.248';
-// real_rip is the address that connected to Dovecot (for proxied connections the proxy), client_ip the client (for logs)
-$client_ip = filter_var($post['client_ip'] ?? '', FILTER_VALIDATE_IP) ? $post['client_ip'] : $post['real_rip'];
 $result = false;
-// set when the password is right, but the account does not allow this login
-$denied = false;
 if ($isSOGoRequest) {
   // This is a SOGo Auth request. First check for SSO password.
   $sogo_sso_pass = file_get_contents("/etc/sogo-sso/sogo-sso.pass");
   if ($sogo_sso_pass === $post['password']){
     error_log('MAILCOWAUTH: SOGo SSO auth for user ' . $post['username']);
-    set_sasl_log($post['username'], $client_ip, "SOGO");
+    set_sasl_log($post['username'], $post['real_rip'], "SOGO");
     $result = true;
   }
 }
@@ -95,10 +91,10 @@ if ($result === false){
     'service' => $post['service'],
     'is_internal' => true,
     'remote_addr' => $post['real_rip']
-  ), $denied);
+  ));
   if ($result) {
-    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+    error_log('MAILCOWAUTH: App auth for user ' . $post['username'] . " with service " . $service . " from IP " . $post['real_rip']);
-    set_sasl_log($post['username'], $client_ip, $service);
+    set_sasl_log($post['username'], $post['real_rip'], $service);
   }
 }
 if ($result === false){
@@ -106,37 +102,26 @@ if ($result === false){
   $iam_provider = identity_provider('init');
   $iam_settings = identity_provider('get');
   try {
-    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']), $denied);
+    $result = user_login($post['username'], $post['password'], array('is_internal' => true, 'service' => $post['service']));
   } catch (\Mailcow\Auth\IdpUnavailable $e) {
     // temporary failure for Dovecot, no failed login
-    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip . ": " . $e->getMessage());
+    error_log("MAILCOWAUTH: Identity provider unavailable for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip'] . ": " . $e->getMessage());
     http_response_code(503); // Service Unavailable
     echo json_encode($return);
     session_destroy();
     exit;
   }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
-    set_sasl_log($post['username'], $client_ip, $post['service']);
+    set_sasl_log($post['username'], $post['real_rip'], $post['service']);
   }
 }
 
 if ($result) {
   http_response_code(200); // OK
   $return['success'] = true;
-} elseif ($denied) {
-  // Logged, but no failed login for netfilter: e.g. a client of a shared mailbox or without IMAP access
-  error_log("MAILCOWAUTH: Login denied for user " . $post['username'] . " with service " . $service . " from IP " . $client_ip);
-  http_response_code(401); // Unauthorized
 } else {
-  // Same line for the log and netfilter, which counts it for IMAP, POP3, Sieve and SMTP (Postfix asks Dovecot);
+  error_log("MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']);
-  // SOGo logins have their own rule
-  $failed = "MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " .
-    preg_replace('/[^a-z0-9]/', '', strtolower((string)$service)) . " from IP " . $client_ip;
-  error_log($failed);
-  if (!$isSOGoRequest) {
-    $redis->publish("F2B_CHANNEL", $failed);
-  }
   http_response_code(401); // Unauthorized
 }
 

data/conf/dovecot/auth/passwd-verify.lua

@@ -1,4 +1,8 @@
 function auth_password_verify(request, password)
+  if request.domain == nil then
+    return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user"
+  end
+
   local json = require "cjson"
   local ltn12 = require "ltn12"
   local https = require "ssl.https"
@@ -7,10 +11,7 @@ function auth_password_verify(request, password)
   local req = {
     username = request.user,
     password = password,
-    -- address that connected to Dovecot, for proxied connections the proxy
     real_rip = request.real_rip,
-    -- address of the client, also for proxied connections
-    client_ip = request.rip,
     service = request.service
   }
   local req_json = json.encode(req)

data/conf/dovecot/dovecot.conf

@@ -51,6 +51,13 @@ recipient_delimiter = +
 auth_master_user_separator = *
 mail_shared_explicit_inbox = yes
 mail_prefetch_count = 30
+passdb {
+  driver = lua
+  args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
+  result_success = return-ok
+  result_failure = continue
+  result_internalfail = continue
+}
 # try a master passwd
 passdb {
   driver = passwd-file
@@ -60,9 +67,8 @@ passdb {
 }
 # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail
 # a return of the following passdb is mandatory
-# each check asks mailcowauth.php, which reports a failed login to netfilter once
 # cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry
-# of a passdb lookup (master logins) and reject the login as unknown user
+# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login
 passdb {
   driver = lua
   args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w
@@ -199,8 +205,6 @@ plugin {
   sieve_plugins = sieve_imapsieve sieve_extprograms
   sieve_vacation_send_from_recipient = yes
   sieve_redirect_envelope_from = recipient
-  # Notifications are sent with the address of the mailbox (not :from or postmaster), so mail policies apply
-  sieve_notify_mailto_envelope_from = recipient
   # From elsewhere to Spam folder
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY

data/conf/nginx/templates/nginx.conf.j2

@@ -47,13 +47,6 @@ http {
         default "";
     }
 
-    # Client address for PHP: on the internal sogo-auth listener nginx itself is the client,
-    # the real address comes from the X-Real-IP header set by /sogo-auth-verify
-    map $sogo_auth_internal $mailcow_client_ip {
-        "1"     $http_x_real_ip;
-        default $remote_addr;
-    }
-
     {% if HTTP_REDIRECT %}
     # HTTP to HTTPS redirect
     server {
@@ -150,15 +143,6 @@ http {
         access_log /var/log/nginx/access.log;
         root /dynmaps;
 
-        # backup endpoint of the restic runner: large restore requests, long merges of mailboxes and domains
-        location = /backup.php {
-            client_max_body_size 32m;
-            fastcgi_read_timeout 600s;
-            fastcgi_pass {{ PHPFPMHOST }}:9001;
-            include fastcgi_params;
-            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-        }
-
         location ~ \.php$ {
             try_files $uri =404;
             fastcgi_split_path_info ^(.+\.php)(/.+)$;

data/conf/nginx/templates/sites-default.conf.j2

@@ -22,8 +22,7 @@ add_header Referrer-Policy strict-origin;
 
 index index.php index.html;
 
-# PHP accepts no larger requests (post_max_size), SOGo webmail, CalDAV/CardDAV and ActiveSync use NGINX_MAX_BODY_SIZE
+client_max_body_size 0;
-client_max_body_size 64m;
 
 gzip on;
 gzip_disable "msie6";
@@ -117,8 +116,7 @@ location ~ \.php$ {
     include /etc/nginx/fastcgi_params;
     fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
     fastcgi_param PATH_INFO $fastcgi_path_info;
-    fastcgi_param REMOTE_ADDR $mailcow_client_ip;
+    fastcgi_param HTTP_X_REAL_IP $remote_addr;
-    fastcgi_param HTTP_X_REAL_IP $mailcow_client_ip;
     # trusted internal-auth marker; empty for external clients (see nginx.conf map)
     fastcgi_param SOGO_AUTH_INTERNAL $sogo_auth_internal;
     fastcgi_read_timeout 3600;
@@ -183,7 +181,6 @@ location /sogo-auth-verify {
     proxy_set_header  Content-Length "";
     proxy_pass        http://127.0.0.1:65510/sogo-auth;
     proxy_pass_request_body off;
-    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
 }
 
 location ^~ /Microsoft-Server-ActiveSync {
@@ -207,7 +204,7 @@ location ^~ /Microsoft-Server-ActiveSync {
     proxy_busy_buffers_size 512k;
     proxy_set_header Host $http_host;
     client_body_buffer_size 512k;
-    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
+    client_max_body_size 0;
 }
 
 location ^~ /SOGo {
@@ -258,7 +255,7 @@ location ^~ /SOGo {
     proxy_send_timeout 3600;
     proxy_read_timeout 3600;
     client_body_buffer_size 128k;
-    client_max_body_size {{ NGINX_MAX_BODY_SIZE }};
+    client_max_body_size 0;
     break;
 }
 

data/conf/postfix/main.cf

@@ -50,7 +50,6 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   $sender_dependent_default_transport_maps,
@@ -91,7 +90,6 @@ smtpd_hard_error_limit = ${stress?1}${stress:5}
 smtpd_helo_required = yes
 smtpd_proxy_timeout = 600s
 smtpd_recipient_restrictions = check_recipient_mx_access proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
-  check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   permit_sasl_authenticated,
   permit_mynetworks,
   check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
@@ -172,11 +170,5 @@ parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks
 # This Option is added to correctly set the X-Original-To Header when mails are send to lmtp (dovecot)
 lmtp_destination_recipient_limit=1
 
-# Mail policies: while policies or internal aliases exist, mysql_mail_policy_maps.cf returns this class and smtpd
-# asks the policy service for every recipient (mailcow-policy in master.cf); no answer defers the recipient
-smtpd_restriction_classes = mailcow_mail_policy
-mailcow_mail_policy = check_policy_service { unix:private/mailcow-policy, timeout=10s, try_limit=1, default_action=DEFER_IF_PERMIT }
-mailcow-policy_time_limit = 3600s
-
 # DO NOT EDIT ANYTHING BELOW #
 # Overrides #

data/conf/postfix/master.cf

@@ -117,9 +117,6 @@ smtp_sender_cleanup unix n - y - 0 cleanup
 127.0.0.1:10027 inet n n n - 0 spawn user=nobody argv=/usr/local/bin/whitelist_forwardinghosts.sh
 # end whitelist_fwd
 
-# mail policies: policy service of smtpd, asks the mailcow UI (mailcow_mail_policy in main.cf)
-mailcow-policy unix - n n - 0 spawn user=nobody argv=/usr/local/bin/mailcow-policy.py
-
 # start watchdog-specific
 # logs to local7 (hidden)
 589 inet n      -       n       -       -       smtpd

data/conf/rspamd/dynmaps/backup.php

@@ -1,97 +0,0 @@
-<?php
-// Backup with restic: endpoint of the runner in the restic container (Mailcow\Backup)
-// GET ?domains=1: {"domains": [...]}, all domains
-// GET ?export=<domain>[&after=<username>]: a page of the export of a domain (the first with domain_data and aliases,
-//   every page with a part of the mailboxes and "next_after", the last username to ask for the next page with, null
-//   after the last), incl. password hashes, DKIM keys and secrets; the runner merges the pages into
-//   /mailcow/export/<domain>.json of the snapshot.
-// POST {"action": "restore_domain", "job": <id>, "data": <domain_data>}: recreates a deleted domain
-// POST {"action": "restore_aliases", "job": <id>, "aliases": [...]}: a part (at most 500) of the aliases of a domain restore
-// POST {"action": "restore", "job": <id>, "username": <mailbox>, "data": <its section of the export>}: one mailbox
-//   (created first when it was deleted); all answer {"result": {<component>: {"added", "skipped", "errors"}}}
-// All need the header X-Backup-Token: HMAC-SHA256 of "backup" with the database password as key
-header('Content-Type: application/json');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'backup', $database_pass), (string)($_SERVER['HTTP_X_BACKUP_TOKEN'] ?? ''))) {
-  http_response_code(403);
-  exit;
-}
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("BACKUP: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-try {
-  // DKIM keys, rate limits and the domain map are kept in Redis
-  $redis = new Redis();
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-  $config = array(
-    'timezone' => getenv('TZ') ?: 'UTC',
-    'alias_disable_external_domains' => ($ALIAS_DISABLE_EXTERNAL_DOMAINS ?? false) === true,
-  );
-  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
-    $body = json_decode((string)file_get_contents('php://input'), true);
-    if (!is_array($body)) {
-      http_response_code(400);
-      exit;
-    }
-    // The restore has the side effects of the UI: SOGo view and restart, Dovecot auth cache, group shares (legacy code)
-    require_once __DIR__ . '/../web/inc/functions.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-    // The legacy mailbox functions run as admin, without a session
-    $_SESSION = array(
-      'mailcow_cc_username' => 'admin',
-      'mailcow_cc_role' => 'admin',
-      'access_all_exception' => '0',
-      'acl' => array_fill_keys(array('tls_policy', 'quarantine_notification', 'quarantine_category', 'ratelimit', 'sogo_access',
-        'protocol_access', 'mailbox_relayhost', 'unlimited_quota'), '1'),
-    );
-    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
-    echo json_encode($backup->api->post($body), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
-  }
-  else {
-    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
-    if (isset($_GET['export'])) {
-      echo json_encode($backup->api->export((string)$_GET['export'], (isset($_GET['after']) && is_string($_GET['after'])) ? $_GET['after'] : null),
-        JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
-    }
-    elseif (isset($_GET['domains'])) {
-      echo json_encode($backup->api->domains());
-    }
-    else {
-      http_response_code(400);
-    }
-  }
-}
-catch (\Mailcow\Core\UserError $e) {
-  http_response_code(($e->key === 'domain_not_found') ? 404 : 400);
-  echo json_encode(array('error' => $e->key, 'args' => $e->args));
-}
-catch (Throwable $e) {
-  error_log("BACKUP: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/mailpolicy.php

@@ -1,39 +0,0 @@
-<?php
-// Mail policy check for one recipient, asked by the Postfix policy service (mailcow-policy in master.cf)
-// Request: POST with the attributes of the Postfix policy request; answer: the action (DUNNO or the rejection)
-header('Content-Type: text/plain');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("MAILPOLICY: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-if (!isset($_POST['request']) || $_POST['request'] !== 'smtpd_access_policy') {
-  http_response_code(400);
-  exit;
-}
-$value = fn(string $key): string => is_string($_POST[$key] ?? null) ? $_POST[$key] : '';
-
-try {
-  $policy = \Mailcow\MailPolicy\MailPolicyModule::create($pdo, \Mailcow\Core\Context::system());
-  echo $policy->api->check($value('sender'), $value('recipient'), $value('sasl_username'), (int)$value('server_port'), $value('helo_name'));
-}
-catch (Throwable $e) {
-  // the policy service answers with a temporary error
-  error_log("MAILPOLICY: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/retention.php

@@ -1,52 +0,0 @@
-<?php
-// Retention rules for the nightly run in the dovecot container (retention_runner.py)
-// GET ?after=<username>&limit=<n>: a page of the plan, per mailbox the folders to clean up (JSON)
-// POST (JSON {"results": [...], "summary": [...], "run": {...}}): the results
-// Both need the header X-Retention-Token: HMAC-SHA256 of "retention" with the database password as key
-header('Content-Type: application/json');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'retention', $database_pass), (string)($_SERVER['HTTP_X_RETENTION_TOKEN'] ?? ''))) {
-  http_response_code(403);
-  exit;
-}
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("RETENTION: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-try {
-  $retention = \Mailcow\Retention\RetentionModule::create($pdo, \Mailcow\Core\Context::system());
-  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
-    $body = json_decode((string)file_get_contents('php://input'), true);
-    if (!is_array($body) || !is_array($body['results'] ?? null)) {
-      http_response_code(400);
-      exit;
-    }
-    echo json_encode(['stored' => $retention->api->report($body['results'], is_array($body['summary'] ?? null) ? $body['summary'] : [],
-      is_array($body['run'] ?? null) ? $body['run'] : null)]);
-  }
-  else {
-    echo json_encode($retention->api->plan((string)($_GET['after'] ?? ''), (int)($_GET['limit'] ?? 0)));
-  }
-}
-catch (\Mailcow\Core\UserError $e) {
-  http_response_code(400);
-}
-catch (Throwable $e) {
-  error_log("RETENTION: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/settings.php

@@ -469,5 +469,35 @@ while ($row = array_shift($rows)) {
 }
 ?>
 
+<?php
+// Start internal aliases
+
+$stmt = $pdo->query("SELECT `id`, `address`, `domain` FROM `alias` WHERE `active` = '1' AND `internal` = '1'");
+$aliases = $stmt->fetchAll(PDO::FETCH_ASSOC);
+while ($alias = array_shift($aliases)) {
+  // build allowed_domains regex and add target domain and alias domains
+  $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `active` = '1' AND `target_domain` = :target_domain");
+  $stmt->execute(array(':target_domain' => $alias['domain']));
+  $allowed_domains = $stmt->fetchAll(PDO::FETCH_ASSOC);
+  $allowed_domains = array_map(function($item) {
+    return str_replace('.', '\.', $item['alias_domain']);
+  }, $allowed_domains);
+  $allowed_domains[] = str_replace('.', '\.', $alias['domain']);
+  $allowed_domains = implode('|', $allowed_domains);
+?>
+  internal_alias_<?=$alias['id'];?> {
+    priority = 10;
+    rcpt = "<?=$alias['address'];?>";
+    from = "/^((?!.*@(<?=$allowed_domains;?>)).)*$/";
+    apply "default" {
+      MAILCOW_INTERNAL_ALIAS = 9999.0;
+    }
+    symbols [
+      "MAILCOW_INTERNAL_ALIAS"
+    ]
+  }
+<?php
+}
+?>
 
 }

data/conf/rspamd/lua/rspamd.local.lua

@@ -765,22 +765,13 @@ rspamd_config:register_symbol({
   end
 })
 
--- Expand a domain footer: replace only "{= key =}" placeholders with their values.
--- Unknown placeholders render empty, matching the previous engine behaviour.
-local function expand_footer(template, replacements)
-  return (template:gsub('{=%s*([%w_]+)%s*=}', function(key)
-    local value = replacements[key]
-    if value == nil then return '' end
-    return tostring(value)
-  end))
-end
-
 rspamd_config:register_symbol({
   name = 'MOO_FOOTER',
   type = 'prefilter',
   callback = function(task)
     local cjson = require "cjson"
     local lua_mime = require "lua_mime"
+    local lua_util = require "lua_util"
     local rspamd_logger = require "rspamd_logger"
     local rspamd_http = require "rspamd_http"
     local envfrom = task:get_from(1)
@@ -853,10 +844,10 @@ rspamd_config:register_symbol({
               end
             end
             if footer.html and footer.html ~= "" then
-              footer.html = expand_footer(footer.html, replacements)
+              footer.html = lua_util.jinja_template(footer.html, replacements, true)
             end
             if footer.plain and footer.plain ~= "" then
-              footer.plain = expand_footer(footer.plain, replacements)
+              footer.plain = lua_util.jinja_template(footer.plain, replacements, true)
             end
 
             -- add footer

data/conf/rspamd/meta_exporter/pipe.php

@@ -216,9 +216,8 @@ foreach ($rcpts as $rcpt) {
 foreach ($rcpt_final_mailboxes as $rcpt_final) {
   error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt_final . PHP_EOL);
   try {
-    // qhash is the secret of the release/delete links in notifications, so it must not be derivable
+    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`)
-    $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`, `fuzzy_hashes`, `qhash`)
+      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes)");
-      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action, :fuzzy_hashes, :qhash)");
     $stmt->execute(array(
       ':qid' => $qid,
       ':subject' => $subject,
@@ -230,9 +229,11 @@ foreach ($rcpt_final_mailboxes as $rcpt_final) {
       ':ip' => $ip,
       ':msg' => $raw_data,
       ':action' => $action,
-      ':fuzzy_hashes' => $fuzzy,
+      ':fuzzy_hashes' => $fuzzy
-      ':qhash' => bin2hex(random_bytes(32))
     ));
+    $lastId = $pdo->lastInsertId();
+    $stmt_update = $pdo->prepare("UPDATE `quarantine` SET `qhash` = SHA2(CONCAT(`id`, `qid`), 256) WHERE `id` = :id");
+    $stmt_update->execute(array(':id' => $lastId));
     $stmt = $pdo->prepare('DELETE FROM `quarantine` WHERE `rcpt` = :rcpt AND `id` NOT IN (
       SELECT `id`
       FROM (

data/web/admin/mailbox.php

@@ -11,8 +11,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
-$js_minifier->add('/web/js/site/mail-policy.js');
-$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -48,8 +46,6 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
-  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
-  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/restore.php

@@ -1,8 +0,0 @@
-<?php
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
-
-protect_route(['admin']);
-
-$role = "admin";
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/admin/system.php

@@ -13,7 +13,6 @@ $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 $js_minifier->add('/web/js/site/identity.js');
-$js_minifier->add('/web/js/site/backup.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -122,9 +121,6 @@ $template_data = [
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
   'iam_settings' => $iam_settings,
   'mbox_templates' => $mbox_templates,
-  'backup_timezone' => getenv('TZ') ?: 'UTC',
-  'backup_hostname' => getenv('MAILCOW_HOSTNAME') ?: '',
-  'backup_demo_mode' => $DEMO_MODE,
   'lang_admin' => json_encode($lang['admin']),
   'lang_datatables' => json_encode($lang['datatables'])
 ];

data/web/autodiscover.php

@@ -144,38 +144,6 @@ try {
 }
 
 $username = trim((string)$email);
-if (!filter_var($username, FILTER_VALIDATE_EMAIL)) {
-  try {
-    $json = json_encode(
-      array(
-        "time" => time(),
-        "ua" => $_SERVER['HTTP_USER_AGENT'],
-        "user" => "none",
-        "ip" => $_SERVER['REMOTE_ADDR'],
-        "service" => "Error: invalid email address in request"
-      )
-    );
-    $redis->lPush('AUTODISCOVER_LOG', $json);
-    $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
-    $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
-    error_log("Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
-  }
-  catch (RedisException $e) {
-    // Silently fail
-  }
-  list($usec, $sec) = explode(' ', microtime());
-?>
-  <Response>
-    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
-      <ErrorCode>600</ErrorCode>
-      <Message>Invalid Request</Message>
-      <DebugData />
-    </Error>
-  </Response>
-</Autodiscover>
-<?php
-  exit(0);
-}
 try {
   $stmt = $pdo->prepare("SELECT `mailbox`.`name`, `mailbox`.`active` FROM `mailbox` 
     INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
@@ -201,8 +169,8 @@ catch(PDOException $e) {
   exit(0);
 }
 
-$email = $username;
+// Mailbox not found or not active - return generic error to prevent user enumeration
-$displayname = $email;
+if (empty($MailboxData)) {
   try {
     $json = json_encode(
       array(
@@ -210,16 +178,49 @@ try {
         "ua" => $_SERVER['HTTP_USER_AGENT'],
         "user" => $email,
         "ip" => $_SERVER['REMOTE_ADDR'],
-      // The answer is the same, only the log tells unknown or inactive mailboxes apart
+        "service" => "Error: mailbox not found or inactive"
-      "service" => (empty($MailboxData)) ? "Error: mailbox not found or inactive" : $autodiscover_config['autodiscoverType']
       )
     );
     $redis->lPush('AUTODISCOVER_LOG', $json);
     $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
-  if (empty($MailboxData)) {
     $redis->publish("F2B_CHANNEL", "Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
     error_log("Autodiscover: Invalid mailbox attempt by " . $_SERVER['REMOTE_ADDR']);
   }
+  catch (RedisException $e) {
+    // Silently fail
+  }
+  list($usec, $sec) = explode(' ', microtime());
+?>
+  <Response>
+    <Error Time="<?=date('H:i:s', $sec) . substr($usec, 0, strlen($usec) - 2);?>" Id="<?=rand(1000000000, 9999999999);?>">
+      <ErrorCode>600</ErrorCode>
+      <Message>Invalid Request</Message>
+      <DebugData />
+    </Error>
+  </Response>
+</Autodiscover>
+<?php
+  exit(0);
+}
+
+if (!empty($MailboxData['name'])) {
+  $displayname = $MailboxData['name'];
+}
+else {
+  $displayname = $email;
+}
+try {
+  $json = json_encode(
+    array(
+      "time" => time(),
+      "ua" => $_SERVER['HTTP_USER_AGENT'],
+      "user" => $email,
+      "ip" => $_SERVER['REMOTE_ADDR'],
+      "service" => $autodiscover_config['autodiscoverType']
+    )
+  );
+  $redis->lPush('AUTODISCOVER_LOG', $json);
+  $redis->lTrim('AUTODISCOVER_LOG', 0, 100);
 }
 catch (RedisException $e) {
   $_SESSION['return'][] = array(
@@ -242,7 +243,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['imap']['server'];?></Server>
         <Port><?=$autodiscover_config['imap']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
+        <LoginName><?=$email;?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -252,7 +253,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
         <Server><?=$autodiscover_config['smtp']['server'];?></Server>
         <Port><?=$autodiscover_config['smtp']['port'];?></Port>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
+        <LoginName><?=$email;?></LoginName>
         <SPA>off</SPA>
         <SSL>on</SSL>
         <AuthRequired>on</AuthRequired>
@@ -264,15 +265,15 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     ?>
       <Protocol>
         <Type>CalDAV</Type>
-        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
+        <Server>https://<?=$autodiscover_config['caldav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['caldav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
+        <LoginName><?=$email;?></LoginName>
       </Protocol>
       <Protocol>
         <Type>CardDAV</Type>
-        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?>/</Server>
+        <Server>https://<?=$autodiscover_config['carddav']['server'];?><?php if ($autodiscover_config['caldav']['port'] != 443) echo ':'.$autodiscover_config['carddav']['port']; ?>/SOGo/dav/<?=$email;?>/</Server>
         <DomainRequired>off</DomainRequired>
-        <LoginName><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></LoginName>
+        <LoginName><?=$email;?></LoginName>
       </Protocol>
     <?php
     }
@@ -287,7 +288,7 @@ if ($autodiscover_config['autodiscoverType'] == 'imap') {
     <Culture>en:en</Culture>
     <User>
       <DisplayName><?=htmlspecialchars($displayname, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></DisplayName>
-      <EMailAddress><?=htmlspecialchars($email, ENT_XML1 | ENT_QUOTES, 'UTF-8');?></EMailAddress>
+      <EMailAddress><?=$email;?></EMailAddress>
     </User>
     <Action>
       <Settings>

data/web/css/site/admin.css

@@ -81,53 +81,3 @@ table tbody tr td input[type="checkbox"] {
 .dkim-label {
   margin: 0 0 8px !important;
 }
-/* backup schedule: the weekday toggles and the remove buttons of the entries react to the mouse, in every theme */
-.backup-schedule-days .btn:hover {
-  background-image: linear-gradient(rgba(128, 128, 128, .3), rgba(128, 128, 128, .3));
-}
-.backup-schedule-remove {
-  opacity: .6;
-}
-.backup-schedule-remove:hover, .backup-schedule-remove:focus-visible {
-  opacity: 1;
-  color: var(--bs-danger) !important;
-}
-/* result and log of a backup job: the border of the theme in light and dark mode (Bootstrap's .border stays light) */
-.backup-pre {
-  border: 1px solid var(--mc-border);
-}
-/* result of a restore job: an entry to open per mailbox, numbers in columns */
-.backup-result {
-  border: 1px solid var(--mc-border);
-}
-.backup-result > details + details {
-  border-top: 1px solid var(--mc-border);
-}
-.backup-result summary {
-  list-style: none;
-  cursor: pointer;
-}
-.backup-result summary::-webkit-details-marker {
-  display: none;
-}
-.backup-result summary:hover {
-  background-image: linear-gradient(rgba(128, 128, 128, .12), rgba(128, 128, 128, .12));
-}
-.backup-result summary .bi-chevron-right {
-  transition: transform .15s;
-}
-.backup-result details[open] > summary .bi-chevron-right {
-  transform: rotate(90deg);
-}
-.backup-result-body {
-  padding-left: 2.6rem !important;
-}
-.backup-result-body .table {
-  --bs-table-bg: transparent;
-}
-.backup-result-body .num {
-  width: 8.5rem;
-  text-align: right;
-  font-variant-numeric: tabular-nums;
-  white-space: nowrap;
-}

data/web/css/themes/mailcow-darkmode.css

@@ -496,15 +496,3 @@ input::placeholder {
 .btn-light, .btn-light:hover {
     background-image: none;
 }
-/* native time picker icon and popup in dark mode */
-input[type="time"] {
-    color-scheme: dark;
-}
-/* switches while off: a light knob and a visible border (Bootstrap's dark knob is for light pages, the dark mode
-   does not set data-bs-theme) */
-.form-switch .form-check-input:not(:checked):not(:focus) {
-    --bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.6%29'/%3e%3c/svg%3e");
-}
-.form-switch .form-check-input:not(:checked) {
-    border-color: var(--mc-btn-outline-border);
-}

data/web/domainadmin/mailbox.php

@@ -11,8 +11,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
-$js_minifier->add('/web/js/site/mail-policy.js');
-$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -47,8 +45,6 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
-  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
-  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/domainadmin/restore.php

@@ -1,8 +0,0 @@
-<?php
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.domainadmin.inc.php';
-
-protect_route(['domainadmin']);
-
-$role = "domainadmin";
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/edit.php

@@ -115,28 +115,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
         ];
     }
-    elseif (isset($_GET['mail-policy']) &&
-      is_numeric($_GET['mail-policy'])) {
-        $mail_policy = \Mailcow\MailPolicy\MailPolicyModule::fromGlobals();
-        $result = $mail_policy->api->getPolicy($_GET['mail-policy']);
-        $template = 'edit/mail-policy.twig';
-        // Choices: mailboxes and groups of the domain of the policy
-        $template_data = [
-          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
-          'mail_policy_groups' => !empty($result) ? $mail_policy->policies->groupChoices([$result['domain']]) : [],
-        ];
-    }
-    elseif (isset($_GET['retention-rule']) &&
-      is_numeric($_GET['retention-rule'])) {
-        $retention = \Mailcow\Retention\RetentionModule::fromGlobals();
-        $result = $retention->api->getRule($_GET['retention-rule']);
-        $template = 'edit/retention-rule.twig';
-        // Choices: mailboxes and groups of the domain of the rule
-        $template_data = [
-          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
-          'retention_groups' => !empty($result) ? $retention->rules->groupChoices([$result['domain']]) : [],
-        ];
-    }
     elseif (isset($_GET['api-key']) &&
       is_numeric($_GET['api-key'])) {
         $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
@@ -249,8 +227,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
     if (isset($_GET['syncjob']) &&
       is_numeric($_GET['syncjob'])) {
         $id = $_GET["syncjob"];
-        // Without the syncjobs ACL a job cannot be saved, so do not show the form
+        $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id);
-        $result = (($_SESSION['acl']['syncjobs'] ?? '0') == '1') ? \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id) : [];
         $template = 'edit/syncjob.twig';
       }
     elseif (isset($_GET['syncjob_source']) &&

data/web/inc/footer.inc.php

@@ -74,9 +74,6 @@ $globalVariables = [
   'lang_danger' => json_encode($lang['danger']),
   'lang_syncjobs' => json_encode($lang['syncjobs']),
   'lang_groups' => json_encode($lang['groups']),
-  'lang_mail_policy' => json_encode($lang['mail_policy'] ?? []),
-  'lang_retention' => json_encode($lang['retention'] ?? []),
-  'lang_backup' => json_encode($lang['backup'] ?? []),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.admin.inc.php

@@ -74,15 +74,6 @@ function admin($_action, $_data = null) {
         $usernames = $_data['username'];
       }
       foreach ($usernames as $username) {
-        // A pending password update only allows changing the own password
-        if (!empty($_SESSION['pending_pw_update']) && $username != $_SESSION['mailcow_cc_username']) {
-          $_SESSION['return'][] = array(
-            'type' => 'danger',
-            'log' => array(__FUNCTION__, $_action, $_data_log),
-            'msg' => 'access_denied'
-          );
-          continue;
-        }
         $is_now = admin('details', $username);
         if (!empty($is_now)) {
           $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];

data/web/inc/functions.app_passwd.inc.php

@@ -5,15 +5,6 @@ function app_passwd($_action, $_data = null) {
   $_data_log = $_data;
   !isset($_data_log['app_passwd']) ?: $_data_log['app_passwd'] = '*';
   !isset($_data_log['app_passwd2']) ?: $_data_log['app_passwd2'] = '*';
-  if (in_array($_action, array('add', 'edit', 'delete')) &&
-    (!isset($_SESSION['acl']['app_passwds']) || $_SESSION['acl']['app_passwds'] != "1")) {
-    $_SESSION['return'][] = array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $_action, $_data_log),
-      'msg' => 'access_denied'
-    );
-    return false;
-  }
   if (isset($_data['username']) && filter_var($_data['username'], FILTER_VALIDATE_EMAIL)) {
     if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data['username'])) {
       $_SESSION['return'][] = array(

data/web/inc/functions.auth.inc.php

@@ -6,8 +6,6 @@ function check_login($user, $pass, $extra = null) {
   $is_internal = $extra['is_internal'];
   $role = $extra['role'];
   $extra['service'] = !isset($extra['service']) ? 'NONE' : $extra['service'];
-  // set when the password is right, but the account does not allow this login
-  $denied = false;
 
   // Try validate admin
   if (!isset($role) || $role == "admin") {
@@ -28,7 +26,7 @@ function check_login($user, $pass, $extra = null) {
 
   // Try validate app password
   if (!isset($role) || $role == "app") {
-    $result = apppass_login($user, $pass, $extra, $denied);
+    $result = apppass_login($user, $pass, $extra);
     if ($result !== false) {
       $real_rip = ($_SERVER['HTTP_X_REAL_IP'] ?? $_SERVER['REMOTE_ADDR']);
       set_sasl_log($user, $real_rip, $extra['service'], $pass);
@@ -39,7 +37,7 @@ function check_login($user, $pass, $extra = null) {
   // Try validate user
   if (!isset($role) || $role == "user") {
     try {
-      $result = user_login($user, $pass, $extra, $denied);
+      $result = user_login($user, $pass, $extra);
     } catch (\Mailcow\Auth\IdpUnavailable $e) {
       // no failed login, netfilter must not count it
       error_log("mailcow UI: Identity provider unavailable for " . $user . " by " . $_SERVER['REMOTE_ADDR'] . ": " . $e->getMessage());
@@ -56,20 +54,19 @@ function check_login($user, $pass, $extra = null) {
       return $result;
     }
   }
+
   // skip log and only return false if it's an internal request
   if ($is_internal == true) return false;
 
-  // A denied login is logged, but not counted by netfilter (e.g. EAS/DAV clients of accounts without access)
-  $msg = ($denied ? "mailcow UI: Login denied for " : "mailcow UI: Invalid password for ") . $user . " by " . $_SERVER['REMOTE_ADDR'];
   if (!isset($_SESSION['ldelay'])) {
     $_SESSION['ldelay'] = "0";
-    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
+    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
-    error_log($msg);
+    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
   }
   elseif (!isset($_SESSION['mailcow_cc_username'])) {
     $_SESSION['ldelay'] = $_SESSION['ldelay']+0.5;
-    if (!$denied) $redis->publish("F2B_CHANNEL", $msg);
+    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
-    error_log($msg);
+    error_log("mailcow UI: Invalid password for " . $user . " by " . $_SERVER['REMOTE_ADDR']);
   }
   $_SESSION['return'][] =  array(
     'type' => 'danger',
@@ -210,7 +207,7 @@ function domainadmin_login($user, $pass){
 
   return false;
 }
-function user_login($user, $pass, $extra = null, &$denied = false){
+function user_login($user, $pass, $extra = null){
   global $pdo;
   global $iam_provider;
   global $iam_settings;
@@ -264,14 +261,11 @@ function user_login($user, $pass, $extra = null, &$denied = false){
         if ($extra['service'] != 'NONE') {
           $key = strtolower($extra['service']) . "_access";
           if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
-            $denied = true;
             return false;
           }
         }
         return true;
       }
-      // the IdP accepted the password, the mailbox is inactive
-      $denied = true;
     }
     clear_session();
     return false;
@@ -279,16 +273,18 @@ function user_login($user, $pass, $extra = null, &$denied = false){
 
   // check if user has access to service (imap, smtp, pop3, sieve) if service is set
   $row['attributes'] = json_decode($row['attributes'], true);
-  $key = strtolower($extra['service']) . "_access";
+  // Shared mailboxes have no login of their own, members open them through group shares
-  // Shared mailboxes have no login of their own, members open them through group shares.
+  if (($row['attributes']['shared'] ?? '0') == '1') {
-  // Checked before asking an IdP (no template side effects).
+    return false;
-  if (($row['attributes']['shared'] ?? '0') == '1'
-    || ($extra['service'] != 'NONE' && isset($row['attributes'][$key]) && $row['attributes'][$key] != '1')
-    || $row['active'] != '1' || $row['d_active'] != '1') {
-    // Denied, not failed, unless the password is wrong; IdP passwords are not checked for that
-    if ($row['authsource'] != 'mailcow' || verify_hash($row['password'], $pass) !== false) {
-      $denied = true;
   }
+  if ($extra['service'] != 'NONE') {
+    $key = strtolower($extra['service']) . "_access";
+    if (isset($row['attributes'][$key]) && $row['attributes'][$key] != '1') {
+      return false;
+    }
+  }
+  // IdP accounts: refuse inactive mailboxes and domains before asking the IdP (no template side effects)
+  if ($row['authsource'] != 'mailcow' && ($row['active'] != '1' || $row['d_active'] != '1')) {
     return false;
   }
   switch ($row['authsource']) {
@@ -307,7 +303,6 @@ function user_login($user, $pass, $extra = null, &$denied = false){
           $stmt->execute(array(':user' => $user));
           $row = $stmt->fetch(PDO::FETCH_ASSOC);
           if (empty($row)) {
-            $denied = true;
             return false;
           }
 
@@ -366,7 +361,6 @@ function user_login($user, $pass, $extra = null, &$denied = false){
         $stmt->execute(array(':user' => $user));
         $row = $stmt->fetch(PDO::FETCH_ASSOC);
         if (empty($row)) {
-          $denied = true;
           return false;
         }
 
@@ -409,6 +403,9 @@ function user_login($user, $pass, $extra = null, &$denied = false){
       return $result;
     break;
     case 'mailcow':
+      if ($row['active'] != 1 || $row['d_active'] != 1) {
+        return false;
+      }
       // verify password
       if (verify_hash($row['password'], $pass) !== false) {
 
@@ -449,15 +446,13 @@ function user_login($user, $pass, $extra = null, &$denied = false){
           }
           return "user";
         }
-        // TFA accounts use app passwords in mail clients
-        $denied = true;
       }
     break;
   }
 
   return false;
 }
-function apppass_login($user, $pass, $extra = null, &$denied = false){
+function apppass_login($user, $pass, $extra = null){
   global $pdo;
 
   $is_internal = $extra['is_internal'];
@@ -478,12 +473,15 @@ function apppass_login($user, $pass, $extra = null, &$denied = false){
     return false;
   }
 
-  // fetch app password data, inactive ones too: their right password is denied, not failed
+  // fetch app password data
-  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id`,
+  $stmt = $pdo->prepare("SELECT `app_passwd`.*, `app_passwd`.`password` as `password`, `app_passwd`.`id` as `app_passwd_id` FROM `app_passwd`
-    `mailbox`.`active` as `mailbox_active`, `domain`.`active` as `domain_active`, `mailbox`.`attributes` as `mailbox_attributes` FROM `app_passwd`
     INNER JOIN `mailbox` ON `mailbox`.`username` = `app_passwd`.`mailbox`
     INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`
     WHERE `mailbox`.`kind` NOT REGEXP 'location|thing|group'
+      AND `mailbox`.`active` = '1'
+      AND `domain`.`active` = '1'
+      AND `app_passwd`.`active` = '1'
+      AND COALESCE(JSON_UNQUOTE(JSON_VALUE(`mailbox`.`attributes`, '$.shared')), '0') != '1'
       AND `app_passwd`.`mailbox` = :user"
   );
   // fetch password data
@@ -493,26 +491,17 @@ function apppass_login($user, $pass, $extra = null, &$denied = false){
   $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
 
   foreach ($rows as $row) {
+    if ($extra['service'] != 'NONE' && $row[strtolower($extra['service']) . '_access'] != '1'){
+      continue;
+    }
+
     // verify password
-    if (verify_hash($row['password'], $pass) === false) {
+    if (verify_hash($row['password'], $pass) !== false) {
-      continue;
-    }
-
-    $key = strtolower($extra['service']) . '_access';
-    $mailbox_attributes = json_decode($row['mailbox_attributes'], true);
-    // the protocols of the mailbox only limit direct user access, app passwords have their own
-    if ($row['active'] != '1' || $row['mailbox_active'] != '1' || $row['domain_active'] != '1'
-      || ($mailbox_attributes['shared'] ?? '0') == '1'
-      || ($extra['service'] != 'NONE' && $row[$key] != '1')) {
-      // right password, but the app password does not allow this login
-      $denied = true;
-      continue;
-    }
-
       $_SESSION['app_passwd_id'] = $row['app_passwd_id'];
       unset($_SESSION['ldelay']);
       return "user";
     }
+  }
 
   return false;
 }
@@ -549,8 +538,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
 
   // get the mailcow_password attribute from keycloak user
   $url = "{$iam_settings['server_url']}/admin/realms/{$iam_settings['realm']}/users";
-  // as string, http_build_query() would send exact=1 and Keycloak would search for substrings
+  $queryParams = array('email' => $user, 'exact' => true);
-  $queryParams = array('email' => $user, 'exact' => 'true');
   $queryString = http_build_query($queryParams);
   $curl = curl_init();
   curl_setopt($curl, CURLOPT_TIMEOUT, 7);
@@ -571,10 +559,7 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
   if ($code != 200) {
     throw new \Mailcow\Auth\IdpUnavailable('Keycloak returned HTTP ' . $code);
   }
-  // exact=true can still return several users with that email, only the exact address counts
+  // exact=true can still return several users with that email
-  if (is_array($users)) {
-    $users = array_values(array_filter($users, fn($u) => is_array($u) && strcasecmp((string)($u['email'] ?? ''), $user) === 0));
-  }
   if (!is_array($users) || count($users) != 1) {
     return false;
   }
@@ -725,15 +710,12 @@ function ldap_mbox_login($user, $pass, $extra = null){
       ->select(array_values(array_filter(array_merge([$iam_settings['username_field'], $iam_settings['attribute_field'], 'displayname', 'distinguishedname', 'dn'],
         \Mailcow\Identity\Directory\LdapDirectory::idAttributes($iam_settings['id_attribute'] ?? '')))));
 
-    $entries = $ldap_query->get();
+    $user_res = $ldap_query->firstOrFail();
+  } catch (\LdapRecord\Query\ObjectNotFoundException $e) {
+    return false;
   } catch (Exception $e) {
     throw new \Mailcow\Auth\IdpUnavailable('LDAP search failed: ' . $e->getMessage(), 0, $e);
   }
-  // the login name must identify exactly one entry, several matches are ambiguous
-  if (!is_array($entries) || count($entries) != 1) {
-    return false;
-  }
-  $user_res = reset($entries);
   $idp_login = \Mailcow\Auth\AuthModule::fromGlobals()->idp;
   if (!$idp_login->verifyLdapPassword($iam_provider, (string)$user_res['dn'], (string)$pass)) {
     return false;

data/web/inc/functions.fail2ban.inc.php

@@ -145,28 +145,6 @@ function fail2ban($_action, $_data = null, $_extra = null) {
           );
           return true;
         }
-        // Queue all automatic bans for unbanning, the denylist stays
-        elseif ($_data['action'] == "unban-all") {
-          try {
-            foreach ($redis->hKeys('F2B_ACTIVE_BANS') as $network) {
-              $redis->hSet('F2B_QUEUE_UNBAN', $network, 1);
-            }
-          }
-          catch (RedisException $e) {
-            $_SESSION['return'][] = array(
-              'type' => 'danger',
-              'log' => array(__FUNCTION__, $_action, $_data_log),
-              'msg' => array('redis_error', $e)
-            );
-            return false;
-          }
-          $_SESSION['return'][] = array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, $_action, $_data_log),
-            'msg' => 'f2b_modified'
-          );
-          return true;
-        }
 
         // Start actions in dependency of network
         if (!empty($_data['network'])) {

data/web/inc/functions.inc.php

@@ -1857,40 +1857,6 @@ function get_tfa($username = null, $id = null) {
     }
 }
 function verify_tfa_login($username, $_data) {
-  global $redis;
-  global $TFA_MAX_FAILS;
-  global $TFA_FAIL_WINDOW;
-
-  if (empty($username)) {
-    return false;
-  }
-  // Count failed second factors per account, so codes cannot be guessed from many addresses
-  $fails_key = 'TFA_FAILS_' . $username;
-  if (intval($redis->get($fails_key)) >= $TFA_MAX_FAILS) {
-    $_SESSION['return'][] =  array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $username, '*'),
-      'msg' => 'tfa_login_locked'
-    );
-    $verified = false;
-  }
-  else {
-    $verified = verify_tfa_token($username, $_data);
-  }
-  if ($verified !== true) {
-    // The window starts with the first failure and is not extended by further attempts
-    if ($redis->incr($fails_key) == 1) {
-      $redis->expire($fails_key, $TFA_FAIL_WINDOW);
-    }
-    // Same message format as failed passwords, so the default fail2ban rule counts it
-    error_log("mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
-    $redis->publish("F2B_CHANNEL", "mailcow UI: Invalid password for " . $username . " (second factor) by " . $_SERVER['REMOTE_ADDR']);
-    return false;
-  }
-  $redis->del($fails_key);
-  return true;
-}
-function verify_tfa_token($username, $_data) {
   global $pdo;
   global $yubi;
   global $u2f;
@@ -3047,18 +3013,6 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         return false;
       }
 
-      // A refresh keeps the logged-in mailbox: a changed IdP email or another IdP user ends the session
-      if (strtolower($info['email']) !== strtolower($_SESSION['mailcow_cc_username'] ?? '') ||
-        !\Mailcow\Auth\AuthModule::fromGlobals()->idp->verifyLink($iam_settings['authsource'], $info['email'], (string)($info['sub'] ?? ''))) {
-        clear_session();
-        $_SESSION['return'][] =  array(
-          'type' => 'danger',
-          'log' => array(__FUNCTION__, $info['email'], 'The IdP user does not match the logged-in mailbox'),
-          'msg' => 'refresh_login_failed'
-        );
-        return false;
-      }
-
       // The mailbox may have been deleted or deactivated since the login
       $stmt = $pdo->prepare("SELECT `mailbox`.`authsource` FROM `mailbox`
         INNER JOIN `domain` ON `mailbox`.`domain` = `domain`.`domain`

data/web/inc/functions.mailbox.inc.php

@@ -2432,7 +2432,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               if ($address != $is_now['address']) {
                 // Send-as grants of groups follow the address
                 \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
-                \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
               }
             }
             $_SESSION['return'][] = array(
@@ -3296,9 +3295,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'mailbox_group_imap_grant' => ['owner'],
               'mailbox_group_sogo_grant' => ['owner'],
               'mailbox_group_sogo_acl' => ['c_uid'],
-              'mailbox_group_subscription' => ['username', 'owner'],
+              'mailbox_group_subscription' => ['username', 'owner']
-              'mail_policy_member' => ['username'],
-              'retention_rule_member' => ['username']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3314,8 +3311,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
             // Group memberships and send-as grants follow, group aliases are rebuilt
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
-            // Address entries of mail policies follow
-            \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($old_username, $new_username);
 
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
@@ -5353,7 +5348,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ':domain' => $domain,
             ));
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
-            \Mailcow\Retention\RetentionModule::fromGlobals()->api->domainDeleted($domain);
             $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,

data/web/inc/functions.quarantine.inc.php

@@ -767,9 +767,8 @@ function quarantine($_action, $_data = null) {
       else {
         $domains = array_merge(mailbox('get', 'domains'), mailbox('get', 'alias_domains'));
         foreach ($domains as $domain) {
-          // Compare the domain part exactly, a regex would treat dots in the domain as wildcards
+          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE `rcpt` REGEXP :domain');
-          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, LOCATE("VIRUS_FOUND", `symbols`) AS `virus_flag`, `score`, `rcpt`, `sender`, `action`, UNIX_TIMESTAMP(`created`) AS `created`, `notified` FROM `quarantine` WHERE SUBSTRING_INDEX(`rcpt`, "@", -1) = :domain');
+          $stmt->execute(array(':domain' => '@' . $domain . '$'));
-          $stmt->execute(array(':domain' => $domain));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while($row = array_shift($rows)) {
             $q_meta[] = $row;

data/web/inc/init_db.inc.php

@@ -491,257 +491,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      // Mail policies: whom the assigned mailboxes and groups may send to and receive from (Mailcow\MailPolicy)
-      "mail_policy" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          // all, internal or list; the entries are JSON lists of addresses, domains and group:<id>
-          "send_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
-          "send_allow" => "MEDIUMTEXT NOT NULL",
-          "send_block" => "MEDIUMTEXT NOT NULL",
-          "receive_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
-          "receive_allow" => "MEDIUMTEXT NOT NULL",
-          "receive_block" => "MEDIUMTEXT NOT NULL",
-          "reject_text" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "domain_name" => array("domain", "name")
-          ),
-          "fkey" => array(
-            "fk_mail_policy_domain" => array(
-              "col" => "domain",
-              "ref" => "domain.domain",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // a mailbox or a group (its members and its address) a policy is assigned to
-      "mail_policy_member" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "policy_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NULL DEFAULT NULL",
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "policy_mailbox" => array("policy_id", "username"),
-            "policy_group" => array("policy_id", "group_id")
-          ),
-          "key" => array(
-            "username" => array("username"),
-            "group_id" => array("group_id")
-          ),
-          "fkey" => array(
-            "fk_mail_policy_member_policy" => array(
-              "col" => "policy_id",
-              "ref" => "mail_policy.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mail_policy_member_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mail_policy_member_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "retention_rule" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          // JSON list: trash, junk, sent, drafts, archive, inbox, all, folder:<name>
-          "folders" => "TEXT NOT NULL",
-          "days" => "INT NOT NULL DEFAULT '30'",
-          "keep_flagged" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // simulate (count only) or delete
-          "mode" => "VARCHAR(16) NOT NULL DEFAULT 'simulate'",
-          "all_mailboxes" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "domain_name" => array("domain", "name")
-          ),
-          "fkey" => array(
-            "fk_retention_rule_domain" => array(
-              "col" => "domain",
-              "ref" => "domain.domain",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // a mailbox or a group (its members of the rule's domain) a retention rule is assigned to
-      "retention_rule_member" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "rule_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NULL DEFAULT NULL",
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "rule_mailbox" => array("rule_id", "username"),
-            "rule_group" => array("rule_id", "group_id")
-          ),
-          "key" => array(
-            "username" => array("username"),
-            "group_id" => array("group_id")
-          ),
-          "fkey" => array(
-            "fk_retention_rule_member_rule" => array(
-              "col" => "rule_id",
-              "ref" => "retention_rule.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_retention_rule_member_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_retention_rule_member_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // results of the nightly run: per mailbox and rule with found mail or an error (folder: the folder names), per
-      // rule a summary (username '') and the whole run (username '', rule_id NULL); kept 90 days, of simulating
-      // rules only the last run
-      "retention_log" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "rule_id" => "INT NULL DEFAULT NULL",
-          "rule_name" => "VARCHAR(100) NOT NULL DEFAULT ''",
-          "folder" => "TEXT NULL DEFAULT NULL",
-          "days" => "INT NOT NULL DEFAULT '0'",
-          "mailboxes" => "INT NOT NULL DEFAULT '0'",
-          "matched" => "INT NOT NULL DEFAULT '0'",
-          "deleted" => "INT NOT NULL DEFAULT '0'",
-          "errors" => "INT NOT NULL DEFAULT '0'",
-          "simulated" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "error" => "TEXT NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "domain_created" => array("domain", "created"),
-            "rule_created" => array("rule_id", "username", "created"),
-            "created" => array("created")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_setting" => array(
-        "cols" => array(
-          "name" => "VARCHAR(64) NOT NULL",
-          "value" => "TEXT NOT NULL",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("name")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_job" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "type" => "ENUM('backup', 'check', 'prune', 'init', 'restore') NOT NULL",
-          "state" => "ENUM('queued', 'running', 'done', 'failed', 'cancelled') NOT NULL DEFAULT 'queued'",
-          "step" => "VARCHAR(32) NULL DEFAULT NULL",
-          "params" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "result" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "log" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "error" => "TEXT NULL DEFAULT NULL",
-          "domain" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "started" => "DATETIME NULL DEFAULT NULL",
-          "finished" => "DATETIME NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "state_type" => array("state", "type"),
-            "domain_created" => array("domain", "created")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_snapshot" => array(
-        "cols" => array(
-          "id" => "VARCHAR(64) NOT NULL",
-          "short_id" => "VARCHAR(16) NOT NULL",
-          "time" => "DATETIME NOT NULL",
-          "hostname" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "tags" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "summary" => "TEXT NULL DEFAULT NULL",
-          "domains" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "content" => "MEDIUMBLOB NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "time" => array("time")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "mailbox_group_imap_grant" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -954,8 +703,7 @@ function init_db_schema()
             "group_id" => array("group_id")
           ),
           "key" => array(
-            "domain" => array("domain"),
+            "domain" => array("domain")
-            "internal" => array("internal", "active")
           ),
           "fkey" => array(
             "fk_alias_mailbox_group" => array(
@@ -1305,9 +1053,7 @@ function init_db_schema()
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
-          "mail_policies" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "retention" => "TINYINT(1) NOT NULL DEFAULT '0'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1352,7 +1098,6 @@ function init_db_schema()
           "host1" => "VARCHAR(255) NOT NULL",
           "port1" => "SMALLINT UNSIGNED NOT NULL",
           "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'",
-          "verify_cert" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "auth_type" => "ENUM('PLAIN','LOGIN','CRAM-MD5','XOAUTH2') DEFAULT 'PLAIN'",
           "oauth_flow" => "ENUM('client_credentials','authorization_code') NOT NULL DEFAULT 'client_credentials'",
           "oauth_token_endpoint" => "VARCHAR(500) DEFAULT NULL",
@@ -1934,15 +1679,9 @@ function init_db_schema()
             // Converted JSON may not fit into the old VARCHAR(512)
             $pdo->exec("ALTER TABLE `imapsync` MODIFY COLUMN `custom_params` TEXT");
             $imapsync_allow = \Mailcow\Imapsync\Model\CustomParams::allowlist($GLOBALS['IMAPSYNC_OPTIONS'] ?? [], $GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? []);
-            // A legacy source is found by its connection and its only target domain (none for jobs without domain)
+            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
-            $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` s WHERE `created_by` = '' AND `host1` = :host AND `port1` = :port
+            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `auth_type`, `active`)
-              AND `enc1` = :enc AND `auth_type` = :auth
+              VALUES (:name, '', '', 'domain', :host, :port, :enc, :auth, 1)");
-              AND (SELECT GROUP_CONCAT(`domain`) FROM `imapsync_source_domain` d WHERE d.`source_id` = s.`id`) <=> NULLIF(:domain, '')
-              ORDER BY `id` LIMIT 1");
-            $source_name_sel = $pdo->prepare("SELECT COUNT(*) FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name");
-            // The old runner never checked certificates, legacy sources keep that
-            $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `verify_cert`, `auth_type`, `active`)
-              VALUES (:name, 'Migrated sync jobs', '', 'domain', :host, :port, :enc, 0, :auth, 1)");
             $source_domain_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)");
             $job_upd = $pdo->prepare("UPDATE `imapsync` SET `source_id` = :source_id, `custom_params` = :cp, `custom_params_legacy` = :cp_legacy,
               `dry` = GREATEST(`dry`, :dry) WHERE `id` = :id AND `source_id` IS NULL");
@@ -2002,21 +1741,17 @@ function init_db_schema()
               $enc = in_array($row['enc1'], array('TLS', 'SSL', 'PLAIN'), true) ? $row['enc1'] : 'TLS';
               $at = strrchr((string)$row['user2'], '@');
               $domain = ($at === false) ? '' : strtolower(substr($at, 1));
-              // One source per (host, port, enc, auth, domain), named "host (domain)" plus " 2", " 3"... when taken (<= 100 chars)
+              // One source per (host, port, enc, auth, domain), name always unique and <= 100 chars
-              $conn = array(':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth);
+              $name = 'legacy-' . mb_substr($host, 0, 60) . '-' . $port . '-' . substr(sha1($host . '|' . $port . '|' . $enc . '|' . $auth . '|' . $domain), 0, 8);
-              $source_sel->execute($conn + array(':domain' => $domain));
+              $source_sel->execute(array(':name' => $name));
               if (($source_id = $source_sel->fetchColumn()) === false) {
-                $base = mb_substr($host . ($domain !== '' ? ' (' . $domain . ')' : ''), 0, 94);
+                $source_ins->execute(array(':name' => $name, ':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth));
-                for ($i = 1, $name = $base; $i < 1000; $name = $base . ' ' . ++$i) {
+                $source_sel->execute(array(':name' => $name));
-                  $source_name_sel->execute(array(':name' => $name));
+                if (($source_id = $source_sel->fetchColumn()) === false) continue;
-                  if ($source_name_sel->fetchColumn() == 0) break;
               }
-                $source_ins->execute($conn + array(':name' => $name));
-                if (($source_id = $pdo->lastInsertId()) == 0) continue;
               if ($domain !== '') {
                 $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain));
               }
-              }
               $job_upd->execute(array(
                 ':source_id' => $source_id,
                 ':cp' => json_encode($pairs, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE),

data/web/inc/lib/mailcow/Backup/BackupModule.php

@@ -1,90 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup;
-
-use Mailcow\Backup\Controller\BackupApiController;
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Legacy\NoTemplates;
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Repository\ExportRepository;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\RestoreRepository;
-use Mailcow\Backup\Repository\SettingsRepository;
-use Mailcow\Backup\Repository\SnapshotRepository;
-use Mailcow\Backup\Service\DomainRestoreService;
-use Mailcow\Backup\Service\ExportService;
-use Mailcow\Backup\Service\GroupMembers;
-use Mailcow\Backup\Service\JobService;
-use Mailcow\Backup\Service\RestoreService;
-use Mailcow\Backup\Service\SettingsService;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Dovecot\DovecotModule;
-use Mailcow\MailboxGroup\Repository\GroupRepository;
-use Mailcow\MailboxGroup\Repository\LookupRepository;
-use Mailcow\MailboxGroup\Service\ChangeEffects;
-use Mailcow\MailboxGroup\Service\SharingService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use Mailcow\Sogo\SogoModule;
-use PDO;
-use Redis;
-
-/**
- * Composition root of the backup with restic: settings, jobs of the runners, snapshots, the export of a domain and the
- * restore of mailboxes and domains. The runner in the restic container reaches the export and the restore actions
- * through data/conf/rspamd/dynmaps/backup.php.
- *
- *   process_get_return(BackupModule::fromGlobals()->api->getSettings());
- *   echo json_encode(BackupModule::create($pdo, Context::system(), $config, $redis)->api->export($domain, $page));  // backup.php
- *
- * $config: timezone (TZ of mailcow, for the schedule), alias_disable_external_domains ($ALIAS_DISABLE_EXTERNAL_DOMAINS),
- * demo_mode ($DEMO_MODE: the API changes nothing).
- * $redis: for DKIM keys, rate limits and the domain map (export and restore).
- */
-class BackupModule
-{
-    public function __construct(
-        public readonly SettingsService $settings,
-        public readonly JobService $jobs,
-        public readonly ExportService $export,
-        public readonly RestoreService $restore,
-        public readonly DomainRestoreService $domainRestore,
-        public readonly BackupApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context, array $config = [], ?Redis $redis = null): self
-    {
-        $jobRepo = new JobRepository($pdo);
-        $restoreRepo = new RestoreRepository($pdo);
-        $legacy = new LegacyDomain($redis);
-        $settings = new SettingsService(new SettingsRepository($pdo), $context);
-        $jobs = new JobService($jobRepo, new SnapshotRepository($pdo), $settings, $context, (string)($config['timezone'] ?? 'UTC'));
-        $export = new ExportService(new ExportRepository($pdo), $legacy);
-        $sogo = SogoModule::create()->sogo;
-        $dovecot = DovecotModule::create()->dovecot;
-        // the group module without group templates: a restored mailbox keeps the settings of the backup
-        $groupRepo = new GroupRepository($pdo);
-        $lookup = new LookupRepository($pdo);
-        $templates = new TemplateService($lookup, new NoTemplates());
-        $effects = new ChangeEffects($templates, $sogo, $groupRepo, $dovecot);
-        $groups = new GroupMembers($groupRepo, $lookup, $templates, $effects, new SharingService($groupRepo, $dovecot, $sogo, $effects));
-        $restore = new RestoreService($restoreRepo, $jobRepo, $groups, $sogo, $dovecot, $legacy, (bool)($config['alias_disable_external_domains'] ?? false));
-        $domainRestore = new DomainRestoreService($restoreRepo, $jobRepo, $groups, $legacy);
-        return new self($settings, $jobs, $export, $restore, $domainRestore,
-            new BackupApiController($settings, $jobs, $export, $restore, $domainRestore, new ApiResponse(Settings::SECRETS),
-                (bool)($config['demo_mode'] ?? false)));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo, $redis, $_SESSION, TZ, $ALIAS_DISABLE_EXTERNAL_DOMAINS and $DEMO_MODE */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), [
-            'timezone' => getenv('TZ') ?: 'UTC',
-            'alias_disable_external_domains' => ($GLOBALS['ALIAS_DISABLE_EXTERNAL_DOMAINS'] ?? false) === true,
-            'demo_mode' => ($GLOBALS['DEMO_MODE'] ?? false) === true,
-        ],($GLOBALS['redis'] ?? null) instanceof Redis ? $GLOBALS['redis'] : null);
-    }
-}

data/web/inc/lib/mailcow/Backup/Controller/BackupApiController.php

@@ -1,181 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Controller;
-
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Service\DomainRestoreService;
-use Mailcow\Backup\Service\ExportService;
-use Mailcow\Backup\Service\JobService;
-use Mailcow\Backup\Service\RestoreService;
-use Mailcow\Backup\Service\SettingsService;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php (routes backup-settings, backup-status, backup-snapshots, backup-snapshot-content,
- * backup-jobs, backup-job, backup-restore) and the runner's endpoint data/conf/rspamd/dynmaps/backup.php (domains,
- * export pages, restore actions). Messages are logged as ['backup', action, object, data].
- */
-class BackupApiController
-{
-    public function __construct(private SettingsService $settings, private JobService $jobs, private ExportService $export,
-        private RestoreService $restore, private DomainRestoreService $domainRestore, private ApiResponse $response,
-        private bool $demoMode = false)
-    {
-    }
-
-    /** GET backup-settings: secrets as <key>_set; {} for non-admins */
-    public function getSettings(): array
-    {
-        return $this->settings->get()?->toArray() ?? [];
-    }
-
-    /** POST edit/backup-settings */
-    public function editSettings(array $attr): bool
-    {
-        return $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
-            $this->settings->save($attr);
-            return $this->response->success('backup_settings_saved', $log);
-        });
-    }
-
-    /** GET backup-status */
-    public function getStatus(): array
-    {
-        return $this->jobs->status();
-    }
-
-    /** GET backup-snapshots */
-    public function getSnapshots(): array
-    {
-        return $this->jobs->snapshots();
-    }
-
-    /** GET backup-snapshot-content/<snapshot>[/<domain>]: domains of a snapshot, or the mailboxes of one, with their state now */
-    public function getSnapshotContent(mixed $snapshot, mixed $domain = null): array
-    {
-        return ($domain === null || $domain === '') ? $this->jobs->content($snapshot) : $this->jobs->domainContent($snapshot, $domain);
-    }
-
-    /** GET backup-snapshot-search/<snapshot>/<term>: domains and mailboxes of a snapshot matching the term */
-    public function getSnapshotSearch(mixed $snapshot, mixed $term): array
-    {
-        return $this->jobs->search($snapshot, $term);
-    }
-
-    /** GET backup-jobs[/<count>[/<type>]]: newest first, without logs; count 0 = default */
-    public function getJobs(int $limit = 0, mixed $type = null): array
-    {
-        return array_map(fn(Job $job): array => $this->jobs->view($job), $this->jobs->list($limit, $type));
-    }
-
-    /** GET backup-job/<id>: with its log */
-    public function getJob(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $job = ($id === null) ? null : $this->jobs->get($id);
-        return ($job === null) ? [] : $this->jobs->view($job, true);
-    }
-
-    /** POST add/backup-job: {type: backup|check|prune|init} */
-    public function addJob(array $attr): bool
-    {
-        return $this->handle('add', 'job', $attr, function (array $log) use ($attr): bool {
-            $job = $this->jobs->create($attr);
-            return $this->response->success(['backup_job_added', $job->type, $job->id], $log);
-        });
-    }
-
-    /** POST add/backup-restore: {snapshot, scope, mailboxes | domain, components, mail_target} */
-    public function addRestore(array $attr): bool
-    {
-        return $this->handle('add', 'restore', $attr, function (array $log) use ($attr): bool {
-            $job = $this->jobs->restore($attr);
-            return $this->response->success(['backup_restore_added', $job->id, count($job->mailboxes())], $log);
-        });
-    }
-
-    /** POST delete/backup-job: cancels queued and running jobs, one message per id */
-    public function deleteJobs(array $items): bool
-    {
-        return $this->handle('delete', 'job', ['id' => $items], function (array $log) use ($items): bool {
-            foreach ($items as $id) {
-                try {
-                    $job = $this->jobs->cancel($this->requireId($id));
-                    $this->response->success(['backup_job_cancelled', $job->id], $log);
-                } catch (UserError $e) {
-                    $this->response->error($e, $log);
-                }
-            }
-            return true;
-        });
-    }
-
-    /** backup.php ?domains=1 */
-    public function domains(): array
-    {
-        return ['domains' => $this->export->domains()];
-    }
-
-    /** backup.php ?export=<domain>[&after=<username>] @throws UserError */
-    public function export(string $domain, ?string $after = null): array
-    {
-        return $this->export->export($domain, $after);
-    }
-
-    /**
-     * backup.php POST of the runner, {"result": ...} of
-     *   {"action": "restore_domain", "job": id, "data": <domain_data>}: creates a deleted domain
-     *   {"action": "restore_aliases", "job": id, "aliases": [...]}: a part of the domain's aliases (scope domain)
-     *   {"action": "restore", "job": id, "username": u, "data": <mailbox section>}: one mailbox
-     *
-     * @throws UserError
-     */
-    public function post(array $body): array
-    {
-        return ['result' => match ($body['action'] ?? null) {
-            'restore' => $this->restore->merge($body['job'] ?? null, $body['username'] ?? null, $body['data'] ?? null),
-            'restore_aliases' => $this->restore->mergeAliases($body['job'] ?? null, $body['aliases'] ?? null),
-            'restore_domain' => $this->domainRestore->restore($body['job'] ?? null, $body['data'] ?? null),
-            default => throw new UserError('backup_action_invalid'),
-        }];
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     * In demo mode no write call runs.
-     */
-    private function handle(string $action, string $object, mixed $data, callable $work): bool
-    {
-        $log = ['backup', $action, $object, $data];
-        try {
-            if ($this->demoMode) {
-                throw new UserError('demo_mode_enabled');
-            }
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/Backup/Legacy/LegacyDomain.php

@@ -1,84 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Legacy;
-
-use Redis;
-use RuntimeException;
-
-/**
- * What the legacy code keeps outside the database for domains and mailboxes: DKIM keys, rate limits and the domain
- * map in Redis (functions.dkim/ratelimit/mailbox.inc.php), and the SOGo restart through dockerapi. Redis and docker()
- * sit in protected methods so tests can replace them.
- */
-class LegacyDomain
-{
-    public function __construct(private ?Redis $redis = null)
-    {
-    }
-
-    /** @return ?array{selector: string, pubkey: string, privkey: string} the DKIM key of a domain */
-    public function dkim(string $domain): ?array
-    {
-        $pubkey = $this->redis()->hGet('DKIM_PUB_KEYS', $domain);
-        if (!is_string($pubkey) || $pubkey === '') {
-            return null;
-        }
-        $selector = (string)$this->redis()->hGet('DKIM_SELECTORS', $domain);
-        return ['selector' => $selector, 'pubkey' => $pubkey, 'privkey' => (string)$this->redis()->hGet('DKIM_PRIV_KEYS', $selector . '.' . $domain)];
-    }
-
-    /** Stores a DKIM key like dkim('import') does */
-    public function addDkim(string $domain, string $selector, string $pubkey, string $privkey): void
-    {
-        $this->redis()->hSet('DKIM_PUB_KEYS', $domain, $pubkey);
-        $this->redis()->hSet('DKIM_SELECTORS', $domain, $selector);
-        $this->redis()->hSet('DKIM_PRIV_KEYS', $selector . '.' . $domain, $privkey);
-    }
-
-    /**
-     * @param string[] $objects domains or mailboxes
-     * @return array<string, string> object => rate limit ("10 / 1h") of those with one
-     */
-    public function rateLimits(array $objects): array
-    {
-        if (empty($objects)) {
-            return [];
-        }
-        $values = $this->redis()->hMGet('RL_VALUE', array_values($objects));
-        return array_filter(is_array($values) ? $values : [], fn(mixed $value): bool => is_string($value) && $value !== '');
-    }
-
-    public function setRateLimit(string $object, string $value): void
-    {
-        $this->redis()->hSet('RL_VALUE', $object, $value);
-    }
-
-    /** The domain is local for rspamd (legacy domain and alias domain add) */
-    public function addDomainMap(string $domain): void
-    {
-        $this->redis()->hSet('DOMAIN_MAP', $domain, 1);
-    }
-
-    /**
-     * SOGo reads its domains at start: restart it after a domain was added (like the "restart SOGo" option of the form)
-     *
-     * @return ?string null on success, else the error of dockerapi
-     */
-    public function restartSogo(): ?string
-    {
-        $result = json_decode((string)$this->dockerRestart('sogo-mailcow'), true);
-        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
-    }
-
-    protected function redis(): Redis
-    {
-        return $this->redis ?? throw new RuntimeException('Redis is not connected');
-    }
-
-    protected function dockerRestart(string $service): mixed
-    {
-        return docker('post', $service, 'restart');
-    }
-}

data/web/inc/lib/mailcow/Backup/Legacy/NoTemplates.php

@@ -1,19 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Legacy;
-
-use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
-
-/**
- * The group module's bridge to the mailbox code for restores: a membership restored from a backup never applies the
- * group's mailbox template (the mailbox gets its settings from the backup).
- */
-class NoTemplates extends LegacyMailbox
-{
-    public function applyTemplate(string $username, string $name, string $template): ?string
-    {
-        return null;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Job.php

@@ -1,129 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * A job of the backup runners (table `backup_job`): backup, check, prune and init run in the restic container; a
- * restore runs there (step `files`, then the merge through backup.php) and, for mails, IMAP ACLs and Sieve
- * scripts, in the dovecot container (step `mails`). Times are unix timestamps.
- */
-class Job
-{
-    public const TYPES = ['backup', 'check', 'prune', 'init', 'restore'];
-    /** Types an admin starts with add/backup-job */
-    public const MANUAL_TYPES = ['backup', 'check', 'prune', 'init'];
-    public const ACTIVE_STATES = ['queued', 'running'];
-
-    public ?int $id = null;
-    public string $type = 'backup';
-    public string $state = 'queued';
-    public ?string $step = null;
-    /**
-     * restore: snapshot, snapshot_time, scope, domain, mailboxes, components, mail_target, requested_role, create_domain (0|1),
-     * create_mailboxes (missing when requested), acl and allowed_domains (domain admins)
-     */
-    public array $params = [];
-    public ?array $result = null;
-    /** Only loaded for a single job */
-    public ?string $log = null;
-    public ?string $error = null;
-    /** restore: the domain of its mailboxes, so domain admins see their restores */
-    public string $domain = '';
-    public string $requestedBy = '';
-    public ?int $created = null;
-    public ?int $started = null;
-    public ?int $finished = null;
-
-    public function __construct(string $type = 'backup')
-    {
-        $this->type = $type;
-    }
-
-    /** Row of `backup_job` with the times as unix timestamps */
-    public static function fromRow(array $row): self
-    {
-        $job = new self((string)$row['type']);
-        $job->id = (int)$row['id'];
-        $job->state = (string)$row['state'];
-        $job->step = isset($row['step']) ? (string)$row['step'] : null;
-        $params = is_string($row['params'] ?? null) ? json_decode($row['params'], true) : null;
-        $job->params = is_array($params) ? $params : [];
-        $result = is_string($row['result'] ?? null) ? json_decode($row['result'], true) : null;
-        $job->result = is_array($result) ? $result : null;
-        $job->log = isset($row['log']) ? (string)$row['log'] : null;
-        $job->error = isset($row['error']) ? (string)$row['error'] : null;
-        $job->domain = (string)($row['domain'] ?? '');
-        $job->requestedBy = (string)($row['requested_by'] ?? '');
-        foreach (['created', 'started', 'finished'] as $time) {
-            $job->$time = isset($row[$time]) ? (int)$row[$time] : null;
-        }
-        return $job;
-    }
-
-    /** Column values for a new row */
-    public function toRow(): array
-    {
-        return [
-            'type' => $this->type,
-            'state' => $this->state,
-            'step' => $this->step,
-            'params' => json_encode($this->params ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
-            'domain' => $this->domain,
-            'requested_by' => $this->requestedBy,
-        ];
-    }
-
-    /** @return string[] mailboxes of a restore */
-    public function mailboxes(): array
-    {
-        return array_values(array_filter((array)($this->params['mailboxes'] ?? []), 'is_string'));
-    }
-
-    /** @return string[] components of a restore */
-    public function components(): array
-    {
-        return array_values(array_filter((array)($this->params['components'] ?? []), 'is_string'));
-    }
-
-    /** @return string[] mailboxes of a restore that did not exist when it was requested */
-    public function createMailboxes(): array
-    {
-        return array_values(array_filter((array)($this->params['create_mailboxes'] ?? []), 'is_string'));
-    }
-
-    /** The restore creates its domain (requested by an admin while the domain did not exist) */
-    public function createsDomain(): bool
-    {
-        return (int)($this->params['create_domain'] ?? 0) === 1;
-    }
-
-    public function isActive(): bool
-    {
-        return in_array($this->state, self::ACTIVE_STATES, true);
-    }
-
-    /** Entry of the API; the log only with $withLog (get/backup-job/<id>) */
-    public function toArray(bool $withLog = false): array
-    {
-        $job = [
-            'id' => $this->id,
-            'type' => $this->type,
-            'state' => $this->state,
-            'step' => $this->step,
-            'params' => $this->params ?: new \stdClass(),
-            'result' => $this->result,
-            'error' => $this->error,
-            'domain' => $this->domain,
-            'requested_by' => $this->requestedBy,
-            'created' => $this->created,
-            'started' => $this->started,
-            'finished' => $this->finished,
-        ];
-        if ($withLog) {
-            $job['log'] = (string)$this->log;
-        }
-        return $job;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/RestoreRun.php

@@ -1,116 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-use Mailcow\Core\Context;
-
-/**
- * The merge of one mailbox of a restore job: what the backup holds, who asked for the restore and what the merge
- * did per component (added, skipped, errors), plus the caches to refresh afterwards.
- */
-class RestoreRun
-{
-    /** @var array<string, array{added: int, skipped: int, errors: string[]}> */
-    public array $results = [];
-    /** Mailboxes whose SOGo identities (aliases, send-as addresses) changed */
-    public array $sogoUsers = [];
-    /** Mailboxes whose userdb or passdb values changed (Dovecot auth cache) */
-    public array $authUsers = [];
-
-    /**
-     * @param array $mailbox  the mailbox row as it is now
-     * @param array $params   params of the restore job (requested_role, acl of a domain admin, scope, ...)
-     * @param array $data     the mailbox's section of the domain export in the snapshot
-     * @param string[] $domains domains whose aliases and send-as addresses the restore may change, [] = all (admins)
-     * @param string $requestedBy who requested the restore
-     */
-    /** The restore created the mailbox (its own alias gets the forwarding of the backup) */
-    public bool $created = false;
-
-    public function __construct(
-        public readonly string $username,
-        public readonly string $domain,
-        public readonly array $mailbox,
-        public readonly array $params,
-        public readonly array $data,
-        public readonly array $domains = [],
-        public readonly string $requestedBy = ''
-    ) {
-    }
-
-    /**
-     * The caller the group module checks: an admin, or the domain admin who requested the restore with his ACL of then
-     * and his domains now
-     */
-    public function requester(): Context
-    {
-        if ($this->byAdmin()) {
-            return new Context('admin', 'backup', [], true);
-        }
-        return new Context('domainadmin', $this->requestedBy, array_map(fn(mixed $flag): string => (string)(int)$flag, (array)($this->params['acl'] ?? [])));
-    }
-
-    public function start(string $component): void
-    {
-        $this->results[$component] = ['added' => 0, 'skipped' => 0, 'errors' => []];
-    }
-
-    public function added(string $component, int $count = 1): void
-    {
-        $this->results[$component]['added'] += $count;
-    }
-
-    public function skipped(string $component, int $count = 1): void
-    {
-        $this->results[$component]['skipped'] += $count;
-    }
-
-    public function error(string $component, string $error): void
-    {
-        $this->results[$component]['errors'][] = $error;
-    }
-
-    /** The part of a component was rolled back: nothing of it counts */
-    public function failed(string $component, string $error): void
-    {
-        $this->start($component);
-        $this->error($component, $error);
-    }
-
-    /** The restore was requested by an admin (domain admins are limited by their ACL and the job's domain) */
-    public function byAdmin(): bool
-    {
-        return ($this->params['requested_role'] ?? '') === 'admin';
-    }
-
-    /** Admins, or a domain admin with this ACL when he requested the restore */
-    public function may(string $acl): bool
-    {
-        return $this->byAdmin() || (int)($this->params['acl'][$acl] ?? 0) === 1;
-    }
-
-    public function mayChangeDomain(string $domain): bool
-    {
-        return $this->domains === [] || in_array(strtolower($domain), $this->domains, true);
-    }
-
-    /** Rows of a list in the backup (sieve_filters, tfa, ...) */
-    public function rows(string $key): array
-    {
-        return array_values(array_filter((array)($this->data[$key] ?? []), 'is_array'));
-    }
-
-    public function sogo(string ...$usernames): void
-    {
-        array_push($this->sogoUsers, ...$usernames);
-        $this->sogoUsers = array_values(array_unique($this->sogoUsers));
-    }
-
-    public function auth(string ...$usernames): void
-    {
-        array_push($this->authUsers, ...$usernames);
-        $this->authUsers = array_values(array_unique($this->authUsers));
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Settings.php

@@ -1,189 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * Backup settings (table `backup_setting`, name/value pairs) over their defaults. Secrets are stored sealed for the
- * runner (`sealed:v1:<base64>`) and never leave toArray(): the API gets `<key>_set` instead. The state_* values, the
- * SSH public key and the SFTP host key are written by the runner in the restic container and cannot be set through
- * the API (PHP writes state_target_changed when the target changes).
- */
-class Settings
-{
-    public const DEFAULTS = [
-        'enabled' => '0',
-        'target' => 's3',
-        's3_endpoint' => 'https://s3.amazonaws.com',
-        's3_bucket' => '',
-        's3_path' => '',
-        's3_region' => '',
-        's3_access_key' => '',
-        's3_secret_key' => '',
-        's3_path_style' => '0',
-        'sftp_host' => '',
-        'sftp_port' => '22',
-        'sftp_user' => '',
-        'sftp_path' => '',
-        'sftp_public_key' => '',
-        'sftp_host_key' => '',
-        'rest_url' => '',
-        'rest_user' => '',
-        'rest_password' => '',
-        'local_path' => 'mailcow',
-        'password' => '',
-        'password_confirmed' => '0',
-        // JSON list of the scheduled backups: {"days": [weekdays, 0 = Sunday], "time": "HH:MM"}
-        'schedule' => '[{"days":[0,1,2,3,4,5,6],"time":"01:00"}]',
-        'keep_hourly' => '0',
-        'keep_daily' => '7',
-        'keep_weekly' => '4',
-        'keep_monthly' => '12',
-        'keep_yearly' => '0',
-        'prune_weekday' => '0',
-        'max_unused' => '10%',
-        'check_subset' => '30',
-        'include_index' => '1',
-        'include_fts' => '1',
-        'limit_upload' => '0',
-        'notify_email' => '',
-        'state_initialized' => '0',
-        'state_runner_seen' => '',
-        'state_last_backup' => '',
-        'state_last_success' => '',
-        'state_repo_size' => '',
-        'state_repo_stats' => '',
-        'state_schedule_since' => '',
-        'state_schedule' => '',
-        // public key (base64, X25519) the secrets are sealed with; the private key stays in the restic container
-        'state_secrets_pubkey' => '',
-        'state_secrets_error' => '',
-        // JSON {old_target, old_addresses, changed, changed_by, time}: the runner mails the old addresses and clears it
-        'state_target_changed' => '',
-        // RESTIC_BACKUP_MODE of mailcow.conf as the runner sees it
-        'state_backup_mode' => '',
-    ];
-    public const SECRETS = ['password', 's3_secret_key', 'rest_password'];
-    /** Written by the runner only */
-    public const RUNNER = ['sftp_public_key', 'sftp_host_key', 'state_initialized', 'state_runner_seen', 'state_last_backup',
-        'state_last_success', 'state_repo_size', 'state_repo_stats', 'state_schedule_since', 'state_schedule', 'state_secrets_pubkey',
-        'state_secrets_error', 'state_target_changed', 'state_backup_mode'];
-    /** Prefix of a sealed secret (sodium_crypto_box_seal with state_secrets_pubkey) */
-    public const SEALED = 'sealed:v1:';
-    public const TARGETS = ['s3', 'sftp', 'rest', 'local'];
-    /** The connection of each target: a change (or another target or repository password) resets state_initialized */
-    public const CONNECTION = [
-        's3' => ['s3_endpoint', 's3_bucket', 's3_path', 's3_region', 's3_access_key', 's3_secret_key', 's3_path_style'],
-        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
-        'rest' => ['rest_url', 'rest_user', 'rest_password'],
-        'local' => ['local_path'],
-    ];
-    /** Values a target cannot work without (besides the repository password) */
-    public const REQUIRED = [
-        's3' => ['s3_endpoint', 's3_bucket', 's3_access_key', 's3_secret_key'],
-        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
-        'rest' => ['rest_url'],
-        'local' => ['local_path'],
-    ];
-    /** Returned as numbers by the API ('' = not known yet: null; prune_weekday '' = never stays '') */
-    private const NUMBERS = ['enabled', 's3_path_style', 'sftp_port', 'password_confirmed', 'keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly',
-        'keep_yearly', 'prune_weekday', 'check_subset', 'include_index', 'include_fts', 'limit_upload', 'state_initialized',
-        'state_runner_seen', 'state_last_backup', 'state_last_success', 'state_repo_size', 'state_schedule_since'];
-    private const JSON = ['schedule', 'state_repo_stats', 'state_target_changed'];
-
-    /** @param array<string, string> $values every key of DEFAULTS */
-    public function __construct(public array $values = self::DEFAULTS)
-    {
-    }
-
-    /** Stored values (name => value) over the defaults; unknown names are dropped */
-    public static function fromStored(array $stored): self
-    {
-        $values = self::DEFAULTS;
-        foreach ($stored as $name => $value) {
-            if (array_key_exists($name, $values)) {
-                $values[$name] = (string)$value;
-            }
-        }
-        return new self($values);
-    }
-
-    public function get(string $name): string
-    {
-        return $this->values[$name] ?? '';
-    }
-
-    /** The target has its required values and the repository password is set */
-    public function configured(): bool
-    {
-        $required = self::REQUIRED[$this->get('target')] ?? null;
-        if ($required === null || $this->get('password') === '') {
-            return false;
-        }
-        foreach ($required as $name) {
-            if ($this->get($name) === '') {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /**
-     * The schedule as [weekday (0 = Sunday), hour, minute] slots, sorted and unique
-     *
-     * @return array<int, array{int, int, int}>
-     */
-    public function scheduleSlots(): array
-    {
-        $entries = json_decode($this->get('schedule'), true);
-        $slots = [];
-        foreach (is_array($entries) ? $entries : [] as $entry) {
-            if (!is_array($entry) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', (string)($entry['time'] ?? ''), $time) || !is_array($entry['days'] ?? null)) {
-                continue;
-            }
-            foreach ($entry['days'] as $day) {
-                if (is_int($day) && $day >= 0 && $day <= 6) {
-                    $slots[] = $day * 1440 + (int)$time[1] * 60 + (int)$time[2];
-                }
-            }
-        }
-        $slots = array_unique($slots);
-        sort($slots);
-        return array_map(fn(int $slot): array => [intdiv($slot, 1440), intdiv($slot % 1440, 60), $slot % 60], $slots);
-    }
-
-    /** The longest time between two scheduled backups of a week in seconds, null without a schedule */
-    public function scheduleGap(): ?int
-    {
-        $slots = array_map(fn(array $slot): int => $slot[0] * 86400 + $slot[1] * 3600 + $slot[2] * 60, $this->scheduleSlots());
-        if (empty($slots)) {
-            return null;
-        }
-        $gaps = [$slots[0] + 7 * 86400 - end($slots)];
-        for ($i = 1; $i < count($slots); $i++) {
-            $gaps[] = $slots[$i] - $slots[$i - 1];
-        }
-        return max($gaps);
-    }
-
-    /** Entry of get/backup-settings: every key, secrets as <key>_set, plus `configured` */
-    public function toArray(): array
-    {
-        $settings = [];
-        foreach ($this->values as $name => $value) {
-            if (in_array($name, self::SECRETS, true)) {
-                $settings[$name . '_set'] = (int)($value !== '');
-            } elseif (in_array($name, self::JSON, true)) {
-                $decoded = json_decode($value, true);
-                $settings[$name] = is_array($decoded) ? $decoded : null;
-            } elseif (in_array($name, self::NUMBERS, true)) {
-                $settings[$name] = ($value === '') ? (($name === 'prune_weekday') ? '' : null) : (int)$value;
-            } else {
-                $settings[$name] = $value;
-            }
-        }
-        $settings['configured'] = (int)$this->configured();
-        return $settings;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Snapshot.php

@@ -1,142 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * A snapshot of the repository (table `backup_snapshot`, the runner's cache of `restic snapshots --json`).
- */
-class Snapshot
-{
-    public string $id = '';
-    public string $shortId = '';
-    /** unix timestamp */
-    public int $time = 0;
-    public string $hostname = '';
-    /** @var string[] */
-    public array $tags = [];
-    /** restic's summary of the backup (data_added, files_new, ...) */
-    public ?array $summary = null;
-    /** @var ?string[] domains of its manifest, null for snapshots not made by mailcow */
-    public ?array $domains = null;
-    /** @var array<string, ?int> domain => its creation time (unix) when the snapshot was made, from the snapshot row */
-    public array $created = [];
-    /**
-     * @var ?array<string, array{created: ?int, mailboxes: array<string, string>}> per domain its creation time (unix, null
-     *      when not recorded) and username => name of its mailboxes; null = not known (snapshots this installation did
-     *      not make itself, e.g. of another server in the same repository)
-     */
-    public ?array $content = null;
-
-    /** Row of `backup_snapshot` with the time as unix timestamp */
-    public static function fromRow(array $row): self
-    {
-        $snapshot = new self();
-        $snapshot->id = (string)$row['id'];
-        $snapshot->shortId = (string)$row['short_id'];
-        $snapshot->time = (int)$row['time'];
-        $snapshot->hostname = (string)($row['hostname'] ?? '');
-        $tags = (string)($row['tags'] ?? '');
-        $decoded = json_decode($tags, true);
-        $tags = is_array($decoded) ? $decoded : explode(',', $tags);
-        $snapshot->tags = array_values(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen'));
-        $summary = is_string($row['summary'] ?? null) ? json_decode($row['summary'], true) : null;
-        $snapshot->summary = is_array($summary) ? $summary : null;
-        // {domain: creation time}: lists of snapshots need no content
-        $domains = is_string($row['domains'] ?? null) ? json_decode($row['domains'], true) : null;
-        if (is_array($domains)) {
-            $snapshot->domains = [];
-            foreach ($domains as $domain => $created) {
-                $snapshot->domains[] = (string)$domain;
-                $snapshot->created[strtolower((string)$domain)] = is_numeric($created) ? (int)$created : null;
-            }
-        }
-        if (is_string($row['content'] ?? null) && $row['content'] !== '') {
-            $snapshot->content = self::decodeContent($row['content']);
-        }
-        return $snapshot;
-    }
-
-    /**
-     * `content` (gzip JSON) written by the runner: {"domains": {"<domain>": {"created": <unix>, "mailboxes": {"<username>": "<name>"}}}}.
-     * Keys lower case; null when broken.
-     */
-    public static function decodeContent(string $gzip): ?array
-    {
-        $json = @gzdecode($gzip);
-        $data = is_string($json) ? json_decode($json, true) : null;
-        if (!is_array($data) || !is_array($data['domains'] ?? null)) {
-            return null;
-        }
-        $content = [];
-        foreach ($data['domains'] as $domain => $entry) {
-            $mailboxes = is_array($entry) ? ($entry['mailboxes'] ?? []) : [];
-            $created = is_numeric($entry['created'] ?? null) ? (int)$entry['created'] : null;
-            $names = [];
-            foreach (is_array($mailboxes) ? $mailboxes : [] as $username => $name) {
-                $names[strtolower((string)$username)] = is_scalar($name) ? (string)$name : '';
-            }
-            $content[strtolower((string)$domain)] = ['created' => $created, 'mailboxes' => $names];
-        }
-        return $content;
-    }
-
-    /** Domains of the snapshot: those of its content and of its manifest (an export may have failed) */
-    public function domainList(): array
-    {
-        return array_values(array_unique(array_merge(array_map('strval', array_keys($this->content ?? [])), array_map('strtolower', $this->domains ?? []))));
-    }
-
-    /** @return ?array<string, string> username => name of the domain's mailboxes, null when the content does not list the domain */
-    public function mailboxesOf(string $domain): ?array
-    {
-        return $this->content[strtolower($domain)]['mailboxes'] ?? null;
-    }
-
-    /** Creation time of the domain when the snapshot was made, null when not recorded */
-    public function createdOf(string $domain): ?int
-    {
-        return $this->content[strtolower($domain)]['created'] ?? $this->created[strtolower($domain)] ?? null;
-    }
-
-    public function hasDomain(string $domain): bool
-    {
-        return in_array(strtolower($domain), $this->domainList(), true);
-    }
-
-    /**
-     * RESTIC_BACKUP_MODE of the backup (tag mode:<mode>, none for all): all, data (without the mailcow directory) or
-     * config (without mails)
-     */
-    public function mode(): string
-    {
-        foreach ($this->tags as $tag) {
-            if (str_starts_with($tag, 'mode:')) {
-                return substr($tag, 5);
-            }
-        }
-        return 'all';
-    }
-
-    /** Snapshots not made by mailcow know neither domains nor mailboxes */
-    public function knowsDomains(): bool
-    {
-        return $this->content !== null || $this->domains !== null;
-    }
-
-    /** Entry of the API for admins */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'short_id' => $this->shortId,
-            'time' => $this->time,
-            'hostname' => $this->hostname,
-            'tags' => $this->tags,
-            'summary' => $this->summary,
-            'domains' => $this->domains,
-            'mode' => $this->mode(),
-        ];
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/ExportRepository.php

@@ -1,169 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * Reads what the export of a domain holds: the domain with its settings, its aliases and a page of its mailboxes with
- * every row that belongs to a mailbox. Rows are read with all their columns (SELECT *).
- */
-class ExportRepository extends Repository
-{
-    /** @return string[] all domains */
-    public function domains(): array
-    {
-        return $this->query("SELECT `domain` FROM `domain` ORDER BY `domain`")->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    public function dbVersion(): string
-    {
-        return (string)$this->query("SELECT `version` FROM `versions` WHERE `application` = 'db_schema'")->fetchColumn();
-    }
-
-    /** @return array[] rows of a table matching column => value (table and column names from code) */
-    public function rows(string $table, string $column, string $value): array
-    {
-        self::identifier($table);
-        self::identifier($column);
-        return $this->query("SELECT * FROM `$table` WHERE `$column` = :value", ['value' => $value])->fetchAll();
-    }
-
-    /** @return string[] usernames of the domain's mailboxes */
-    public function usernames(string $domain): array
-    {
-        return $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain", ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /** @return array[] the domain's mailbox rows after $after in username order (keyset paging), at most $limit */
-    public function mailboxesAfter(string $domain, string $after, int $limit): array
-    {
-        return $this->query("SELECT * FROM `mailbox` WHERE `domain` = :domain AND `username` > :after ORDER BY `username` LIMIT " . max(1, $limit),
-            ['domain' => $domain, 'after' => $after])->fetchAll();
-    }
-
-    /** @return array[] alias rows of the domain and its alias domains */
-    public function aliases(string $domain): array
-    {
-        return $this->query("SELECT * FROM `alias` WHERE `domain` = :domain
-            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target) ORDER BY `id`",
-            ['domain' => $domain, 'target' => $domain])->fetchAll();
-    }
-
-    /** @return array[] alias rows of any domain whose goto may name a mailbox of the domain (the caller checks) */
-    public function aliasesPointingTo(string $domain): array
-    {
-        return $this->query("SELECT * FROM `alias` WHERE `goto` LIKE :pattern ORDER BY `id`",
-            ['pattern' => '%@' . addcslashes($domain, '%_\\') . '%'])->fetchAll();
-    }
-
-    /**
-     * @param string $table  table with rows of mailboxes
-     * @param string $column its column holding the username
-     * @param string[] $usernames
-     * @return array[] rows of these mailboxes
-     */
-    public function rowsOfMailboxes(string $table, string $column, array $usernames): array
-    {
-        self::identifier($table);
-        self::identifier($column);
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT * FROM `$table` WHERE `$column` IN ($in)", $params)->fetchAll();
-    }
-
-    /** @return array[] username, id, name, domain, source of the group memberships of these mailboxes */
-    public function groupMemberships(array $usernames): array
-    {
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT `gm`.`username`, `g`.`id`, `g`.`name`, `g`.`domain`, `gm`.`source` FROM `mailbox_group_member` `gm`
-            JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id` WHERE `gm`.`username` IN ($in) ORDER BY `g`.`id`", $params)->fetchAll();
-    }
-
-    /**
-     * @param string $table  mail_policy or retention_rule
-     * @param string $member its member table
-     * @param string $key    the member table's column with the id
-     * @return array[] username, id, name, domain of the direct assignments of these mailboxes
-     */
-    public function assignments(string $table, string $member, string $key, array $usernames): array
-    {
-        self::identifier($table);
-        self::identifier($member);
-        self::identifier($key);
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT `a`.`username`, `p`.`id`, `p`.`name`, `p`.`domain` FROM `$member` `a`
-            JOIN `$table` `p` ON `p`.`id` = `a`.`$key` WHERE `a`.`username` IN ($in) ORDER BY `p`.`id`", $params)->fetchAll();
-    }
-
-    /** @return array[] sync jobs of these mailboxes (without their log) with host1, port1 and name of the source */
-    public function syncJobs(array $usernames): array
-    {
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        $rows = $this->query("SELECT `i`.*, `s`.`host1` AS `source_host1`, `s`.`port1` AS `source_port1`, `s`.`name` AS `source_name`
-            FROM `imapsync` `i` LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` IN ($in) ORDER BY `i`.`id`", $params)->fetchAll();
-        foreach ($rows as &$row) {
-            unset($row['returned_text']);
-        }
-        return $rows;
-    }
-
-    /** @return array[] BCC maps of the whole domain (local_dest "@domain", older rows "domain") */
-    public function domainBccMaps(string $domain): array
-    {
-        return $this->query("SELECT * FROM `bcc_maps` WHERE `local_dest` IN (:domain, :at)", ['domain' => $domain, 'at' => '@' . $domain])->fetchAll();
-    }
-
-    /**
-     * @param string $table  mail_policy or retention_rule
-     * @param string $member its member table
-     * @param string $key    the member table's column with the id
-     * @return array[] rows of the domain, each with `group_members`: the names of the groups it is assigned to
-     */
-    public function domainRules(string $table, string $member, string $key, string $domain): array
-    {
-        self::identifier($table);
-        self::identifier($member);
-        self::identifier($key);
-        $rows = $this->query("SELECT * FROM `$table` WHERE `domain` = :domain ORDER BY `id`", ['domain' => $domain])->fetchAll();
-        foreach ($rows as &$row) {
-            $row['group_members'] = $this->query("SELECT `g`.`name` FROM `$member` `a` JOIN `mailbox_group` `g` ON `g`.`id` = `a`.`group_id`
-                WHERE `a`.`$key` = :id ORDER BY `g`.`name`", ['id' => $row['id']])->fetchAll(PDO::FETCH_COLUMN);
-        }
-        return $rows;
-    }
-
-    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
-    private static function in(array $values): array
-    {
-        $names = [];
-        $params = [];
-        foreach (array_values($values) as $i => $value) {
-            $names[] = ':in' . $i;
-            $params['in' . $i] = $value;
-        }
-        return [implode(', ', $names), $params];
-    }
-
-    /** Table and column names come from code only; this keeps it that way */
-    private static function identifier(string $name): void
-    {
-        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
-            throw new \InvalidArgumentException('invalid identifier');
-        }
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/JobRepository.php

@@ -1,176 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Backup\Model\Job;
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\Core\UserError;
-use PDO;
-
-/**
- * Table `backup_job` (the runners claim, log and finish the jobs) and the lookups a new restore job needs.
- */
-class JobRepository extends Repository
-{
-    private const COLUMNS = "`id`, `type`, `state`, `step`, `params`, `result`, `error`, `domain`, `requested_by`,
-        UNIX_TIMESTAMP(`created`) AS `created`, UNIX_TIMESTAMP(`started`) AS `started`, UNIX_TIMESTAMP(`finished`) AS `finished`";
-    /** Serializes the check for active jobs and the insert of a new one */
-    private const LOCK = 'mailcow_backup_job';
-
-    public function find(int $id, bool $withLog = false): ?Job
-    {
-        $row = $this->query("SELECT " . self::COLUMNS . ($withLog ? ", `log`" : "") . " FROM `backup_job` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? Job::fromRow($row) : null;
-    }
-
-    /**
-     * @param ?string $type only jobs of this type
-     * @return Job[] newest first: all jobs for admins, the restores of their domains (since the domain was created) for
-     *         domain admins
-     */
-    public function visible(Context $context, int $limit, ?string $type = null): array
-    {
-        if ($context->hasAdminRights()) {
-            [$where, $params] = ['1=1', []];
-        } elseif ($context->isDomainAdmin()) {
-            // restores of their domains, requested since the domain was created (not those of a former domain of that name)
-            [$where, $params] = ["`type` = 'restore' AND `domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :da)
-              AND `created` >= (SELECT `d`.`created` FROM `domain` `d` WHERE `d`.`domain` = `backup_job`.`domain`)", ['da' => $context->username]];
-        } else {
-            return [];
-        }
-        if ($type !== null) {
-            $where .= " AND `type` = :type";
-            $params['type'] = $type;
-        }
-        return array_map(fn(array $row): Job => Job::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE $where ORDER BY `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll());
-    }
-
-    /** @return Job[] queued and running jobs */
-    public function active(): array
-    {
-        return array_map(fn(array $row): Job => Job::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `state` IN ('queued', 'running') ORDER BY `id`")->fetchAll());
-    }
-
-    /** @return array<string, Job> type => its newest job */
-    public function latestOfEachType(): array
-    {
-        $jobs = [];
-        foreach ($this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `id` IN (SELECT MAX(`id`) FROM `backup_job` GROUP BY `type`)")->fetchAll() as $row) {
-            $jobs[(string)$row['type']] = Job::fromRow($row);
-        }
-        return $jobs;
-    }
-
-    /** A backup job was created by the schedule since this unix time */
-    public function scheduledSince(int $since): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `backup_job` WHERE `type` = 'backup' AND `requested_by` = 'schedule' AND `created` >= FROM_UNIXTIME(:since) LIMIT 1",
-            ['since' => $since])->fetchColumn();
-    }
-
-    /**
-     * Inserts the job after $check(active jobs) passed; $check throws a UserError on a conflict. A named lock keeps two
-     * requests from queueing the same work at once.
-     *
-     * @param callable(Job[]): void $check
-     * @throws UserError
-     */
-    public function insertExclusive(Job $job, callable $check): int
-    {
-        if ((int)$this->query("SELECT GET_LOCK('" . self::LOCK . "', 10)")->fetchColumn() !== 1) {
-            throw new UserError('backup_job_active', [$job->type]);
-        }
-        try {
-            $check($this->active());
-            $job->id = $this->insertRow('backup_job', $job->toRow());
-            return $job->id;
-        } finally {
-            $this->query("SELECT RELEASE_LOCK('" . self::LOCK . "')");
-        }
-    }
-
-    /**
-     * Cancels a queued or running job; the restic runner stops at its next check. A restore in the dovecot step
-     * (mails, mails_running) cannot be cancelled. False when the job finished or reached that step meanwhile.
-     */
-    public function cancel(int $id): bool
-    {
-        return $this->query("UPDATE `backup_job` SET `state` = 'cancelled', `finished` = NOW() WHERE `id` = :id
-            AND `state` IN ('queued', 'running') AND NOT (`type` = 'restore' AND COALESCE(`step`, '') IN ('mails', 'mails_running'))",
-            ['id' => $id])->rowCount() > 0;
-    }
-
-    public function isDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** Domain of a mailbox, null if there is none */
-    public function mailboxDomain(string $username): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username", ['username' => $username])->fetchColumn();
-        return is_string($domain) ? strtolower($domain) : null;
-    }
-
-    /** Unix time the domain was created, null if it does not exist */
-    public function domainCreated(string $domain): ?int
-    {
-        $created = $this->query("SELECT UNIX_TIMESTAMP(`created`) FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-        return ($created === false || $created === null) ? null : (int)$created;
-    }
-
-    /**
-     * @param string[] $domains
-     * @return array{0: array<string, true>, 1: array<string, array<string, true>>} the domains that exist, and per domain
-     *         the usernames of its mailboxes; one query each per 500 domains
-     */
-    public function domainsAndUsernames(array $domains): array
-    {
-        $existing = [];
-        $usernames = [];
-        foreach (array_chunk(array_values(array_unique($domains)), 500) as $chunk) {
-            $params = [];
-            foreach ($chunk as $i => $domain) {
-                $params['d' . $i] = $domain;
-            }
-            $in = implode(', ', array_map(fn(string $name): string => ':' . $name, array_keys($params)));
-            foreach ($this->query("SELECT `domain` FROM `domain` WHERE `domain` IN ($in)", $params)->fetchAll(PDO::FETCH_COLUMN) as $domain) {
-                $existing[strtolower((string)$domain)] = true;
-            }
-            foreach ($this->query("SELECT `domain`, `username` FROM `mailbox` WHERE `domain` IN ($in)", $params)->fetchAll() as $row) {
-                $usernames[strtolower((string)$row['domain'])][strtolower((string)$row['username'])] = true;
-            }
-        }
-        return [$existing, $usernames];
-    }
-
-    /** @return array<string, string> username => name of the mailboxes of a domain */
-    public function mailboxNames(string $domain): array
-    {
-        $names = [];
-        foreach ($this->query("SELECT `username`, `name` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`", ['domain' => $domain])->fetchAll() as $row) {
-            $names[strtolower((string)$row['username'])] = (string)$row['name'];
-        }
-        return $names;
-    }
-
-    /** @return string[] all mailboxes of a domain */
-    public function mailboxesOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** @return string[] alias domains of a domain */
-    public function aliasDomainsOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/RestoreRepository.php

@@ -1,237 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-use Throwable;
-
-/**
- * Reads and writes of the restore merge. Rows of a backup may come from an older schema, so inserts and updates only
- * use the columns the table has now. Table and column names come from code, never from the backup.
- */
-class RestoreRepository extends Repository
-{
-    /** @var array<string, string[]> table => columns */
-    private array $columns = [];
-
-    /** Runs $work in one transaction */
-    public function inTransaction(callable $work): mixed
-    {
-        return $this->transaction($work);
-    }
-
-    /** Runs $work in a savepoint of the running transaction: an error undoes only its writes and is thrown on */
-    public function savepoint(string $name, callable $work): mixed
-    {
-        self::identifier($name);
-        $this->pdo->exec("SAVEPOINT `$name`");
-        try {
-            $result = $work();
-            $this->pdo->exec("RELEASE SAVEPOINT `$name`");
-            return $result;
-        } catch (Throwable $e) {
-            $this->pdo->exec("ROLLBACK TO SAVEPOINT `$name`");
-            throw $e;
-        }
-    }
-
-    /**
-     * Runs $work while holding a named lock, so a retried request of the runner waits for the first one instead of
-     * merging the same mailbox twice; null when the lock was not free within $timeout seconds
-     */
-    public function locked(string $key, callable $work, int $timeout = 120): mixed
-    {
-        $name = 'mailcow_backup_' . md5($key);
-        if ((int)$this->query("SELECT GET_LOCK(:name, " . max(0, $timeout) . ")", ['name' => $name])->fetchColumn() !== 1) {
-            return null;
-        }
-        try {
-            return $work();
-        } finally {
-            $this->query("SELECT RELEASE_LOCK(:name)", ['name' => $name]);
-        }
-    }
-
-    /** Quota rows of a new mailbox (like the legacy mailbox add) */
-    public function resetQuota(string $username): void
-    {
-        foreach (['quota2', 'quota2replica'] as $table) {
-            $this->query("INSERT INTO `$table` (`username`, `bytes`, `messages`) VALUES (:username, 0, 0)
-              ON DUPLICATE KEY UPDATE `bytes` = 0, `messages` = 0", ['username' => $username]);
-        }
-    }
-
-    /** Regular mailboxes of a domain (the domain's mailbox limit counts these) */
-    public function mailboxCount(string $domain): int
-    {
-        return (int)$this->query("SELECT COUNT(*) FROM `mailbox` WHERE `domain` = :domain AND `kind` = ''", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** External send-as rights for a domain that is local now (legacy domain and alias domain add) */
-    public function dropExternalSenderAcl(string $domain): void
-    {
-        $this->query("DELETE FROM `sender_acl` WHERE `external` = 1 AND `send_as` LIKE :domain", ['domain' => '%@' . addcslashes($domain, '%_\\')]);
-    }
-
-    /** @return string[] the columns of a table in the current schema */
-    public function columns(string $table): array
-    {
-        self::identifier($table);
-        return $this->columns[$table] ??= $this->query("SELECT `COLUMN_NAME` FROM `information_schema`.`COLUMNS`
-            WHERE `TABLE_SCHEMA` = DATABASE() AND `TABLE_NAME` = :table ORDER BY `ORDINAL_POSITION`", ['table' => $table])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /** @return array[] rows of a table matching every condition (column => value) */
-    public function rows(string $table, array $where): array
-    {
-        [$condition, $params] = $this->where($table, $where);
-        return $this->query("SELECT * FROM `$table` WHERE $condition", $params)->fetchAll();
-    }
-
-    /** Inserts a row with the values of the columns the table has now; returns the new id (0 without one) */
-    public function insert(string $table, array $values): int
-    {
-        return $this->insertRow($table, $this->existing($table, $values));
-    }
-
-    /** Updates the rows matching $where with the values of the columns the table has now */
-    public function update(string $table, array $where, array $values): int
-    {
-        $values = $this->existing($table, $values);
-        if (empty($values)) {
-            return 0;
-        }
-        [$condition, $params] = $this->where($table, $where);
-        $set = [];
-        foreach (array_keys($values) as $i => $column) {
-            $set[] = "`$column` = :v$i";
-            $params['v' . $i] = $values[$column];
-        }
-        return $this->query("UPDATE `$table` SET " . implode(', ', $set) . " WHERE $condition", $params)->rowCount();
-    }
-
-    /** A primary or alias domain */
-    public function domainExists(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias",
-            ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-    }
-
-    /** The primary domain of a domain or alias domain, null if mailcow has neither */
-    public function primaryDomain(string $domain): ?string
-    {
-        $primary = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain
-            UNION SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :alias", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-        return is_string($primary) ? strtolower($primary) : null;
-    }
-
-    /** An active primary or alias domain (like the legacy is_local_mailcow_domain()) */
-    public function isLocalDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain AND `active` = 1
-            UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias AND `active` = 1", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-    }
-
-    /** @return string[] alias domains of a domain */
-    public function aliasDomainsOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /**
-     * What an address is now: 'mailbox' (also local@alias-domain of a mailbox), 'resource' (a mailbox of kind
-     * location, thing or group), 'alias', 'spamalias' or null
-     */
-    public function addressKind(string $address): ?string
-    {
-        $kind = $this->query("SELECT `kind` FROM `mailbox` WHERE `username` = :address", ['address' => $address])->fetchColumn();
-        if (is_string($kind)) {
-            return preg_match('/location|thing|group/', $kind) ? 'resource' : 'mailbox';
-        }
-        [$local, $domain] = array_pad(explode('@', $address, 2), 2, '');
-        if ((bool)$this->query("SELECT 1 FROM `mailbox` `m` JOIN `alias_domain` `ad` ON `ad`.`alias_domain` = :domain
-            WHERE `m`.`username` = CONCAT(:local, '@', `ad`.`target_domain`)", ['domain' => $domain, 'local' => $local])->fetchColumn()) {
-            return 'mailbox';
-        }
-        if ((bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
-            return 'alias';
-        }
-        if ((bool)$this->query("SELECT 1 FROM `spamalias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
-            return 'spamalias';
-        }
-        return null;
-    }
-
-    /** Aliases the domain may still get (domain limit minus its aliases and those of its alias domains, like the legacy code) */
-    public function aliasesLeft(string $domain): int
-    {
-        $limit = $this->query("SELECT `aliases` FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-        if ($limit === false) {
-            return 0;
-        }
-        $used = (int)$this->query("SELECT COUNT(*) FROM `alias` WHERE (`domain` = :domain
-            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target))
-            AND `address` NOT IN (SELECT `username` FROM `mailbox`)", ['domain' => $domain, 'target' => $domain])->fetchColumn();
-        return (int)$limit - $used;
-    }
-
-    /** Quota (bytes) the other mailboxes of the domain have */
-    public function quotaInUse(string $domain, string $exceptUsername): int
-    {
-        return (int)$this->query("SELECT COALESCE(SUM(`quota`), 0) FROM `mailbox` WHERE `domain` = :domain AND `kind` = '' AND `username` != :username",
-            ['domain' => $domain, 'username' => $exceptUsername])->fetchColumn();
-    }
-
-    /** @return array[] user1, host1 and port1 of the mailbox's sync jobs */
-    public function syncJobsOf(string $username): array
-    {
-        return $this->query("SELECT `i`.`user1`, `s`.`host1`, `s`.`port1` FROM `imapsync` `i`
-            LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` = :username", ['username' => $username])->fetchAll();
-    }
-
-    /** @return int[] ids of the sync sources with this host and port the mailbox may use, oldest first */
-    public function syncSourcesFor(string $host, int $port, string $username, string $domain): array
-    {
-        return array_map('intval', $this->query("SELECT `s`.`id` FROM `imapsync_source` `s` WHERE `s`.`host1` = :host AND `s`.`port1` = :port
-            AND (`s`.`scope` = 'all'
-              OR (`s`.`scope` = 'domain' AND EXISTS (SELECT 1 FROM `imapsync_source_domain` `d` WHERE `d`.`source_id` = `s`.`id` AND `d`.`domain` = :domain))
-              OR (`s`.`scope` = 'user' AND EXISTS (SELECT 1 FROM `imapsync_source_user` `u` WHERE `u`.`source_id` = `s`.`id` AND `u`.`username` = :username)))
-            ORDER BY `s`.`id`", ['host' => $host, 'port' => $port, 'domain' => $domain, 'username' => $username])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** @return array{0: string, 1: array} condition and parameters of column => value pairs */
-    private function where(string $table, array $where): array
-    {
-        self::identifier($table);
-        $parts = [];
-        $params = [];
-        foreach (array_keys($where) as $i => $column) {
-            self::identifier((string)$column);
-            if ($where[$column] === null) {
-                $parts[] = "`$column` IS NULL";
-                continue;
-            }
-            $parts[] = "`$column` = :w$i";
-            $params['w' . $i] = $where[$column];
-        }
-        return [empty($parts) ? '1=1' : implode(' AND ', $parts), $params];
-    }
-
-    /** The values of the columns the table has now */
-    private function existing(string $table, array $values): array
-    {
-        return array_intersect_key($values, array_flip($this->columns($table)));
-    }
-
-    /** Table, column and savepoint names come from code only; this keeps it that way */
-    private static function identifier(string $name): void
-    {
-        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
-            throw new \InvalidArgumentException('invalid identifier');
-        }
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/SettingsRepository.php

@@ -1,31 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * Table `backup_setting` (name/value pairs; the runner writes its state_* values into it as well).
- */
-class SettingsRepository extends Repository
-{
-    /** @return array<string, string> name => value of the stored settings */
-    public function load(): array
-    {
-        return $this->query("SELECT `name`, `value` FROM `backup_setting`")->fetchAll(PDO::FETCH_KEY_PAIR);
-    }
-
-    /** Stores the given values (name => value) */
-    public function save(array $values): void
-    {
-        $this->transaction(function () use ($values): void {
-            foreach ($values as $name => $value) {
-                $this->query("INSERT INTO `backup_setting` (`name`, `value`) VALUES (:name, :value)
-                  ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)", ['name' => (string)$name, 'value' => (string)$value]);
-            }
-        });
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/SnapshotRepository.php

@@ -1,31 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Backup\Model\Snapshot;
-use Mailcow\Core\Repository;
-
-/**
- * Table `backup_snapshot`: the runner's cache of the repository's snapshots (read only here).
- */
-class SnapshotRepository extends Repository
-{
-    private const COLUMNS = "`id`, `short_id`, UNIX_TIMESTAMP(`time`) AS `time`, `hostname`, `tags`, `summary`, `domains`";
-
-    /** By id or short id, with its content (the mailboxes in it) */
-    public function find(string $id): ?Snapshot
-    {
-        $row = $this->query("SELECT " . self::COLUMNS . ", `content` FROM `backup_snapshot` WHERE `id` = :id OR `short_id` = :short_id ORDER BY `id` = :exact DESC LIMIT 1",
-            ['id' => $id, 'short_id' => $id, 'exact' => $id])->fetch();
-        return is_array($row) ? Snapshot::fromRow($row) : null;
-    }
-
-    /** @return Snapshot[] newest first; the content (the mailboxes in it) only with $withContent */
-    public function all(bool $withContent = false): array
-    {
-        return array_map(fn(array $row): Snapshot => Snapshot::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . ($withContent ? ", `content`" : "") . " FROM `backup_snapshot` ORDER BY `time` DESC, `id`")->fetchAll());
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/DomainRestoreService.php

@@ -1,320 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\RestoreRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Throwable;
-
-/**
- * Action `restore_domain`: recreates a domain deleted since the snapshot from its export (`domain_data`), before its
- * mailboxes are restored. Only for running restores an admin requested while the domain did not exist
- * (params.create_domain).
- *
- * Creates the domain like the legacy domain add (row, tags, domain map, external send-as rights for it dropped, SOGo
- * restarted so it knows the domain), unless it exists by now. Then adds what is missing: alias domains, DKIM keys, rate
- * limit, spam settings, BCC maps, footer, MTA-STS, domain admin assignments of accounts that still exist, groups (no
- * members, send-as rights or shares; members follow with the mailboxes), mail policies and retention rules with their
- * group assignments (matched by name).
- */
-class DomainRestoreService
-{
-    private const NEW_ROW = ['id', 'created', 'modified'];
-    private const POLICY_LISTS = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
-    private const RATE_LIMIT = '/^\d+ \/ 1[smhd]$/';
-
-    public function __construct(private RestoreRepository $repo, private JobRepository $jobs, private GroupMembers $groups,
-        private LegacyDomain $legacy)
-    {
-    }
-
-    /**
-     * @return array{domain: array{added: int, skipped: int, errors: string[], created: int}}
-     * @throws UserError
-     */
-    public function restore(mixed $jobId, mixed $data): array
-    {
-        $id = Input::intInRange($jobId, 1, PHP_INT_MAX);
-        $job = ($id === null) ? null : $this->jobs->find($id);
-        if ($job === null || $job->type !== 'restore' || $job->state !== 'running') {
-            throw new UserError('backup_job_not_running');
-        }
-        if (!$job->createsDomain() || ($job->params['requested_role'] ?? '') !== 'admin') {
-            throw new AccessDenied();
-        }
-        $domain = strtolower((string)$job->domain);
-        if (!is_array($data) || !is_array($data['domain'] ?? null) || strtolower((string)($data['domain']['domain'] ?? '')) !== $domain) {
-            throw new UserError('backup_export_invalid');
-        }
-        $result = $this->repo->locked($job->id . ':domain', fn(): array => $this->restoreDomain($job, $domain, $data));
-        return is_array($result) ? ['domain' => $result] : throw new UserError('backup_job_busy');
-    }
-
-    private function restoreDomain(Job $job, string $domain, array $data): array
-    {
-        $result = ['added' => 0, 'skipped' => 0, 'errors' => [], 'created' => 0];
-        if (!empty($this->repo->rows('alias_domain', ['alias_domain' => $domain]))) {
-            $result['errors'][] = "$domain is an alias domain now";
-            return $result;
-        }
-        if (!empty($this->repo->rows('domain', ['domain' => $domain]))) {
-            // created meanwhile: only what is missing is added
-            $result['skipped']++;
-        } else {
-            $this->createDomain($domain, $data, $result);
-        }
-        $this->repo->inTransaction(function () use ($domain, $data, &$result): void {
-            $this->aliasDomains($domain, $data, $result);
-            $this->domainSettings($domain, $data, $result);
-        });
-        $this->redisSettings($domain, $data, $result);
-        $groupIds = $this->createGroups($domain, $data, $result);
-        $this->repo->inTransaction(function () use ($domain, $data, $groupIds, &$result): void {
-            $this->rules($domain, $data, 'mail_policies', 'mail_policy', 'mail_policy_member', 'policy_id', $groupIds, $result);
-            $this->rules($domain, $data, 'retention_rules', 'retention_rule', 'retention_rule_member', 'rule_id', $groupIds, $result);
-        });
-        return $result;
-    }
-
-    /** The domain row, its tags and the effects of the legacy domain add */
-    private function createDomain(string $domain, array $data, array &$result): void
-    {
-        // the domain keeps its creation time: domain admins may use the snapshots made after it
-        $row = array_diff_key(RestoreService::decode($data['domain']), array_flip(['modified']));
-        $relayhost = (string)($row['relayhost'] ?? '0');
-        if ($relayhost !== '0' && $relayhost !== '' && empty($this->repo->rows('relayhosts', ['id' => (int)$relayhost]))) {
-            $row['relayhost'] = '0';
-            $result['errors'][] = 'relayhost: does not exist';
-        }
-        $this->repo->inTransaction(function () use ($domain, $row, $data): void {
-            $this->repo->insert('domain', ['domain' => $domain] + $row);
-            $this->repo->dropExternalSenderAcl($domain);
-            foreach (array_unique(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', (array)($data['tags'] ?? [])), 'strlen')) as $tag) {
-                $this->repo->insert('tags_domain', ['domain' => $domain, 'tag_name' => $tag]);
-            }
-        });
-        $result['added']++;
-        $result['created'] = 1;
-        $this->effect($result, 'domain map', fn() => $this->legacy->addDomainMap($domain));
-        $error = $this->legacy->restartSogo();
-        if ($error !== null) {
-            $result['errors'][] = 'SOGo restart failed: ' . $error;
-        }
-    }
-
-    private function aliasDomains(string $domain, array $data, array &$result): void
-    {
-        foreach (self::rows($data, 'alias_domains') as $row) {
-            $alias = strtolower(trim((string)($row['alias_domain'] ?? '')));
-            if ($alias === '' || $this->repo->domainExists($alias)) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert('alias_domain', ['alias_domain' => $alias, 'target_domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
-            $this->repo->dropExternalSenderAcl($alias);
-            $this->effect($result, "domain map of $alias", fn() => $this->legacy->addDomainMap($alias));
-            $result['added']++;
-        }
-    }
-
-    /** Spam settings, BCC maps, footer, MTA-STS and domain admins of the domain that are missing */
-    private function domainSettings(string $domain, array $data, array &$result): void
-    {
-        foreach (self::rows($data, 'filterconf') as $row) {
-            $option = (string)($row['option'] ?? '');
-            $value = (string)($row['value'] ?? '');
-            $current = $this->repo->rows('filterconf', ['object' => $domain]);
-            $present = in_array($option, ['whitelist_from', 'blacklist_from'], true)
-                ? array_filter($current, fn(array $c): bool => in_array($c['option'], ['whitelist_from', 'blacklist_from'], true) && strtolower((string)$c['value']) === strtolower($value))
-                : array_filter($current, fn(array $c): bool => $c['option'] === $option && (in_array($option, ['lowspamlevel', 'highspamlevel'], true) || (string)$c['value'] === $value));
-            if ($option === '' || !empty($present)) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert('filterconf', ['object' => $domain, 'option' => $option, 'value' => $value]);
-            $result['added']++;
-        }
-        foreach (self::rows($data, 'bcc_maps') as $row) {
-            $localDest = strtolower((string)($row['local_dest'] ?? ''));
-            if (!in_array($localDest, [$domain, '@' . $domain], true) || !filter_var((string)($row['bcc_dest'] ?? ''), FILTER_VALIDATE_EMAIL)
-                || !in_array($row['type'] ?? '', ['sender', 'rcpt'], true)) {
-                $result['errors'][] = "BCC map $localDest: invalid";
-            } elseif (!empty($this->repo->rows('bcc_maps', ['local_dest' => $localDest, 'type' => $row['type']]))) {
-                $result['skipped']++;
-            } else {
-                $this->repo->insert('bcc_maps', ['domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
-                $result['added']++;
-            }
-        }
-        foreach (['footer' => 'domain_wide_footer', 'mta_sts' => 'mta_sts'] as $key => $table) {
-            if (!is_array($data[$key] ?? null)) {
-                continue;
-            }
-            if (!empty($this->repo->rows($table, ['domain' => $domain]))) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert($table, ['domain' => $domain] + array_diff_key(RestoreService::decode($data[$key]), array_flip(['created', 'modified'])));
-            $result['added']++;
-        }
-        foreach (self::rows($data, 'domain_admins') as $row) {
-            $username = (string)($row['username'] ?? '');
-            if (empty($this->repo->rows('admin', ['username' => $username, 'superadmin' => 0]))) {
-                $result['errors'][] = "domain admin $username: does not exist";
-            } elseif (!empty($this->repo->rows('domain_admins', ['username' => $username, 'domain' => $domain]))) {
-                $result['skipped']++;
-            } else {
-                $this->repo->insert('domain_admins', ['username' => $username, 'domain' => $domain, 'active' => (int)($row['active'] ?? 1)]);
-                $result['added']++;
-            }
-        }
-    }
-
-    /** DKIM keys of the domain and its alias domains, and the rate limit of the domain (Redis), when missing */
-    private function redisSettings(string $domain, array $data, array &$result): void
-    {
-        $domains = array_merge([$domain], $this->repo->aliasDomainsOf($domain));
-        foreach (self::rows($data, 'dkim') as $key) {
-            $name = strtolower((string)($key['domain'] ?? ''));
-            $selector = (string)($key['selector'] ?? '');
-            if (!in_array($name, $domains, true)) {
-                continue;
-            }
-            if (!preg_match('/^[a-z0-9._-]{1,63}$/i', $selector) || (string)($key['pubkey'] ?? '') === '' || (string)($key['privkey'] ?? '') === '') {
-                $result['errors'][] = "DKIM key of $name: invalid";
-                continue;
-            }
-            $this->effect($result, "DKIM key of $name", function () use ($name, $selector, $key, &$result): void {
-                if ($this->legacy->dkim($name) !== null) {
-                    $result['skipped']++;
-                    return;
-                }
-                $this->legacy->addDkim($name, $selector, (string)$key['pubkey'], (string)$key['privkey']);
-                $result['added']++;
-            });
-        }
-        $rateLimit = $data['ratelimit'] ?? null;
-        if (is_string($rateLimit) && preg_match(self::RATE_LIMIT, $rateLimit)) {
-            $this->effect($result, 'rate limit', function () use ($domain, $rateLimit, &$result): void {
-                if (isset($this->legacy->rateLimits([$domain])[$domain])) {
-                    $result['skipped']++;
-                    return;
-                }
-                $this->legacy->setRateLimit($domain, $rateLimit);
-                $result['added']++;
-            });
-        }
-    }
-
-    /**
-     * Groups of the domain missing by name, through the group module (address, flags, priority, IdP link)
-     *
-     * @return array<int, int> id in the backup => id now, of all groups found by name
-     */
-    private function createGroups(string $domain, array $data, array &$result): array
-    {
-        $ids = [];
-        $admin = new Context('admin', 'backup', [], true);
-        foreach (self::rows($data, 'groups') as $row) {
-            $name = (string)($row['name'] ?? '');
-            $current = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => $name])[0] ?? null;
-            if ($current !== null) {
-                $ids[(int)($row['id'] ?? 0)] = (int)$current['id'];
-                $result['skipped']++;
-                continue;
-            }
-            $input = ['domain' => $domain, 'name' => $name, 'description' => (string)($row['description'] ?? '')];
-            foreach (['sender_allowed', 'sogo_visible', 'internal', 'active', 'priority'] as $key) {
-                if (isset($row[$key])) {
-                    $input[$key] = (int)$row[$key];
-                }
-            }
-            foreach (['address', 'idp_ref'] as $key) {
-                if (($row[$key] ?? null) !== null && $row[$key] !== '') {
-                    $input[$key] = (string)$row[$key];
-                }
-            }
-            try {
-                $ids[(int)($row['id'] ?? 0)] = $this->groups->create($admin, $input);
-                $result['added']++;
-            } catch (UserError $e) {
-                $result['errors'][] = "group $name: " . $e->key;
-            }
-        }
-        return $ids;
-    }
-
-    /**
-     * Mail policies or retention rules missing by name, with their group assignments; group:<id> entries of policies
-     * follow the groups' new ids
-     */
-    private function rules(string $domain, array $data, string $key, string $table, string $member, string $column, array $groupIds, array &$result): void
-    {
-        foreach (self::rows($data, $key) as $row) {
-            $name = (string)($row['name'] ?? '');
-            if (!empty($this->repo->rows($table, ['domain' => $domain, 'name' => $name]))) {
-                $result['skipped']++;
-                continue;
-            }
-            $values = array_diff_key($row, array_flip([...self::NEW_ROW, 'group_members']));
-            foreach (self::POLICY_LISTS as $list) {
-                if (isset($values[$list])) {
-                    $values[$list] = json_encode($this->remapGroups((string)$values[$list], $groupIds, "$name ($list)", $result));
-                }
-            }
-            $id = $this->repo->insert($table, ['domain' => $domain, 'name' => $name] + $values);
-            foreach ((array)($row['group_members'] ?? []) as $groupName) {
-                $group = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => (string)$groupName])[0] ?? null;
-                if ($group === null) {
-                    $result['errors'][] = "$name: group $groupName does not exist";
-                    continue;
-                }
-                $this->repo->insert($member, [$column => $id, 'group_id' => (int)$group['id']]);
-            }
-            $result['added']++;
-        }
-    }
-
-    /** @return string[] entries of a JSON list with group:<id> of the backup turned into the group's id now */
-    private function remapGroups(string $json, array $groupIds, string $what, array &$result): array
-    {
-        $entries = [];
-        foreach ((array)json_decode($json, true) as $entry) {
-            if (!is_string($entry)) {
-                continue;
-            }
-            if (preg_match('/^group:(\d+)$/', $entry, $match)) {
-                if (!isset($groupIds[(int)$match[1]])) {
-                    $result['errors'][] = "$what: $entry dropped (group does not exist)";
-                    continue;
-                }
-                $entry = 'group:' . $groupIds[(int)$match[1]];
-            }
-            $entries[] = $entry;
-        }
-        return $entries;
-    }
-
-    /** A Redis or dockerapi step: a failure is reported and the rest goes on */
-    private function effect(array &$result, string $what, callable $work): void
-    {
-        try {
-            $work();
-        } catch (Throwable) {
-            $result['errors'][] = "$what: not stored (Redis)";
-        }
-    }
-
-    private static function rows(array $data, string $key): array
-    {
-        return array_values(array_filter((array)($data[$key] ?? []), 'is_array'));
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/ExportService.php

@@ -1,227 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Repository\ExportRepository;
-use Mailcow\Core\UserError;
-
-/**
- * Export of a domain for the backup (/mailcow/export/<domain>.json in a snapshot), in pages of mailboxes the runner
- * merges into one file. The first page holds version, db_version, domain, domain_data (what recreating the domain
- * needs), the aliases and `mailbox_aliases` (username => the aliases of any domain whose goto names the mailbox,
- * without its own alias and group addresses; the runner moves them into the mailbox sections when it reads the file);
- * every page a part of `mailboxes` and `next_after`, the last username of the page to ask for the next one with
- * (keyset paging; null after the last page).
- *
- * Per mailbox every row a restore needs, with all columns. Password hashes, DKIM keys and the secrets of app
- * passwords, 2FA and sync jobs are part of it on purpose; only the runner gets it (backup.php with the token). Binary
- * values (fido2) are written as {"base64": "..."}. Beside the keys of the spec, a mailbox has `own_alias` (its alias
- * row: forwarding with or without a copy), `aliases` (empty, see mailbox_aliases) and `ratelimit` (Redis).
- */
-class ExportService
-{
-    public const VERSION = 1;
-    public const PAGE_SIZE = 500;
-    /** Lists of a mailbox: key => [table, column with the username] */
-    private const MAILBOX_ROWS = [
-        'sender_acl' => ['sender_acl', 'logged_in_as'],
-        'sender_acl_granted' => ['sender_acl', 'send_as'],
-        'sieve_filters' => ['sieve_filters', 'username'],
-        'filterconf' => ['filterconf', 'object'],
-        'spamalias' => ['spamalias', 'goto'],
-        'bcc_maps' => ['bcc_maps', 'local_dest'],
-        'app_passwd' => ['app_passwd', 'mailbox'],
-        'tfa' => ['tfa', 'username'],
-        // FIDO2 keys of mailbox users (passwordless login on the user page), like those of admins
-        'fido2' => ['fido2', 'username'],
-        'identity_link' => ['identity_link', 'username'],
-    ];
-    /** Single rows of a mailbox: key => [table, column with the username] */
-    private const MAILBOX_ROW = [
-        'user_acl' => ['user_acl', 'username'],
-        'pushover' => ['pushover', 'username'],
-    ];
-
-    public function __construct(private ExportRepository $repo, private LegacyDomain $legacy)
-    {
-    }
-
-    /** @return string[] all domains */
-    public function domains(): array
-    {
-        return array_values(array_map('strval', $this->repo->domains()));
-    }
-
-    /**
-     * @param ?string $after the last username of the previous page (keyset paging), null for the first page
-     * @throws UserError domain_not_found
-     */
-    public function export(string $domain, ?string $after = null, int $pageSize = self::PAGE_SIZE): array
-    {
-        $domain = strtolower(trim($domain));
-        $row = ($domain === '') ? null : ($this->repo->rows('domain', 'domain', $domain)[0] ?? null);
-        if ($row === null) {
-            throw new UserError('domain_not_found', [htmlspecialchars($domain)]);
-        }
-        $rows = $this->repo->mailboxesAfter($domain, strtolower($after ?? ''), $pageSize + 1);
-        $more = count($rows) > $pageSize;
-        $rows = array_slice($rows, 0, $pageSize);
-        $export = ['version' => self::VERSION, 'domain' => $domain];
-        if ($after === null) {
-            $export += [
-                'db_version' => $this->repo->dbVersion(),
-                'domain_data' => $this->domainData($domain, $row),
-                'aliases' => array_map(fn(array $alias): array => self::encode($alias), $this->repo->aliases($domain)),
-                'mailbox_aliases' => $this->mailboxAliases($domain) ?: new \stdClass(),
-            ];
-        }
-        $export['mailboxes'] = $this->mailboxes($domain, $rows) ?: new \stdClass();
-        $export['next_after'] = ($more && !empty($rows)) ? strtolower((string)end($rows)['username']) : null;
-        return $export;
-    }
-
-    /** What recreating the domain needs: its row and settings, alias domains, DKIM keys, groups, policies, rules */
-    private function domainData(string $domain, array $row): array
-    {
-        $aliasDomains = $this->repo->rows('alias_domain', 'target_domain', $domain);
-        $dkim = [];
-        foreach (array_merge([$domain], array_map(fn(array $alias): string => strtolower((string)$alias['alias_domain']), $aliasDomains)) as $name) {
-            $key = $this->legacy->dkim($name);
-            if ($key !== null) {
-                $dkim[] = ['domain' => $name] + $key;
-            }
-        }
-        return [
-            'domain' => self::encode($row),
-            'tags' => array_map(fn(array $tag): string => (string)$tag['tag_name'], $this->repo->rows('tags_domain', 'domain', $domain)),
-            'alias_domains' => $aliasDomains,
-            'dkim' => $dkim,
-            'ratelimit' => $this->legacy->rateLimits([$domain])[$domain] ?? null,
-            'filterconf' => $this->repo->rows('filterconf', 'object', $domain),
-            'bcc_maps' => $this->repo->domainBccMaps($domain),
-            'footer' => $this->repo->rows('domain_wide_footer', 'domain', $domain)[0] ?? null,
-            'mta_sts' => $this->repo->rows('mta_sts', 'domain', $domain)[0] ?? null,
-            'domain_admins' => $this->repo->rows('domain_admins', 'domain', $domain),
-            'groups' => $this->repo->rows('mailbox_group', 'domain', $domain),
-            'mail_policies' => $this->repo->domainRules('mail_policy', 'mail_policy_member', 'policy_id', $domain),
-            'retention_rules' => $this->repo->domainRules('retention_rule', 'retention_rule_member', 'rule_id', $domain),
-        ];
-    }
-
-    /** @return array<string, array> username => its section */
-    private function mailboxes(string $domain, array $rows): array
-    {
-        $mailboxes = [];
-        foreach ($rows as $row) {
-            $mailboxes[strtolower((string)$row['username'])] = [
-                'mailbox' => self::encode($row) + ['tags' => []],
-                'user_acl' => null,
-                'pushover' => null,
-                'ratelimit' => null,
-                'own_alias' => null,
-                'aliases' => [],
-                'groups' => [],
-                'mail_policies' => [],
-                'retention_rules' => [],
-                'imapsync' => [],
-            ] + array_fill_keys(array_keys(self::MAILBOX_ROWS), []);
-        }
-        $usernames = array_map('strval', array_keys($mailboxes));
-        if (empty($usernames)) {
-            return [];
-        }
-        // the mailbox of the page a row belongs to, null for anything else
-        $owner = fn(array $row, string $column): ?string => isset($mailboxes[strtolower((string)$row[$column])]) ? strtolower((string)$row[$column]) : null;
-        foreach ($this->repo->rowsOfMailboxes('tags_mailbox', 'username', $usernames) as $row) {
-            if (($username = $owner($row, 'username')) !== null) {
-                $mailboxes[$username]['mailbox']['tags'][] = (string)$row['tag_name'];
-            }
-        }
-        foreach (self::MAILBOX_ROW as $key => [$table, $column]) {
-            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
-                if (($username = $owner($row, $column)) !== null) {
-                    $mailboxes[$username][$key] = self::encode($row);
-                }
-            }
-        }
-        foreach (self::MAILBOX_ROWS as $key => [$table, $column]) {
-            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
-                if (($username = $owner($row, $column)) !== null) {
-                    $mailboxes[$username][$key][] = self::encode($row);
-                }
-            }
-        }
-        foreach ($this->legacy->rateLimits($usernames) as $username => $value) {
-            if (isset($mailboxes[strtolower((string)$username)])) {
-                $mailboxes[strtolower((string)$username)]['ratelimit'] = $value;
-            }
-        }
-        foreach ($this->repo->groupMemberships($usernames) as $row) {
-            if (($username = $owner($row, 'username')) !== null) {
-                $mailboxes[$username]['groups'][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain'],
-                    'source' => (string)$row['source']];
-            }
-        }
-        foreach (['mail_policies' => ['mail_policy', 'mail_policy_member', 'policy_id'], 'retention_rules' => ['retention_rule', 'retention_rule_member', 'rule_id']]
-            as $key => [$table, $member, $column]) {
-            foreach ($this->repo->assignments($table, $member, $column, $usernames) as $row) {
-                if (($username = $owner($row, 'username')) !== null) {
-                    $mailboxes[$username][$key][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain']];
-                }
-            }
-        }
-        foreach ($this->repo->syncJobs($usernames) as $row) {
-            if (($username = $owner($row, 'user2')) !== null) {
-                $mailboxes[$username]['imapsync'][] = self::encode($row);
-            }
-        }
-        foreach ($this->repo->rowsOfMailboxes('alias', 'address', $usernames) as $row) {
-            if (($username = $owner($row, 'address')) !== null) {
-                $mailboxes[$username]['own_alias'] = self::encode($row);
-            }
-        }
-        return $mailboxes;
-    }
-
-    /**
-     * @return array<string, array[]> username => aliases of any domain whose goto names the mailbox (once per domain: the
-     * alias table is scanned, not once per page)
-     */
-    private function mailboxAliases(string $domain): array
-    {
-        $usernames = array_fill_keys(array_map(fn(mixed $username): string => strtolower((string)$username), $this->repo->usernames($domain)), true);
-        $aliases = [];
-        foreach ($this->repo->aliasesPointingTo($domain) as $row) {
-            if ($row['group_id'] !== null) {
-                continue;
-            }
-            $address = strtolower((string)$row['address']);
-            foreach (array_unique(array_map('strtolower', self::gotos((string)$row['goto']))) as $goto) {
-                if ($goto !== $address && isset($usernames[$goto])) {
-                    $aliases[$goto][] = self::encode($row);
-                }
-            }
-        }
-        return $aliases;
-    }
-
-    /** @return string[] addresses of a goto list */
-    public static function gotos(string $goto): array
-    {
-        return array_values(array_filter(array_map('trim', explode(',', $goto)), 'strlen'));
-    }
-
-    /** Binary values (not UTF-8) as {"base64": "..."}, so the row survives JSON */
-    public static function encode(array $row): array
-    {
-        foreach ($row as $column => $value) {
-            if (is_string($value) && !mb_check_encoding($value, 'UTF-8')) {
-                $row[$column] = ['base64' => base64_encode($value)];
-            }
-        }
-        return $row;
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/GroupMembers.php

@@ -1,48 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\UserError;
-use Mailcow\MailboxGroup\Repository\GroupRepository;
-use Mailcow\MailboxGroup\Repository\LookupRepository;
-use Mailcow\MailboxGroup\Service\ChangeEffects;
-use Mailcow\MailboxGroup\Service\GroupService;
-use Mailcow\MailboxGroup\Service\SharingService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-
-/**
- * Group writes of a restore through the group module (addresses, send-as rights, shares and caches follow), as the
- * requester of the restore: an admin, or the domain admin with his rights now (GroupService::canEdit). The module is
- * wired without group templates (Legacy\NoTemplates).
- */
-class GroupMembers
-{
-    public function __construct(private GroupRepository $repo, private LookupRepository $lookup, private TemplateService $templates,
-        private ChangeEffects $effects, private SharingService $sharing)
-    {
-    }
-
-    /** @throws UserError e.g. access_denied when the group is read-only for the requester */
-    public function add(Context $context, int $groupId, string $username): void
-    {
-        $this->service($context)->update($groupId, ['add_members' => [$username]]);
-    }
-
-    /**
-     * Creates a group (restore of a deleted domain); members follow with the mailboxes.
-     *
-     * @throws UserError
-     */
-    public function create(Context $context, array $input): int
-    {
-        return (int)$this->service($context)->create($input)->id;
-    }
-
-    protected function service(Context $context): GroupService
-    {
-        return new GroupService($this->repo, $this->lookup, $this->templates, $this->effects, $this->sharing, $context);
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/JobService.php

@@ -1,666 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use DateTimeImmutable;
-use DateTimeZone;
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Model\Snapshot;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\SnapshotRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Jobs of the backup runners, the snapshots with their content and the status page.
- *
- * Backup, check, prune and init jobs: admins (not read-only API keys) queue them, one of each type at a time.
- * Restores: admins, and domain admins for mailboxes or a domain they administer now. Mailboxes and domains deleted
- * since the snapshot are created from it (deleted domains by admins only). A component a domain admin could not
- * manage himself (e.g. sync jobs without the `syncjobs` ACL) is refused; his ACL and domains are stored with the job,
- * so the merge and the runners apply the same limits.
- * Reading: admins see every job and snapshot; domain admins the restores of their domains and, of each snapshot made
- * after their domain was created, their domains in it.
- */
-class JobService
-{
-    /**
-     * Restore components in their order. mails, imap_acl, sieve_activate and the Sieve scripts of filters are restored by
-     * the dovecot runner, sogo by the restic runner, the others by RestoreService.
-     */
-    public const COMPONENTS = ['mails', 'imap_acl', 'sogo', 'aliases', 'groups', 'sender_acl', 'mail_policies', 'retention_rules',
-        'filters', 'sieve_activate', 'spam', 'bcc', 'syncjobs', 'app_passwords', 'tfa', 'settings', 'password'];
-    /** Components that replace current data with the backup's (all others only add what is missing) */
-    public const OVERWRITE = ['sogo', 'settings', 'password'];
-    /** Components that need the mails: not in snapshots without them (RESTIC_BACKUP_MODE=config) */
-    public const MAIL_COMPONENTS = ['mails', 'imap_acl', 'sogo', 'sieve_activate'];
-    public const MAIL_TARGETS = ['folder', 'original'];
-    /** Domain admin ACL a component needs */
-    private const COMPONENT_ACL = [
-        'groups' => 'mailbox_groups',
-        'mail_policies' => 'mail_policies',
-        'retention_rules' => 'retention',
-        'filters' => 'filters',
-        'sieve_activate' => 'filters',
-        'spam' => 'spam_policy',
-        'bcc' => 'bcc_maps',
-        'syncjobs' => 'syncjobs',
-        'app_passwords' => 'app_passwds',
-    ];
-    /** Components that only work together with another one */
-    private const REQUIRES = ['sieve_activate' => 'filters'];
-    /** Domain admin ACLs the merge respects, stored with the job */
-    private const RESTORE_ACL = ['filters', 'mailbox_groups', 'mail_policies', 'retention', 'spam_policy', 'bcc_maps', 'syncjobs',
-        'app_passwds', 'protocol_access', 'mailbox_relayhost', 'unlimited_quota', 'sogo_access', 'pushover', 'extend_sender_acl',
-        'alias_external_goto', 'ratelimit', 'tls_policy', 'quarantine', 'quarantine_notification', 'quarantine_category'];
-    private const MAX_MAILBOXES = 1000;
-    private const LIST_LIMIT = 100;
-    private const MAX_LIST = 1000;
-    /** Mailboxes a search returns at most */
-    private const SEARCH_LIMIT = 500;
-    /** The runner reports every 10 seconds */
-    private const RUNNER_TIMEOUT = 120;
-    /** @var array<string, array{bool, ?int}> domain => [access of the domain admin, creation time now], per request */
-    private array $domainState = [];
-
-    public function __construct(private JobRepository $jobs, private SnapshotRepository $snapshots, private SettingsService $settings,
-        private Context $context, private string $timezone = 'UTC')
-    {
-    }
-
-    /**
-     * @param mixed $type only jobs of this type ('' or null = all; an unknown type finds nothing)
-     * @return Job[] newest first, at most $limit (0 = LIST_LIMIT)
-     */
-    public function list(int $limit = 0, mixed $type = null): array
-    {
-        $type = Input::text($type);
-        if ($type !== '' && !in_array($type, Job::TYPES, true)) {
-            return [];
-        }
-        return $this->jobs->visible($this->context, ($limit > 0) ? min($limit, self::MAX_LIST) : self::LIST_LIMIT, ($type === '') ? null : $type);
-    }
-
-    /**
-     * Entry of the API. Domain admins get it restricted: e-mail addresses outside the job's domain and its alias domains
-     * and repository details (URLs, restic repository strings, paths of the repository) are shown as "…".
-     */
-    public function view(Job $job, bool $withLog = false): array
-    {
-        $entry = $job->toArray($withLog);
-        if ($this->context->hasAdminRights()) {
-            return $entry;
-        }
-        $domains = array_merge([strtolower($job->domain)], $this->jobs->aliasDomainsOf(strtolower($job->domain)));
-        foreach (['result', 'error', 'log'] as $key) {
-            if (isset($entry[$key])) {
-                $entry[$key] = self::restricted($entry[$key], $domains);
-            }
-        }
-        // the account of an admin who requested the restore stays hidden, the UI shows "Administrator"
-        if (($job->params['requested_role'] ?? '') !== 'domainadmin') {
-            $entry['requested_by'] = '';
-            if (isset($entry['log'])) {
-                $entry['log'] = str_replace('(requested by ' . $job->requestedBy . ')', '(requested by an administrator)', $entry['log']);
-            }
-        }
-        return $entry;
-    }
-
-    /** The job with its log, null when it does not exist or the caller may not see it */
-    public function get(int $id): ?Job
-    {
-        $job = $this->jobs->find($id, true);
-        return ($job !== null && $this->visible($job)) ? $job : null;
-    }
-
-    /**
-     * Queues a backup, check, prune or init job (admins).
-     *
-     * @throws UserError
-     */
-    public function create(array $input): Job
-    {
-        $this->context->requireAdmin();
-        $type = Input::text($input['type'] ?? null);
-        if (!in_array($type, Job::MANUAL_TYPES, true)) {
-            throw new UserError('backup_job_type_invalid', [htmlspecialchars((string)$type)]);
-        }
-        $settings = $this->settings->load();
-        if (!$settings->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        if ($type === 'backup' && $settings->get('password_confirmed') !== '1') {
-            throw new UserError('backup_password_not_confirmed');
-        }
-        $job = new Job($type);
-        $job->requestedBy = (string)$this->context->username;
-        $this->jobs->insertExclusive($job, function (array $active) use ($type): void {
-            foreach ($active as $other) {
-                if ($other->type === $type) {
-                    throw new UserError('backup_job_active', [$type]);
-                }
-            }
-        });
-        return $job;
-    }
-
-    /**
-     * Queues a restore: {snapshot, scope: mailbox|domain, mailboxes | domain, components, mail_target: folder|original}.
-     * All mailboxes belong to one domain. Mailboxes deleted since the snapshot are created from it (create_mailboxes);
-     * an admin may restore a deleted domain (create_domain). Scope domain restores the snapshot's mailboxes of the
-     * domain (the current ones for snapshots without content).
-     *
-     * @throws UserError
-     */
-    public function restore(array $input): Job
-    {
-        $this->requireRestorer();
-        if (!$this->settings->load()->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        $snapshot = $this->snapshot($input['snapshot'] ?? null);
-        $scope = Input::text($input['scope'] ?? 'mailbox');
-        [$domain, $mailboxes] = match ($scope) {
-            'domain' => $this->domainScope($snapshot, $input['domain'] ?? null),
-            'mailbox' => $this->mailboxScope($snapshot, $input['mailboxes'] ?? null, $input['domain'] ?? null),
-            default => throw new UserError('backup_scope_invalid'),
-        };
-        $components = $this->components($input['components'] ?? null);
-        $missing = array_values(array_intersect($components, self::MAIL_COMPONENTS));
-        if ($snapshot->mode() === 'config' && !empty($missing)) {
-            throw new UserError('backup_components_no_mails', [htmlspecialchars(implode(', ', $missing))]);
-        }
-        $mailTarget = Input::text($input['mail_target'] ?? null) ?: 'folder';
-        if (!in_array($mailTarget, self::MAIL_TARGETS, true)) {
-            throw new UserError('backup_mail_target_invalid');
-        }
-        $job = new Job('restore');
-        $job->step = 'files';
-        $job->domain = $domain;
-        $job->requestedBy = (string)$this->context->username;
-        $job->params = [
-            'snapshot' => $snapshot->id,
-            'snapshot_time' => $snapshot->time,
-            'scope' => $scope,
-            'domain' => $domain,
-            'mailboxes' => $mailboxes,
-            'components' => $components,
-            'mail_target' => $mailTarget,
-            'requested_role' => $this->context->hasAdminRights() ? 'admin' : 'domainadmin',
-            'create_domain' => (int)!$this->jobs->isDomain($domain),
-            'create_mailboxes' => array_values(array_filter($mailboxes, fn(string $username): bool => $this->jobs->mailboxDomain($username) === null)),
-        ];
-        if (!$this->context->hasAdminRights()) {
-            $job->params['acl'] = array_combine(self::RESTORE_ACL, array_map(fn(string $acl): int => (int)$this->context->hasAcl($acl), self::RESTORE_ACL));
-            $job->params['allowed_domains'] = array_values(array_unique(array_merge([$domain], $this->jobs->aliasDomainsOf($domain))));
-        }
-        $domainAdmin = !$this->context->hasAdminRights();
-        $this->jobs->insertExclusive($job, function (array $active) use ($mailboxes, $domain, $domainAdmin): void {
-            foreach ($active as $other) {
-                if ($other->type !== 'restore') {
-                    continue;
-                }
-                // a domain admin has one restore per domain at a time
-                if ($domainAdmin && strtolower($other->domain) === $domain) {
-                    throw new UserError('backup_restore_domain_active', [htmlspecialchars($domain)]);
-                }
-                $both = array_intersect($mailboxes, $other->mailboxes());
-                if (!empty($both)) {
-                    throw new UserError('backup_restore_active', [htmlspecialchars((string)reset($both))]);
-                }
-            }
-        });
-        return $job;
-    }
-
-    /**
-     * Cancels a queued or running job (the restic runner stops at its next check): admins any, domain admins the
-     * restores of their domains that domain admins requested. A restore in the dovecot step (mails, mails_running) runs
-     * to its end.
-     *
-     * @throws UserError
-     */
-    public function cancel(int $id): Job
-    {
-        if ($this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
-        $job = $this->jobs->find($id);
-        // domain admins cancel only restores domain admins requested
-        if ($job === null || !$this->visible($job) || (!$this->context->hasAdminRights() && ($job->params['requested_role'] ?? '') !== 'domainadmin')) {
-            throw new AccessDenied();
-        }
-        if (!$job->isActive() || ($job->type === 'restore' && in_array($job->step, ['mails', 'mails_running'], true)) || !$this->jobs->cancel($id)) {
-            throw new UserError('backup_job_not_cancellable', [$id]);
-        }
-        $job->state = 'cancelled';
-        return $job;
-    }
-
-    /**
-     * Snapshots, newest first. Domain admins get those holding one of their domains (see mayUse()), with id, short_id,
-     * time and those domains.
-     */
-    public function snapshots(): array
-    {
-        if ($this->context->hasAdminRights()) {
-            return array_map(fn(Snapshot $snapshot): array => $snapshot->toArray(), $this->snapshots->all());
-        }
-        if (!$this->context->isDomainAdmin()) {
-            return [];
-        }
-        $snapshots = [];
-        // the domains with their creation time come from the snapshot row, the content is not needed here
-        foreach ($this->snapshots->all() as $snapshot) {
-            $domains = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
-            if (!empty($domains)) {
-                $snapshots[] = ['id' => $snapshot->id, 'short_id' => $snapshot->shortId, 'time' => $snapshot->time, 'domains' => $domains,
-                    'mode' => $snapshot->mode()];
-            }
-        }
-        return $snapshots;
-    }
-
-    /**
-     * Domains of a snapshot with their state now: GET backup-snapshot-content/<snapshot>. `available` 0: the snapshot has
-     * no content (not made by this installation), the list comes from its manifest and only existing mailboxes can be restored. A domain
-     * of the manifest missing in the content (its export failed) lists its current mailboxes as well.
-     *
-     * @return array{snapshot: string, time: int, available: int, domains: array[]}|array{} [] when not found or not visible
-     */
-    public function content(mixed $snapshotId): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        if ($snapshot === null) {
-            return [];
-        }
-        $visible = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
-        [$existing, $usernames] = $this->jobs->domainsAndUsernames($visible);
-        $domains = [];
-        foreach ($visible as $domain) {
-            $current = $usernames[$domain] ?? [];
-            $mailboxes = $snapshot->mailboxesOf($domain) ?? $current;
-            $domains[] = ['domain' => $domain, 'exists' => (int)isset($existing[$domain]), 'mailboxes' => count($mailboxes),
-                'missing' => count(array_diff_key($mailboxes, $current))];
-        }
-        return ['snapshot' => $snapshot->id, 'time' => $snapshot->time, 'available' => (int)($snapshot->content !== null), 'domains' => $domains];
-    }
-
-    /**
-     * Search in a snapshot: GET backup-snapshot-search/<snapshot>/<term>. Matches domain names and mailboxes by local
-     * part, address or name ("x@": local part, "@d": domain, "x@d": address), with the rules of content(); at most
-     * SEARCH_LIMIT mailboxes. A matching domain comes with all its mailboxes.
-     *
-     * @return array{domains: array[], truncated: int}|array{} [] when the snapshot is not found or not visible
-     */
-    public function search(mixed $snapshotId, mixed $term): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        $term = mb_strtolower((string)Input::text($term));
-        if ($snapshot === null) {
-            return [];
-        }
-        if (trim($term, '@') === '') {
-            return ['domains' => [], 'truncated' => 0];
-        }
-        $domains = [];
-        $hits = 0;
-        $truncated = 0;
-        [$local, $at, $domainTerm] = str_contains($term, '@') ? [strstr($term, '@', true), true, substr((string)strstr($term, '@'), 1)] : [$term, false, ''];
-        foreach ($snapshot->domainList() as $domain) {
-            if (!$this->mayUse($snapshot, $domain)) {
-                continue;
-            }
-            $domainMatch = $at ? ($domainTerm !== '' && str_contains($domain, $domainTerm)) : str_contains($domain, $term);
-            if ($at && $domainTerm !== '' && !$domainMatch) {
-                continue;
-            }
-            $current = $this->jobs->mailboxNames($domain);
-            $mailboxes = [];
-            foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
-                $username = (string)$username;
-                $part = (string)strstr($username, '@', true);
-                $match = match (true) {
-                    // "x@d": the address, "x@": the local part, "@d": every mailbox of the matching domains
-                    $at && $local !== '' && $domainTerm !== '' => str_contains($username, $term),
-                    $at && $local !== '' => str_contains($part, $local),
-                    $at => true,
-                    default => $domainMatch || str_contains($username, $term) || str_contains(mb_strtolower($name), $term),
-                };
-                if (!$match) {
-                    continue;
-                }
-                if ($hits >= self::SEARCH_LIMIT) {
-                    $truncated = 1;
-                    break;
-                }
-                $hits++;
-                $mailboxes[] = ['username' => $username, 'name' => $name, 'exists' => (int)isset($current[$username])];
-            }
-            // a domain without matching mailboxes only for a search by domain
-            if (!empty($mailboxes) || ($domainMatch && !($at && $local !== '') && !$truncated)) {
-                $domains[] = ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
-            }
-            if ($truncated) {
-                break;
-            }
-        }
-        return ['domains' => $domains, 'truncated' => $truncated];
-    }
-
-    /**
-     * Mailboxes of a domain in a snapshot with their state now: GET backup-snapshot-content/<snapshot>/<domain>
-     *
-     * @return array{domain: string, exists: int, mailboxes: array[]}|array{} [] when not found or not visible
-     */
-    public function domainContent(mixed $snapshotId, mixed $domain): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        $domain = strtolower((string)Input::text($domain));
-        if ($snapshot === null || !$snapshot->hasDomain($domain) || !$this->mayUse($snapshot, $domain)) {
-            return [];
-        }
-        $current = $this->jobs->mailboxNames($domain);
-        $mailboxes = [];
-        foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
-            $mailboxes[] = ['username' => (string)$username, 'name' => $name, 'exists' => (int)isset($current[$username])];
-        }
-        return ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
-    }
-
-    /**
-     * State of the backup for admins: settings state, runner heartbeat, repository, newest job of each type, queued and
-     * running jobs, the next scheduled run. [] for everybody else.
-     */
-    public function status(?int $now = null): array
-    {
-        if (!$this->context->hasAdminRights()) {
-            return [];
-        }
-        $now ??= time();
-        $settings = $this->settings->load();
-        $seen = ($settings->get('state_runner_seen') === '') ? null : (int)$settings->get('state_runner_seen');
-        $snapshots = $this->snapshots->all();
-        $latest = $this->jobs->latestOfEachType();
-        $stats = json_decode($settings->get('state_repo_stats'), true);
-        return [
-            'enabled' => (int)$settings->get('enabled'),
-            'target' => $settings->get('target'),
-            'configured' => (int)$settings->configured(),
-            'password_confirmed' => (int)$settings->get('password_confirmed'),
-            'initialized' => (int)$settings->get('state_initialized'),
-            // the runner cannot decrypt the stored credentials (e.g. a database restored on another server): enter them again
-            'secrets_error' => ($settings->get('state_secrets_error') === '') ? null : $settings->get('state_secrets_error'),
-            // RESTIC_BACKUP_MODE (null until the runner reported it)
-            'mode' => ($settings->get('state_backup_mode') === '') ? null : $settings->get('state_backup_mode'),
-            'runner_seen' => $seen,
-            'runner_age' => ($seen === null) ? null : max(0, $now - $seen),
-            'runner_ok' => (int)($seen !== null && $now - $seen <= self::RUNNER_TIMEOUT),
-            'last_success' => ($settings->get('state_last_success') === '') ? null : (int)$settings->get('state_last_success'),
-            'repo_size' => ($settings->get('state_repo_size') === '') ? null : (int)$settings->get('state_repo_size'),
-            'repo_stats' => is_array($stats) ? $stats : null,
-            'snapshots' => count($snapshots),
-            'latest_snapshot' => empty($snapshots) ? null : $snapshots[0]->time,
-            'next_run' => $this->nextRun($settings, $now),
-            // the longest time between two scheduled backups (seconds): a later last success is overdue
-            'schedule_gap' => $settings->scheduleGap(),
-            'jobs' => array_map(fn(string $type): ?array => isset($latest[$type]) ? $latest[$type]->toArray() : null,
-                array_combine(Job::TYPES, Job::TYPES)),
-            'active' => array_map(fn(Job $job): array => $job->toArray(), $this->jobs->active()),
-        ];
-    }
-
-    /**
-     * Unix time of the next scheduled backup, null while the schedule is off. Like the runner: the schedule is armed
-     * (state_schedule_since/state_schedule) when it is switched on or changed, a run is due at the first schedule time
-     * after that; a due run without a scheduled backup since its time is the next one (now).
-     */
-    private function nextRun(Settings $settings, int $now): ?int
-    {
-        if ($settings->get('enabled') !== '1' || $settings->get('password_confirmed') !== '1' || !$settings->configured()) {
-            return null;
-        }
-        $slots = $settings->scheduleSlots();
-        try {
-            $zone = new DateTimeZone($this->timezone);
-        } catch (\Exception) {
-            $zone = new DateTimeZone('UTC');
-        }
-        $today = (new DateTimeImmutable('@' . $now))->setTimezone($zone);
-        // the last schedule time up to now, and the one after it
-        $due = self::slot($slots, $today, -1);
-        $next = self::slot($slots, $today, 1);
-        $since = $settings->get('state_schedule_since');
-        if ($due === null || $since === '' || $settings->get('state_schedule') !== $settings->get('schedule') || (int)$since > $due) {
-            return $next;
-        }
-        return $this->jobs->scheduledSince($due) ? $next : $due;
-    }
-
-    /**
-     * The newest schedule time up to $now ($direction -1) or the first one after it (1) as Unix time, null without
-     * slots ([weekday, hour, minute], sorted)
-     */
-    private static function slot(array $slots, DateTimeImmutable $now, int $direction): ?int
-    {
-        for ($i = 0; $i <= 7; $i++) {
-            $day = $now->modify(sprintf('%+d day', $direction * $i));
-            foreach (($direction < 0) ? array_reverse($slots) : $slots as [$weekday, $hour, $minute]) {
-                if ($weekday !== (int)$day->format('w')) {
-                    continue;
-                }
-                $slot = $day->setTime($hour, $minute)->getTimestamp();
-                if (($direction < 0) ? $slot <= $now->getTimestamp() : $slot > $now->getTimestamp()) {
-                    return $slot;
-                }
-            }
-        }
-        return null;
-    }
-
-    /** @throws AccessDenied unless the caller may request restores */
-    private function requireRestorer(): void
-    {
-        if ($this->context->isReadOnlyApi() || !($this->context->hasAdminRights() || $this->context->isDomainAdmin())) {
-            throw new AccessDenied();
-        }
-    }
-
-    /** Admins every job; domain admins the restores of their domains requested after the domain was created */
-    private function visible(Job $job): bool
-    {
-        if ($this->context->hasAdminRights()) {
-            return true;
-        }
-        if (!$this->context->isDomainAdmin() || $job->type !== 'restore' || $job->domain === '' || !$this->context->canAccessDomain($job->domain)) {
-            return false;
-        }
-        $created = $this->jobs->domainCreated(strtolower($job->domain));
-        return $created !== null && (int)$job->created >= $created;
-    }
-
-    /**
-     * A value of a job as a domain admin sees it: e-mail addresses outside $domains and repository details become "…"
-     *
-     * @param string[] $domains
-     */
-    private static function restricted(mixed $value, array $domains): mixed
-    {
-        if (is_array($value)) {
-            $restricted = [];
-            foreach ($value as $key => $item) {
-                $restricted[is_string($key) ? (string)self::restricted($key, $domains) : $key] = self::restricted($item, $domains);
-            }
-            return $restricted;
-        }
-        if (!is_string($value)) {
-            return $value;
-        }
-        $value = preg_replace(['~\b(?:s3|sftp|rest|b2|azure|gs|swift|rclone):\S+~i', '~\bhttps?://\S+~i', '~/repository\S*~'], '…', $value);
-        return preg_replace_callback('/[^\s@<>"\'(),;:\[\]]+@([a-z0-9.-]+\.[a-z]{2,})/i',
-            fn(array $match): string => in_array(strtolower($match[1]), $domains, true) ? $match[0] : '…', (string)$value);
-    }
-
-    /** @throws UserError backup_snapshot_not_found (also for snapshots a domain admin may not use) */
-    private function snapshot(mixed $value): Snapshot
-    {
-        $id = (string)Input::text($value);
-        $snapshot = ($id === '') ? null : $this->snapshots->find($id);
-        if ($snapshot === null || (!$this->context->hasAdminRights() && empty(array_filter($snapshot->domainList(),
-            fn(string $domain): bool => $this->mayUse($snapshot, $domain))))) {
-            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($id)]);
-        }
-        return $snapshot;
-    }
-
-    private function visibleSnapshot(mixed $value): ?Snapshot
-    {
-        try {
-            return ($this->context->hasAdminRights() || $this->context->isDomainAdmin()) ? $this->snapshot($value) : null;
-        } catch (UserError) {
-            return null;
-        }
-    }
-
-    /**
-     * Admins every domain of the snapshot; domain admins the domains they administer now, when the snapshot holds this
-     * domain and not a former one of the same name: its recorded creation time is the domain's (snapshots without it:
-     * made after the domain was created)
-     */
-    private function mayUse(Snapshot $snapshot, string $domain): bool
-    {
-        if ($this->context->hasAdminRights()) {
-            return true;
-        }
-        if (!$this->context->isDomainAdmin()) {
-            return false;
-        }
-        $this->domainState[$domain] ??= [$this->context->canAccessDomain($domain), $this->jobs->domainCreated($domain)];
-        [$access, $created] = $this->domainState[$domain];
-        if (!$access) {
-            return false;
-        }
-        $recorded = $snapshot->createdOf($domain);
-        return $created !== null && (($recorded !== null) ? $recorded === $created : $snapshot->time > $created);
-    }
-
-    /**
-     * @return array{0: string, 1: string[]} the domain and its mailboxes in the snapshot (now, for snapshots without content)
-     * @throws UserError
-     */
-    private function domainScope(Snapshot $snapshot, mixed $value): array
-    {
-        $domain = strtolower((string)Input::text($value));
-        if ($domain !== '' && !$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        $exists = $domain !== '' && $this->jobs->isDomain($domain);
-        // a deleted domain is created from the snapshot's export: only when the content lists it
-        if (!$exists && !($this->context->hasAdminRights() && $snapshot->mailboxesOf($domain) !== null)) {
-            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
-        }
-        $this->requireDomain($snapshot, $domain);
-        $listed = $snapshot->mailboxesOf($domain);
-        $mailboxes = ($listed === null) ? $this->jobs->mailboxesOf($domain) : array_map('strval', array_keys($listed));
-        // a deleted domain without mailboxes comes back with its aliases and settings
-        if (empty($mailboxes) && $exists) {
-            throw new UserError('backup_mailboxes_empty');
-        }
-        return [$domain, $mailboxes];
-    }
-
-    /**
-     * @return array{0: string, 1: string[]} the domain of the mailboxes and the mailboxes
-     * @throws UserError
-     */
-    private function mailboxScope(Snapshot $snapshot, mixed $value, mixed $domainValue): array
-    {
-        $mailboxes = array_values(array_unique(array_map('strtolower', self::values($value))));
-        if (empty($mailboxes)) {
-            throw new UserError('backup_mailboxes_empty');
-        }
-        if (count($mailboxes) > self::MAX_MAILBOXES) {
-            throw new UserError('backup_entries_exceeded', [self::MAX_MAILBOXES]);
-        }
-        $domain = strtolower((string)Input::text($domainValue));
-        foreach ($mailboxes as $username) {
-            if (str_contains($username, '@') && !$this->context->hasAdminRights() && !$this->context->canAccessDomain(self::domainOf($username))) {
-                throw new AccessDenied();
-            }
-            // a deleted mailbox can be restored when the snapshot knows it
-            $mailboxDomain = $this->jobs->mailboxDomain($username) ?? (isset($snapshot->mailboxesOf(self::domainOf($username))[$username]) ? self::domainOf($username) : null);
-            if ($mailboxDomain === null) {
-                throw new UserError('backup_mailbox_not_found', [htmlspecialchars($username)]);
-            }
-            if ($domain !== '' && $mailboxDomain !== $domain) {
-                throw new UserError('backup_mailboxes_one_domain');
-            }
-            $domain = $mailboxDomain;
-        }
-        if (!$this->jobs->isDomain($domain) && !$this->context->hasAdminRights()) {
-            throw new AccessDenied();
-        }
-        $this->requireDomain($snapshot, $domain);
-        return [$domain, $mailboxes];
-    }
-
-    /** @throws UserError the snapshot does not hold the domain, or the caller may not restore into it */
-    private function requireDomain(Snapshot $snapshot, string $domain): void
-    {
-        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        if ($snapshot->knowsDomains() && !$snapshot->hasDomain($domain)) {
-            throw new UserError('backup_domain_not_in_snapshot', [htmlspecialchars($domain)]);
-        }
-        if (!$this->mayUse($snapshot, $domain)) {
-            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($snapshot->shortId)]);
-        }
-    }
-
-    /**
-     * @return string[] the requested components in their order
-     * @throws UserError
-     */
-    private function components(mixed $value): array
-    {
-        $requested = array_values(array_unique(self::values($value)));
-        if (empty($requested)) {
-            throw new UserError('backup_components_invalid', ['']);
-        }
-        foreach ($requested as $component) {
-            if (!in_array($component, self::COMPONENTS, true)) {
-                throw new UserError('backup_components_invalid', [htmlspecialchars($component)]);
-            }
-            if (!$this->context->hasAdminRights() && isset(self::COMPONENT_ACL[$component]) && !$this->context->hasAcl(self::COMPONENT_ACL[$component])) {
-                throw new UserError('backup_component_denied', [$component]);
-            }
-            if (isset(self::REQUIRES[$component]) && !in_array(self::REQUIRES[$component], $requested, true)) {
-                throw new UserError('backup_component_requires', [$component, self::REQUIRES[$component]]);
-            }
-        }
-        return array_values(array_intersect(self::COMPONENTS, $requested));
-    }
-
-    private static function domainOf(string $address): string
-    {
-        return strtolower(substr((string)strrchr($address, '@'), 1));
-    }
-
-    /**
-     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
-     *
-     * @return string[]
-     */
-    private static function values(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
-        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/SettingsService.php

@@ -1,327 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Repository\SettingsRepository;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Backup settings: admins read them (also read-only API keys) and save them. Values are checked strictly (at least as
- * strictly as the runner), since the runner passes them to restic and ssh. Secrets are write-only and stored sealed
- * with the public key the runner published (only the runner can decrypt them); an empty secret keeps the stored one,
- * and without a published key no secret can be saved. A changed target, connection or repository password resets
- * state_initialized (the runner tests the repository again); a changed SFTP host or port drops the known host key. A
- * new repository password needs a new confirmation that it is stored outside mailcow. A change of a configured target,
- * its connection or the notify addresses leaves state_target_changed for the runner, which mails the notify addresses
- * of before the change.
- */
-class SettingsService
-{
-    private const FLAGS = ['enabled', 's3_path_style', 'password_confirmed', 'include_index', 'include_fts'];
-    /** Integer settings: name => [min, max] */
-    private const RANGES = [
-        'sftp_port' => [1, 65535],
-        'keep_hourly' => [0, 1000],
-        'keep_daily' => [0, 1000],
-        'keep_weekly' => [0, 1000],
-        'keep_monthly' => [0, 1000],
-        'keep_yearly' => [0, 1000],
-        'prune_weekday' => [0, 6],
-        'check_subset' => [0, 365],
-        'limit_upload' => [0, 100000000],
-    ];
-    private const MIN_PASSWORD = 8;
-    private const MAX_LENGTH = 255;
-    private const MAX_NOTIFY = 20;
-    /** Entries of the schedule */
-    private const MAX_SCHEDULE = 24;
-    private const HOSTNAME = '/^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i';
-
-    public function __construct(private SettingsRepository $repo, private Context $context)
-    {
-    }
-
-    /** The settings for admins, null for everybody else */
-    public function get(): ?Settings
-    {
-        return $this->context->hasAdminRights() ? $this->load() : null;
-    }
-
-    /** The settings without permission check (status, job rules) */
-    public function load(): Settings
-    {
-        return Settings::fromStored($this->repo->load());
-    }
-
-    /**
-     * Saves the given settings; missing ones keep their value, unknown and runner-owned ones are ignored.
-     *
-     * @throws UserError
-     */
-    public function save(array $input): Settings
-    {
-        $this->context->requireAdmin();
-        $current = $this->load();
-        $values = $current->values;
-        foreach ($input as $name => $value) {
-            if (!is_string($name) || !array_key_exists($name, Settings::DEFAULTS) || in_array($name, Settings::RUNNER, true)) {
-                continue;
-            }
-            $value = $this->normalize($name, $value);
-            if (in_array($name, Settings::SECRETS, true)) {
-                if ($value === '') {
-                    continue;
-                }
-                $value = self::seal($value, $current->get('state_secrets_pubkey'));
-            }
-            $values[$name] = $value;
-        }
-        if ($values['password'] !== $current->get('password')
-            && !(array_key_exists('password_confirmed', $input) && $values['password_confirmed'] === '1')) {
-            $values['password_confirmed'] = '0';
-        }
-        foreach (array_merge(['target', 'password'], Settings::CONNECTION[$values['target']]) as $name) {
-            if ($values[$name] !== $current->get($name)) {
-                $values['state_initialized'] = '0';
-                break;
-            }
-        }
-        if ($values['sftp_host'] !== $current->get('sftp_host') || $values['sftp_port'] !== $current->get('sftp_port')) {
-            $values['sftp_host_key'] = '';
-        }
-        $settings = new Settings($values);
-        if ($values['enabled'] === '1' && !$settings->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        // switched on without a time
-        if ($values['enabled'] === '1' && empty($settings->scheduleSlots())) {
-            throw self::invalid('schedule');
-        }
-        $values['state_target_changed'] = $this->targetChange($current, $values);
-        $settings = new Settings($values);
-        $changed = array_diff_assoc($values, $current->values);
-        if (!empty($changed)) {
-            $this->repo->save($changed);
-        }
-        return $settings;
-    }
-
-    /**
-     * state_target_changed after a save: when the target, its connection or the notify addresses change, what the target
-     * was, whom to tell (the notify addresses before the change) and what changed ("target", "notify_email"). An earlier
-     * change the runner has not mailed yet keeps its state and addresses and only gets the new kinds of change, so new
-     * addresses followed by a new target still go to the original addresses. Nothing is recorded for a target that was
-     * not configured or without notify addresses.
-     */
-    private function targetChange(Settings $current, array $values): string
-    {
-        $pending = json_decode($current->get('state_target_changed'), true);
-        if (!$current->configured() && !is_array($pending)) {
-            return $current->get('state_target_changed');
-        }
-        $old = $current->get('target');
-        $watched = array_unique(array_merge(['target'], Settings::CONNECTION[$old] ?? [], Settings::CONNECTION[$values['target']] ?? []));
-        $changed = [];
-        if (!empty(array_filter($watched, fn(string $name): bool => $values[$name] !== $current->get($name)))) {
-            $changed[] = 'target';
-        }
-        if ($values['notify_email'] !== $current->get('notify_email')) {
-            $changed[] = 'notify_email';
-        }
-        if (is_array($pending)) {
-            $pending['changed'] = array_values(array_unique(array_merge((array)($pending['changed'] ?? ['target']), $changed)));
-            return (string)json_encode($pending, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-        }
-        $addresses = array_values(array_filter(explode(',', $current->get('notify_email')), 'strlen'));
-        if (empty($changed) || empty($addresses)) {
-            return '';
-        }
-        $summary = ['target' => $old];
-        foreach (array_diff(Settings::CONNECTION[$old] ?? [], Settings::SECRETS) as $name) {
-            $summary[$name] = $current->get($name);
-        }
-        return (string)json_encode(['old_target' => $summary, 'old_addresses' => $addresses, 'changed' => $changed,
-            'changed_by' => (string)$this->context->username, 'time' => time()], JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-    }
-
-    /**
-     * A secret sealed for the runner: sealed:v1:<base64 of sodium_crypto_box_seal>
-     *
-     * @throws UserError backup_secrets_key_missing while the runner has not published its key
-     */
-    private static function seal(string $secret, string $publicKey): string
-    {
-        $key = base64_decode($publicKey, true);
-        if ($key === false || strlen($key) !== SODIUM_CRYPTO_BOX_PUBLICKEYBYTES) {
-            throw new UserError('backup_secrets_key_missing');
-        }
-        return Settings::SEALED . base64_encode(sodium_crypto_box_seal($secret, $key));
-    }
-
-    /**
-     * The value to store, '' for an empty optional value.
-     *
-     * @throws UserError backup_setting_invalid, backup_password_too_short
-     */
-    private function normalize(string $name, mixed $value): string
-    {
-        if (in_array($name, self::FLAGS, true)) {
-            return self::flag($value) ?? throw self::invalid($name);
-        }
-        if ($name === 'notify_email' && is_string($value)) {
-            // one address per line is fine as well
-            $value = preg_replace('/[\r\n\t]+/', ',', $value);
-        }
-        if ($name === 'schedule') {
-            return self::schedule($value) ?? throw self::invalid($name);
-        }
-        // secrets as typed: spaces at the ends are part of a password
-        $text = (in_array($name, Settings::SECRETS, true) && is_string($value)) ? $value : Input::text($value);
-        if ($text === null || mb_strlen($text) > (($name === 'notify_email') ? self::MAX_LENGTH * self::MAX_NOTIFY : self::MAX_LENGTH) || preg_match('/[\x00-\x1f\x7f]/', $text) || !mb_check_encoding($text, 'UTF-8')) {
-            throw self::invalid($name);
-        }
-        // no scheduled prune (append-only or object-lock targets); a prune job still works
-        if ($name === 'prune_weekday' && $text === '') {
-            return '';
-        }
-        if (isset(self::RANGES[$name])) {
-            $int = Input::intInRange($text, ...self::RANGES[$name]);
-            return ($int === null) ? throw self::invalid($name) : (string)$int;
-        }
-        $valid = match ($name) {
-            'target' => in_array($text, Settings::TARGETS, true),
-            // like the runner: scheme, host and port only
-            's3_endpoint' => self::url($text) && in_array(parse_url($text, PHP_URL_PATH) ?? '', ['', '/'], true),
-            'rest_url' => $text === '' || self::url($text),
-            's3_bucket' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{1,62}$/i', $text),
-            's3_path' => $text === '' || self::path(trim($text, '/'), false),
-            's3_region' => $text === '' || preg_match('/^[a-z0-9-]{1,64}$/i', $text),
-            's3_access_key' => preg_match('/^[\x21-\x7e]*$/', $text),
-            's3_secret_key', 'rest_password' => true,
-            'sftp_host' => $text === '' || preg_match(self::HOSTNAME, $text) || filter_var($text, FILTER_VALIDATE_IP),
-            // the runner's character set (ssh -l)
-            'sftp_user' => $text === '' || preg_match('/^[a-z0-9_][a-z0-9._-]{0,63}$/i', $text),
-            'sftp_path' => $text === '' || self::path($text, true),
-            'rest_user' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{0,63}$/i', $text),
-            // the runner's character set, below the repository folder
-            'local_path' => preg_match('~^[a-z0-9._-]+(/[a-z0-9._-]+)*$~i', $text) && self::path($text, false),
-            'password' => true,
-            'max_unused' => preg_match('/^(\d{1,3})\s*%?$/', $text, $percent) && (int)$percent[1] <= 100,
-            'notify_email' => true,
-            default => false,
-        };
-        if (!$valid) {
-            throw self::invalid($name);
-        }
-        return match ($name) {
-            's3_path' => trim($text, '/'),
-            's3_endpoint' => rtrim($text, '/'),
-            'sftp_path' => rtrim($text, '/'),
-            'password' => self::password($text),
-            'max_unused' => (int)$text . '%',
-            'notify_email' => self::emails($text),
-            default => $text,
-        };
-    }
-
-    /**
-     * The schedule as stored: a list (or its JSON) of {days: weekdays 0 (Sunday) to 6, time: "H:MM"}, up to MAX_SCHEDULE
-     * entries, each with at least one weekday; days sorted, times as HH:MM, repeated entries dropped. Null when invalid.
-     */
-    private static function schedule(mixed $value): ?string
-    {
-        if (is_string($value)) {
-            $value = json_decode($value, true);
-        }
-        if (!is_array($value) || !array_is_list($value) || count($value) > self::MAX_SCHEDULE) {
-            return null;
-        }
-        $entries = [];
-        foreach ($value as $entry) {
-            $days = $entry['days'] ?? null;
-            $time = is_array($entry) && is_string($entry['time'] ?? null) ? $entry['time'] : '';
-            if (!is_array($days) || empty($days) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', $time, $match)) {
-                return null;
-            }
-            foreach ($days as $day) {
-                if (Input::intInRange($day, 0, 6) === null) {
-                    return null;
-                }
-            }
-            $days = array_values(array_unique(array_map('intval', $days)));
-            sort($days);
-            $entries[] = ['days' => $days, 'time' => sprintf('%02d:%02d', $match[1], $match[2])];
-        }
-        return (string)json_encode(array_values(array_unique($entries, SORT_REGULAR)));
-    }
-
-    /** http(s) URL without credentials, query or fragment (rest_user/rest_password hold the credentials) */
-    private static function url(string $url): bool
-    {
-        $parts = parse_url($url);
-        return filter_var($url, FILTER_VALIDATE_URL) !== false && is_array($parts) && in_array(strtolower($parts['scheme'] ?? ''), ['http', 'https'], true)
-            && !isset($parts['user']) && !isset($parts['pass']) && !isset($parts['query']) && !isset($parts['fragment'])
-            && (preg_match(self::HOSTNAME, (string)($parts['host'] ?? '')) || filter_var(trim((string)($parts['host'] ?? ''), '[]'), FILTER_VALIDATE_IP));
-    }
-
-    /** A path of plain names separated by /, without . and .. parts; a leading / only where $absolute is allowed */
-    private static function path(string $path, bool $absolute): bool
-    {
-        if ($path === '' || (str_starts_with($path, '/') && !$absolute)) {
-            return false;
-        }
-        foreach (explode('/', trim($path, '/')) as $part) {
-            if ($part === '.' || $part === '..' || !preg_match('/^[a-z0-9_~+@][a-z0-9._~+@-]*$/i', $part)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /** @throws UserError backup_password_too_short ('' keeps the stored password) */
-    private static function password(string $password): string
-    {
-        if ($password !== '' && mb_strlen($password) < self::MIN_PASSWORD) {
-            throw new UserError('backup_password_too_short', [self::MIN_PASSWORD]);
-        }
-        return $password;
-    }
-
-    /** @throws UserError for an invalid address or too many */
-    private static function emails(string $text): string
-    {
-        $emails = array_values(array_unique(array_filter(preg_split('/[\s,;]+/', $text), 'strlen')));
-        if (count($emails) > self::MAX_NOTIFY) {
-            throw self::invalid('notify_email');
-        }
-        foreach ($emails as $email) {
-            if (!filter_var($email, FILTER_VALIDATE_EMAIL)) {
-                throw self::invalid('notify_email');
-            }
-        }
-        return implode(',', $emails);
-    }
-
-    /** Checkbox or API flag as '0'/'1'; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked; null = invalid */
-    private static function flag(mixed $value): ?string
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        if (!is_scalar($value) && $value !== null) {
-            return null;
-        }
-        $flag = filter_var($value ?? false, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
-        return ($flag === null) ? null : (string)(int)$flag;
-    }
-
-    private static function invalid(string $name): UserError
-    {
-        return new UserError('backup_setting_invalid', [$name]);
-    }
-}

data/web/inc/lib/mailcow/Identity/Service/SyncPlanner.php

@@ -44,8 +44,6 @@ class SyncPlanner
         /** @var array<string, DirectoryUser> $candidates email => user to create, template still open */
         $candidates = [];
         $duplicates = [];
-        /** @var array<string, true> $ambiguous unlinked mailboxes whose email several IdP users have, left as they are */
-        $ambiguous = [];
         foreach ($listing->users as $user) {
             if (isset($owners[$user->id])) {
                 $username = $owners[$user->id];
@@ -57,19 +55,10 @@ class SyncPlanner
                 $plan->skipped++;
             } elseif (isset($state->accounts[$user->email])) {
                 $account = $state->accounts[$user->email];
-                if ($account->externalId !== null) {
+                if ($account->externalId !== null || isset($plan->links[$user->email])) {
                     self::conflict($plan, $user->email . ' is linked to another IdP user');
                     continue;
                 }
-                if (isset($plan->links[$user->email]) || isset($ambiguous[$user->email])) {
-                    // Which IdP user owns the mailbox is unclear: link none of them
-                    if (!isset($ambiguous[$user->email])) {
-                        self::conflict($plan, 'Several IdP users have the email ' . $user->email);
-                    }
-                    unset($plan->links[$user->email], $matched[$user->email]);
-                    $ambiguous[$user->email] = true;
-                    continue;
-                }
                 $plan->links[$user->email] = $user->id;
                 $matched[$user->email] = $user;
             } elseif (isset($state->mailboxes[$user->email]) || !$settings->importUsers || !$user->enabled
@@ -96,7 +85,7 @@ class SyncPlanner
         if ($groups !== null) {
             $this->planGroups($settings, $groups, $state, $usernames, $matched, $plan);
         }
-        $this->planStates($settings, $state, $matched, $ambiguous, $plan);
+        $this->planStates($settings, $state, $matched, $plan);
         if ($settings->periodicSync) {
             $this->planUpdates($settings, $state, $matched, $plan);
         }
@@ -222,13 +211,10 @@ class SyncPlanner
     }
 
     /** @param array<string, DirectoryUser> $matched */
-    private function planStates(Settings $settings, State $state, array $matched, array $ambiguous, SyncPlan $plan): void
+    private function planStates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
     {
         foreach ($state->accounts as $username => $account) {
             $username = (string)$username;
-            if (isset($ambiguous[$username])) {
-                continue;
-            }
             $user = $matched[$username] ?? null;
             if ($user === null && $account->missingSince === null) {
                 $plan->missing[] = $username;

data/web/inc/lib/mailcow/Imapsync/Model/CustomParams.php

@@ -19,14 +19,6 @@ class CustomParams
 {
     private const MAX_PAIRS = 100;
     private const MAX_VALUE_LENGTH = 1000;
-    /**
-     * Never allowed, even when configured: code or command execution, local files, the environment
-     * of the container (tests*), passwords in the output and stopping other runs (abort).
-     * The runner (dovecot/imapsync_runner.py) skips the same options.
-     */
-    public const BLOCKED = ['abort', 'delete2foldersbutnot', 'delete2foldersonly', 'logdir', 'logfile', 'maxlinelengthcmd',
-        'oauthrefreshcmd1', 'oauthrefreshcmd2', 'passfile1', 'passfile2', 'pidfile', 'pipemess', 'regexflag', 'regexmess',
-        'regextrans2', 'showpasswords', 'skipmess', 'tests', 'testsdebug', 'testslive', 'testslive6', 'testsunit', 'tmpdir'];
 
     private static bool $warnedOldFormat = false;
 
@@ -65,7 +57,7 @@ class CustomParams
 
     /**
      * The effective allowlist: $IMAPSYNC_OPTIONS (after vars.local.inc.php) without the old
-     * 'whitelist' key, malformed entries and BLOCKED options; the defaults when nothing valid is left.
+     * 'whitelist' key and malformed entries; the defaults when nothing valid is left.
      */
     public static function allowlist(array $configured, array $defaults): array
     {
@@ -81,7 +73,7 @@ class CustomParams
             self::$warnedOldFormat = true;
             error_log("mailcow: \$IMAPSYNC_OPTIONS['whitelist'] (old format) is ignored, use 'option' => true|false|'regex' (see data/web/inc/vars.inc.php)");
         }
-        return array_diff_key(empty($allowlist) ? $defaults : $allowlist, array_flip(self::BLOCKED));
+        return empty($allowlist) ? $defaults : $allowlist;
     }
 
     /** The value to store for one option; throws when the allowlist rule is violated */

data/web/inc/lib/mailcow/Imapsync/Model/Source.php

@@ -29,7 +29,6 @@ class Source
     public string $host = '';
     public int $port = 0;
     public ?string $enc = 'TLS';
-    public int $verifyCert = 1;
     public ?string $authType = 'PLAIN';
     public string $oauthFlow = self::FLOW_CLIENT_CREDENTIALS;
     public ?string $tokenEndpoint = null;
@@ -66,7 +65,6 @@ class Source
         $source->scope = (string)($row['scope'] ?? 'user');
         $source->host = (string)($row['host1'] ?? '');
         $source->port = (int)($row['port1'] ?? 0);
-        $source->verifyCert = (int)($row['verify_cert'] ?? 1);
         $source->oauthFlow = (string)($row['oauth_flow'] ?? self::FLOW_CLIENT_CREDENTIALS);
         $source->tokenExpires = isset($row['oauth_token_expires']) ? (int)$row['oauth_token_expires'] : null;
         $source->refreshAttemptAge = isset($row['refresh_attempt_age']) ? (int)$row['refresh_attempt_age'] : null;
@@ -114,7 +112,6 @@ class Source
             'host1' => $this->host,
             'port1' => $this->port,
             'enc1' => $this->enc,
-            'verify_cert' => $this->verifyCert,
             'auth_type' => $this->authType,
             'oauth_flow' => $this->oauthFlow,
             'oauth_token_endpoint' => $this->tokenEndpoint,

data/web/inc/lib/mailcow/Imapsync/Repository/SourceRepository.php

@@ -48,12 +48,6 @@ class SourceRepository extends Repository
         return (bool)$this->query("SELECT 1 FROM `imapsync` WHERE `source_id` = :id LIMIT 1", ['id' => $id])->fetchColumn();
     }
 
-    /** @return string[] mailboxes whose sync jobs use the source */
-    public function jobMailboxes(int $id): array
-    {
-        return $this->query("SELECT DISTINCT `user2` FROM `imapsync` WHERE `source_id` = :id", ['id' => $id])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
     public function isSuperAdmin(string $username): bool
     {
         return (bool)$this->query("SELECT 1 FROM `admin` WHERE `username` = :username AND `superadmin` = '1'",
@@ -206,7 +200,6 @@ class SourceRepository extends Repository
             'host1' => $source->host,
             'port1' => $source->port,
             'enc1' => $source->enc,
-            'verify_cert' => $source->verifyCert,
             'auth_type' => $source->authType,
             'oauth_flow' => $source->oauthFlow,
             'oauth_token_endpoint' => $source->tokenEndpoint,

data/web/inc/lib/mailcow/Imapsync/Service/SourceService.php

@@ -121,11 +121,7 @@ class SourceService
         if ($writeTargets) {
             $this->applyScope($source, $input['scope'] ?? $current->scope, $input['domains'] ?? [], $input['users'] ?? []);
         }
-        $connectionChanged = self::connection($current) !== self::connection($source);
+        $this->repo->update($source, $writeTargets, self::connection($current) !== self::connection($source));
-        if ($connectionChanged || ($current->verifyCert && !$source->verifyCert)) {
-            $this->requireAllJobsAccessible($source);
-        }
-        $this->repo->update($source, $writeTargets, $connectionChanged);
         $this->fetchToken($source);
         return $source;
     }
@@ -204,24 +200,6 @@ class SourceService
         return $this->context->hasAcl('syncjobs') || $this->context->isAdmin();
     }
 
-    /**
-     * The connection of a source only changes when the caller may access every mailbox whose sync jobs use it,
-     * their stored passwords would go to the new server.
-     *
-     * @throws UserError imapsync_source_used_by_others
-     */
-    private function requireAllJobsAccessible(Source $source): void
-    {
-        if ($this->context->hasAdminRights()) {
-            return;
-        }
-        foreach ($this->repo->jobMailboxes((int)$source->id) as $mailbox) {
-            if (!$this->context->canAccessMailbox($mailbox)) {
-                throw new UserError('imapsync_source_used_by_others', [htmlspecialchars($source->name)]);
-            }
-        }
-    }
-
     /** @throws AccessDenied when the caller cannot see or edit the source */
     private function editable(int $id): Source
     {
@@ -296,7 +274,6 @@ class SourceService
         $source->host = $host;
         $source->port = $port;
         $source->enc = $enc;
-        $source->verifyCert = isset($input['verify_cert']) ? (int)!empty($input['verify_cert']) : $source->verifyCert;
         $source->authType = $authType;
         $source->active = isset($input['active']) ? (int)$input['active'] : $source->active;
     }

data/web/inc/lib/mailcow/MailPolicy/Controller/MailPolicyApiController.php

@@ -1,132 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Controller;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\MailPolicy\Service\PolicyEngine;
-use Mailcow\MailPolicy\Service\PolicyService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php and edit.php (routes `mail-policy`), and the answer to the Postfix policy
- * service (data/conf/rspamd/dynmaps/mailpolicy.php). Messages are logged as ['mail_policy', action, 'policy', data].
- */
-class MailPolicyApiController
-{
-    public function __construct(private PolicyService $policies, private PolicyEngine $engine, private ApiResponse $response)
-    {
-    }
-
-    /** POST add/mail-policy */
-    public function addPolicy(array $attr): bool
-    {
-        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
-            $policy = $this->policies->create($attr);
-            return $this->response->success(['mail_policy_added', htmlspecialchars($policy->name)], $log);
-        });
-    }
-
-    /** POST edit/mail-policy: one message per id */
-    public function editPolicies(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
-            $this->policies->requireEditor();
-            return $this->batch((array)($data['id'] ?? []), $log,
-                fn(int $id): array => ['mail_policy_modified', htmlspecialchars($this->policies->update($id, $attr)->name)]);
-        });
-    }
-
-    /** POST delete/mail-policy: one message per id */
-    public function deletePolicies(array $items): bool
-    {
-        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
-            $this->policies->requireEditor();
-            return $this->batch($items, $log, fn(int $id): array => ['mail_policy_removed', htmlspecialchars($this->policies->delete($id)->name)]);
-        });
-    }
-
-    /** GET mail-policy/all[/<domain,domain>] */
-    public function getPolicies(?string $domains = null): array
-    {
-        $list = array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
-        return array_values(array_filter(array_map(fn(int $id): array => $this->getPolicy($id), $this->policies->visibleIds($list))));
-    }
-
-    /** GET mail-policy/<id>, also edit.php */
-    public function getPolicy(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $policy = ($id === null) ? null : $this->policies->get($id);
-        return ($policy === null) ? [] : $this->policies->toArray($policy);
-    }
-
-    /** GET mail-policy/test/<sender>/<recipient>[/login]: would the mail be accepted? Sender "-" = null sender; [] when not allowed to ask */
-    public function test(string $sender, string $recipient, bool $loggedIn): array
-    {
-        try {
-            return $this->policies->test(($sender === '-') ? '' : $sender, $recipient, $loggedIn);
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /** functions.mailbox.inc.php: a mailbox or alias was renamed */
-    public function addressRenamed(string $old, string $new): void
-    {
-        $this->policies->addressRenamed($old, $new);
-    }
-
-    /** Answer for the Postfix policy service: DUNNO or the rejection */
-    public function check(string $sender, string $recipient, string $login, int $port, string $helo = ''): string
-    {
-        return $this->engine->check($sender, $recipient, $login, $port, $helo)->action();
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, mixed $data, callable $work): bool
-    {
-        $log = ['mail_policy', $action, 'policy', $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** One message per id: the success message returned by $work(id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id)), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/MailPolicyModule.php

@@ -1,44 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\MailPolicy\Controller\MailPolicyApiController;
-use Mailcow\MailPolicy\Repository\LookupRepository;
-use Mailcow\MailPolicy\Repository\PolicyRepository;
-use Mailcow\MailPolicy\Service\PolicyEngine;
-use Mailcow\MailPolicy\Service\PolicyService;
-use PDO;
-
-/**
- * Composition root of the mail policies: whom mailboxes and groups may send to and receive from, checked by
- * Postfix at RCPT TO through the policy service (mailcow-policy in master.cf, data/conf/rspamd/dynmaps/mailpolicy.php).
- *
- *   process_get_return(MailPolicyModule::fromGlobals()->api->getPolicies());
- *   echo MailPolicyModule::create($pdo, Context::system())->api->check($sender, $recipient, $login, $port);  // policy service
- */
-class MailPolicyModule
-{
-    public function __construct(
-        public readonly PolicyService $policies,
-        public readonly PolicyEngine $engine,
-        public readonly MailPolicyApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context): self
-    {
-        $engine = new PolicyEngine(new LookupRepository($pdo));
-        $policies = new PolicyService(new PolicyRepository($pdo), $engine, $context);
-        return new self($policies, $engine, new MailPolicyApiController($policies, $engine, new ApiResponse()));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Model/Decision.php

@@ -1,56 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Model;
-
-/**
- * Result of the policy check for one recipient: allowed, refused by the send or receive part of a policy, or refused
- * because the recipient is written in a form the check cannot follow (routing through another address).
- */
-class Decision
-{
-    private const TEXT = [
-        'send' => 'Sending to this recipient is not allowed by the mail policy of the sender',
-        'receive' => 'This recipient does not accept mail from this sender',
-        'address' => 'This form of the recipient address is not allowed by the mail policy',
-    ];
-
-    private function __construct(public readonly bool $allowed, public readonly string $side = '', public readonly ?Policy $policy = null)
-    {
-    }
-
-    public static function allow(): self
-    {
-        return new self(true);
-    }
-
-    /** $side send or receive; without $policy the default text is used */
-    public static function deny(string $side, ?Policy $policy = null): self
-    {
-        return new self(false, $side, $policy);
-    }
-
-    /** The recipient routes through another address (user%domain, domain!user, a second @) */
-    public static function routedAddress(): self
-    {
-        return new self(false, 'address');
-    }
-
-    /** Text of the rejection: the text of the policy or the default */
-    public function text(): string
-    {
-        return ($this->policy !== null && $this->policy->rejectText !== '') ? $this->policy->rejectText : self::defaultText($this->side);
-    }
-
-    public static function defaultText(string $side): string
-    {
-        return self::TEXT[$side] ?? '';
-    }
-
-    /** Answer for the Postfix policy protocol */
-    public function action(): string
-    {
-        return $this->allowed ? 'DUNNO' : '550 5.7.1 ' . $this->text();
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Model/Policy.php

@@ -1,111 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Model;
-
-/**
- * A mail policy (table `mail_policy`): whom its mailboxes and groups may send to and receive from.
- *
- * Each direction has a mode: `all` (only the blocked entries are refused), `internal` (the own domain and its
- * alias domains, plus the allowed entries) or `list` (only the allowed entries). Blocked entries are always
- * refused. Entries are addresses, domains or `group:<id>`. A sender counts as internal only when it is logged in
- * to this mailcow (SASL) with a mailbox of the domain, never because of its address.
- */
-class Policy
-{
-    public const MODES = ['all', 'internal', 'list'];
-
-    public ?int $id = null;
-    public string $domain = '';
-    public string $name = '';
-    public string $description = '';
-    public string $sendMode = 'all';
-    /** @var string[] */
-    public array $sendAllow = [];
-    /** @var string[] */
-    public array $sendBlock = [];
-    public string $receiveMode = 'all';
-    /** @var string[] */
-    public array $receiveAllow = [];
-    /** @var string[] */
-    public array $receiveBlock = [];
-    /** Text of the rejection, '' = default text */
-    public string $rejectText = '';
-    public int $active = 1;
-    /** @var string[] usernames the policy is assigned to */
-    public array $mailboxes = [];
-    /** @var int[] groups the policy is assigned to: their members and their address */
-    public array $groups = [];
-    public ?string $created = null;
-    public ?string $modified = null;
-
-    /** Row of `mail_policy`; assignments are set by the repository */
-    public static function fromRow(array $row): self
-    {
-        $policy = new self();
-        $policy->id = (int)$row['id'];
-        $policy->domain = (string)$row['domain'];
-        $policy->name = (string)$row['name'];
-        $policy->description = (string)($row['description'] ?? '');
-        $policy->sendMode = (string)$row['send_mode'];
-        $policy->sendAllow = self::list($row['send_allow'] ?? null);
-        $policy->sendBlock = self::list($row['send_block'] ?? null);
-        $policy->receiveMode = (string)$row['receive_mode'];
-        $policy->receiveAllow = self::list($row['receive_allow'] ?? null);
-        $policy->receiveBlock = self::list($row['receive_block'] ?? null);
-        $policy->rejectText = (string)($row['reject_text'] ?? '');
-        $policy->active = (int)$row['active'];
-        $policy->created = $row['created'] ?? null;
-        $policy->modified = $row['modified'] ?? null;
-        return $policy;
-    }
-
-    /** Column values for `mail_policy` */
-    public function toRow(): array
-    {
-        return [
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'send_mode' => $this->sendMode,
-            'send_allow' => json_encode($this->sendAllow),
-            'send_block' => json_encode($this->sendBlock),
-            'receive_mode' => $this->receiveMode,
-            'receive_allow' => json_encode($this->receiveAllow),
-            'receive_block' => json_encode($this->receiveBlock),
-            'reject_text' => $this->rejectText,
-            'active' => $this->active,
-        ];
-    }
-
-    /** Entry of the API and the edit page */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'send_mode' => $this->sendMode,
-            'send_allow' => $this->sendAllow,
-            'send_block' => $this->sendBlock,
-            'receive_mode' => $this->receiveMode,
-            'receive_allow' => $this->receiveAllow,
-            'receive_block' => $this->receiveBlock,
-            'reject_text' => $this->rejectText,
-            'active' => $this->active,
-            'mailboxes' => $this->mailboxes,
-            'groups' => $this->groups,
-            'created' => $this->created,
-            'modified' => $this->modified,
-        ];
-    }
-
-    /** @return string[] entries of a JSON list column */
-    private static function list(mixed $json): array
-    {
-        $list = is_string($json) ? json_decode($json, true) : null;
-        return is_array($list) ? array_values(array_filter($list, 'is_string')) : [];
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Repository/LookupRepository.php

@@ -1,141 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\MailPolicy\Model\Policy;
-
-/**
- * Read-only lookups of the policy check: mailboxes, aliases, alias domains, groups and the active policies of a
- * mailbox or group. Addresses are passed in lower case.
- */
-class LookupRepository extends Repository
-{
-    /** @return array{username: string, domain: string}|null a regular mailbox that is not disabled */
-    public function mailbox(string $username): ?array
-    {
-        $row = $this->query("SELECT `username`, `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = '' AND `active` != '0'",
-            ['username' => $username])->fetch();
-        return is_array($row) ? ['username' => strtolower((string)$row['username']), 'domain' => strtolower((string)$row['domain'])] : null;
-    }
-
-    /** Target domain of an active alias domain */
-    public function aliasDomainTarget(string $domain): ?string
-    {
-        $target = $this->query("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'",
-            ['domain' => $domain])->fetchColumn();
-        return is_string($target) ? strtolower($target) : null;
-    }
-
-    /** A domain or alias domain Postfix delivers itself (as mysql_virtual_domains_maps.cf: active, no backup MX) */
-    public function isLocalDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :d1 AND `active` = '1'
-          UNION SELECT 1 FROM `domain` WHERE `domain` = :d2 AND `active` = '1' AND `backupmx` = '0'",
-            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
-    }
-
-    /**
-     * Stored spelling of a mailbox, alias or temporary alias address. MySQL compares without case and accents, so
-     * Postfix delivers mail for chéf@ to chef@.
-     */
-    public function storedAddress(string $address): ?string
-    {
-        $stored = $this->query("SELECT `username` FROM `mailbox` WHERE `username` = :a1 UNION SELECT `address` FROM `alias` WHERE `address` = :a2
-          UNION SELECT `address` FROM `spamalias` WHERE `address` = :a3 LIMIT 1", ['a1' => $address, 'a2' => $address, 'a3' => $address])->fetchColumn();
-        return is_string($stored) ? strtolower($stored) : null;
-    }
-
-    /** Stored spelling of a domain or alias domain, see storedAddress() */
-    public function storedDomain(string $domain): ?string
-    {
-        $stored = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :d1 UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :d2 LIMIT 1",
-            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
-        return is_string($stored) ? strtolower($stored) : null;
-    }
-
-    /** Active group with this distribution address */
-    public function groupByAddress(string $address): ?int
-    {
-        $id = $this->query("SELECT `id` FROM `mailbox_group` WHERE `address` = :address AND `active` = '1'", ['address' => $address])->fetchColumn();
-        return ($id === false) ? null : (int)$id;
-    }
-
-    public function groupAddress(int $id): ?string
-    {
-        $address = $this->query("SELECT `address` FROM `mailbox_group` WHERE `id` = :id AND `active` = '1'", ['id' => $id])->fetchColumn();
-        return is_string($address) && $address !== '' ? strtolower($address) : null;
-    }
-
-    /** Member of an active group */
-    public function isGroupMember(int $groupId, string $username): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
-          WHERE `gm`.`group_id` = :id AND `gm`.`username` = :username AND `g`.`active` = '1'",
-            ['id' => $groupId, 'username' => $username])->fetchColumn();
-    }
-
-    /**
-     * What Postfix' virtual alias maps find for this key, in their order: an alias (also the own alias of a mailbox
-     * and the alias of a group address) or a temporary alias.
-     *
-     * @return array{kind: string, domain?: string, internal?: bool, owner?: string}|null
-     */
-    public function aliasTarget(string $address): ?array
-    {
-        $alias = $this->query("SELECT `domain`, `internal` FROM `alias` WHERE `address` = :address AND `active` IN ('1', '2')",
-            ['address' => $address])->fetch();
-        if (is_array($alias)) {
-            return ['kind' => 'alias', 'domain' => strtolower((string)$alias['domain']), 'internal' => $alias['internal'] == '1'];
-        }
-        $owner = $this->query("SELECT `goto` FROM `spamalias` WHERE `address` = :address AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0)",
-            ['address' => $address])->fetchColumn();
-        return is_string($owner) ? ['kind' => 'temporary', 'owner' => strtolower($owner)] : null;
-    }
-
-    /** An active catch-all alias of the domain has the internal flag (applies to the whole domain) */
-    public function internalCatchAll(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address AND `internal` = '1' AND `active` IN ('1', '2')",
-            ['address' => '@' . $domain])->fetchColumn();
-    }
-
-    /** An active policy with members restricts sending */
-    public function hasSendRules(): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mail_policy` `p` JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          WHERE `p`.`active` = '1' AND (`p`.`send_mode` != 'all' OR `p`.`send_block` != '[]') LIMIT 1")->fetchColumn();
-    }
-
-    /**
-     * @return Policy[] active policies of the mailbox: assigned to it or to an active group it is a member of; only
-     *                  policies of its own domain, so a group with members of other domains cannot restrict them
-     */
-    public function policiesForMailbox(string $username): array
-    {
-        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
-          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          JOIN `mailbox` `m` ON `m`.`username` = :u1 AND `m`.`domain` = `p`.`domain`
-          WHERE `p`.`active` = '1' AND (`pm`.`username` = :u2 OR `pm`.`group_id` IN (
-            SELECT `gm`.`group_id` FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
-            WHERE `gm`.`username` = :u3 AND `g`.`active` = '1'))
-          ORDER BY `p`.`id`", ['u1' => $username, 'u2' => $username, 'u3' => $username]);
-    }
-
-    /** @return Policy[] active policies assigned to the group */
-    public function policiesForGroup(int $groupId): array
-    {
-        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
-          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          WHERE `p`.`active` = '1' AND `pm`.`group_id` = :id
-          ORDER BY `p`.`id`", ['id' => $groupId]);
-    }
-
-    /** @return Policy[] */
-    private function policies(string $sql, array $params): array
-    {
-        return array_map(fn(array $row): Policy => Policy::fromRow($row), $this->query($sql, $params)->fetchAll());
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Repository/PolicyRepository.php

@@ -1,148 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Repository;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\MailPolicy\Model\Policy;
-use PDO;
-
-/**
- * Tables `mail_policy` and `mail_policy_member` (the mailboxes and groups a policy is assigned to), plus the
- * lookups the editing needs. Assignments go with their mailbox, group or domain (foreign keys, ON DELETE CASCADE).
- */
-class PolicyRepository extends Repository
-{
-    public function find(int $id): ?Policy
-    {
-        $row = $this->query("SELECT * FROM `mail_policy` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? $this->withMembers(Policy::fromRow($row)) : null;
-    }
-
-    /**
-     * @param string[] $domains limit to these domains, [] = all visible
-     * @return int[] ids of the policies visible to the caller, ordered by domain and name
-     */
-    public function visibleIds(Context $context, array $domains = []): array
-    {
-        if ($context->hasAdminRights()) {
-            [$visible, $params] = ['1=1', []];
-        } elseif ($context->isDomainAdmin()) {
-            [$visible, $params] = ["`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
-                ['vis_da' => $context->username]];
-        } else {
-            return [];
-        }
-        $sql = "SELECT `id` FROM `mail_policy` WHERE $visible";
-        if (!empty($domains)) {
-            $names = [];
-            foreach (array_values($domains) as $i => $domain) {
-                $names[] = ':dom' . $i;
-                $params['dom' . $i] = $domain;
-            }
-            $sql .= " AND `domain` IN (" . implode(', ', $names) . ")";
-        }
-        return array_map('intval', $this->query($sql . " ORDER BY `domain`, `name`", $params)->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mail_policy` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
-            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
-    }
-
-    /** Inserts or updates the policy with its assignments; returns its id */
-    public function save(Policy $policy): int
-    {
-        return $this->transaction(function () use ($policy): int {
-            if ($policy->id === null) {
-                $policy->id = $this->insertRow('mail_policy', $policy->toRow());
-            } else {
-                $this->updateRow('mail_policy', $policy->id, $policy->toRow());
-                $this->query("DELETE FROM `mail_policy_member` WHERE `policy_id` = :id", ['id' => $policy->id]);
-            }
-            foreach ($policy->mailboxes as $username) {
-                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'username' => $username]);
-            }
-            foreach ($policy->groups as $groupId) {
-                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'group_id' => $groupId]);
-            }
-            return $policy->id;
-        });
-    }
-
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `mail_policy` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /** Address entries of all lists follow a renamed mailbox or alias */
-    public function renameAddress(string $old, string $new): void
-    {
-        $this->transaction(function () use ($old, $new): void {
-            $columns = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
-            foreach ($this->query("SELECT `id`, `" . implode('`, `', $columns) . "` FROM `mail_policy`")->fetchAll() as $row) {
-                $changed = [];
-                foreach ($columns as $column) {
-                    $entries = json_decode((string)$row[$column], true);
-                    if (is_array($entries) && in_array($old, $entries, true)) {
-                        $changed[$column] = json_encode(array_values(array_unique(array_map(fn(string $entry): string => ($entry === $old) ? $new : $entry, $entries))));
-                    }
-                }
-                if ($changed !== []) {
-                    $this->updateRow('mail_policy', (int)$row['id'], $changed);
-                }
-            }
-        });
-    }
-
-    /** A primary domain (policies never belong to alias domains) */
-    public function isDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** Domain of a regular mailbox, null if there is none */
-    public function mailboxDomain(string $username): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = ''", ['username' => $username])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    public function groupDomain(int $id): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox_group` WHERE `id` = :id", ['id' => $id])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupsIn(array $domains): array
-    {
-        if (empty($domains)) {
-            return [];
-        }
-        $names = [];
-        $params = [];
-        foreach (array_values($domains) as $i => $domain) {
-            $names[] = ':dom' . $i;
-            $params['dom' . $i] = $domain;
-        }
-        return $this->query("SELECT `id`, `name`, `domain`, `address` FROM `mailbox_group` WHERE `domain` IN (" . implode(', ', $names) . ")
-          ORDER BY `domain`, `name`", $params)->fetchAll();
-    }
-
-    private function withMembers(Policy $policy): Policy
-    {
-        foreach ($this->query("SELECT `username`, `group_id` FROM `mail_policy_member` WHERE `policy_id` = :id ORDER BY `username`, `group_id`",
-            ['id' => $policy->id])->fetchAll() as $row) {
-            if ($row['username'] !== null) {
-                $policy->mailboxes[] = (string)$row['username'];
-            } elseif ($row['group_id'] !== null) {
-                $policy->groups[] = (int)$row['group_id'];
-            }
-        }
-        return $policy;
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyEngine.php

@@ -1,325 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Service;
-
-use Mailcow\MailPolicy\Model\Decision;
-use Mailcow\MailPolicy\Model\Policy;
-use Mailcow\MailPolicy\Repository\LookupRepository;
-
-/**
- * Policy check of one recipient, asked by Postfix at RCPT TO (data/conf/rspamd/dynmaps/mailpolicy.php).
- *
- * Send part: the policies of the logged-in mailbox (assigned to it or to its groups) and of the mailbox the
- * sender address belongs to. Receive part: what Postfix delivers the recipient to, looked up in the order of its
- * virtual alias maps (address, address without +tag, mailbox of an alias domain): the policies of the mailbox, of
- * the group address or of the owner of a temporary alias, and the `internal` flag of the alias (a catch-all alias
- * with the flag covers its whole domain). Mail sent to an alias or group reaches its members without their own
- * policies. Every policy must allow the mail.
- *
- * Internal: a recipient in the domain of the policy or one of its alias domains; a sender logged in with a
- * mailbox of that domain. Entries of a local domain or address count for logged-in senders only, so they cannot
- * be forged; block entries also match the envelope sender. External entries are compared with the envelope sender.
- * Addresses are compared the way Postfix finds them: lower case, without trailing dot, in their stored spelling
- * (MySQL ignores accents). Recipients routed through localhost (user%domain@localhost, domain!user@localhost,
- * user@domain@localhost) are refused, Postfix would deliver them to the routed address.
- *
- * Dovecot submits Sieve redirects, vacation replies and notifications on port 588 without login, SOGo sends
- * reminders with a service login: mail with the address of a mailbox counts as sent by that mailbox. Other mail of
- * these services (bounces redirected with the null sender, Sieve rejects) has no known sender: it is checked like
- * mail from outside and does not leave the mailcow while any policy restricts sending. Quota warnings pass.
- */
-class PolicyEngine
-{
-    /** Ports of mailcow's own deliveries: quarantine release (590), BCC and archive copies (591) */
-    private const INTERNAL_PORTS = [590, 591];
-    /** Submission of SOGo and Dovecot, without login reachable from mailcow's own network only */
-    private const SERVICE_PORT = 588;
-    /** Domain of the Dovecot master login of mailcow's services (SOGo reminders) */
-    private const SERVICE_LOGIN_DOMAIN = 'mailcow.local';
-    /** HELO of the quota warning copies of Dovecot (quota_notify.py), configured by admins */
-    private const QUOTA_HELO = 'quotanotification';
-    /** Recipient without domain every server accepts (RFC 5321) */
-    private const POSTMASTER = 'postmaster';
-    /** Postfix delivers these destinations itself (mydestination), user%domain and domain!user route from there */
-    private const LOCAL_DESTINATIONS = ['localhost', 'localhost.localdomain'];
-
-    /** Lookups of the current check */
-    private array $cache = [];
-
-    public function __construct(private LookupRepository $lookup)
-    {
-    }
-
-    /** $login: SASL username, '' if not logged in; $port: smtpd port of the request, 0 if unknown; $helo: HELO name */
-    public function check(string $sender, string $recipient, string $login, int $port = 0, string $helo = ''): Decision
-    {
-        $this->cache = [];
-        $recipient = self::address($recipient);
-        if (in_array($port, self::INTERNAL_PORTS, true) || $recipient === '' || $recipient === self::POSTMASTER) {
-            return Decision::allow();
-        }
-        if (self::isRouted($recipient)) {
-            return Decision::routedAddress();
-        }
-        $recipient = $this->stored($recipient);
-        $sender = $this->stored(self::address($sender));
-        $login = self::address($login);
-        $serviceLogin = self::domainOf($login) === self::SERVICE_LOGIN_DOMAIN;
-        $account = null;
-        if ($login !== '' && !$serviceLogin) {
-            $account = $this->cached('mailbox', $login);
-        } elseif ($serviceLogin || $port === self::SERVICE_PORT) {
-            if ($port === self::SERVICE_PORT && !$serviceLogin && strtolower($helo) === self::QUOTA_HELO) {
-                return Decision::allow();
-            }
-            // without a mailbox sender the sending mailbox is unknown
-            $account = $this->senderMailbox($sender);
-            if ($account === null && !$this->cached('isLocalDomain', self::domainOf($recipient)) && $this->lookup->hasSendRules()) {
-                return Decision::deny('send');
-            }
-        }
-        foreach ($this->sendingPolicies($account, $sender) as $policy) {
-            if (!$this->allows($policy->sendMode, $policy->sendAllow, $policy->sendBlock, $this->isInternalRecipient($recipient, $policy->domain),
-                fn(string $entry): bool => $this->recipientMatches($entry, $recipient))) {
-                return Decision::deny('send', $policy);
-            }
-        }
-        foreach ($this->receivingPolicies($recipient) as $policy) {
-            if (!$this->allows($policy->receiveMode, $policy->receiveAllow, $policy->receiveBlock,
-                $account !== null && $account['domain'] === $policy->domain,
-                fn(string $entry, bool $block): bool => $this->senderMatches($entry, $sender, $account, $block))) {
-                return Decision::deny('receive', $policy);
-            }
-        }
-        return Decision::allow();
-    }
-
-    /** Blocked entries first; then the mode decides. $matches($entry, $block) */
-    private function allows(string $mode, array $allow, array $block, bool $internal, callable $matches): bool
-    {
-        foreach ($block as $entry) {
-            if ($matches($entry, true)) {
-                return false;
-            }
-        }
-        return match ($mode) {
-            'internal' => $internal || $this->any($allow, $matches),
-            'list' => $this->any($allow, $matches),
-            default => true,
-        };
-    }
-
-    private function any(array $entries, callable $matches): bool
-    {
-        foreach ($entries as $entry) {
-            if ($matches($entry, false)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /** @return array{username: string, domain: string}|null mailbox of the sender address */
-    private function senderMailbox(string $sender): ?array
-    {
-        return str_contains($sender, '@') ? $this->cached('mailbox', $this->resolve(self::untagged($sender))) : null;
-    }
-
-    /** @return Policy[] of the logged-in mailbox and of the mailbox it sends as */
-    private function sendingPolicies(?array $account, string $sender): array
-    {
-        if ($account === null) {
-            return [];
-        }
-        $policies = $this->lookup->policiesForMailbox($account['username']);
-        $sentAs = $this->senderMailbox($sender);
-        if ($sentAs !== null && $sentAs['username'] !== $account['username']) {
-            $policies = array_merge($policies, $this->lookup->policiesForMailbox($sentAs['username']));
-        }
-        return self::unique($policies);
-    }
-
-    /** @return Policy[] of what Postfix delivers the recipient to, see the class comment */
-    private function receivingPolicies(string $recipient): array
-    {
-        // the address, then without +tag; an alias domain maps to the mailbox in its target domain and Postfix
-        // looks up the result again with the +tag
-        [$address, $target] = $this->aliasTarget([$recipient, self::untagged($recipient)]);
-        $resolved = $this->resolve($recipient);
-        if ($target === null && $resolved !== $recipient) {
-            [$address, $target] = $this->aliasTarget([$resolved, self::untagged($resolved)]);
-        }
-        $address ??= self::untagged($resolved);
-        if (($target['kind'] ?? '') === 'temporary') {
-            $address = $target['owner'];
-        }
-        $policies = ($this->cached('mailbox', $address) !== null) ? $this->lookup->policiesForMailbox($address) : [];
-        $group = $this->lookup->groupByAddress($address);
-        if ($group !== null) {
-            $policies = array_merge($policies, $this->lookup->policiesForGroup($group));
-        }
-        // the `internal` flag of an alias or group: only senders of its domain
-        if (!empty($target['internal'])) {
-            $policies[] = $this->internalRule($target['domain']);
-        }
-        foreach (array_unique([self::domainOf($recipient), $this->resolveDomain(self::domainOf($recipient))]) as $domain) {
-            if ($this->lookup->internalCatchAll($domain)) {
-                $policies[] = $this->internalRule($domain);
-            }
-        }
-        return self::unique($policies);
-    }
-
-    /** @return array{0: ?string, 1: ?array} the first key Postfix' virtual alias maps find, with what they find */
-    private function aliasTarget(array $keys): array
-    {
-        foreach (array_unique($keys) as $key) {
-            $target = $this->lookup->aliasTarget($key);
-            if ($target !== null) {
-                return [$key, $target];
-            }
-        }
-        return [null, null];
-    }
-
-    /** Receive rule of the `internal` flag */
-    private function internalRule(string $domain): Policy
-    {
-        $policy = new Policy();
-        $policy->domain = $this->resolveDomain($domain);
-        $policy->receiveMode = 'internal';
-        return $policy;
-    }
-
-    private function isInternalRecipient(string $recipient, string $domain): bool
-    {
-        return $this->resolveDomain(self::domainOf($recipient)) === $domain;
-    }
-
-    private function recipientMatches(string $entry, string $recipient): bool
-    {
-        $address = $this->resolve(self::untagged($recipient));
-        if (str_starts_with($entry, 'group:')) {
-            $id = (int)substr($entry, 6);
-            return $this->cached('groupAddress', $id) === $address
-                || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address));
-        }
-        if (str_contains($entry, '@')) {
-            return $this->resolve($entry) === $address;
-        }
-        return $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry);
-    }
-
-    /** $block: a block entry, which also matches the envelope sender (refusing a forged sender is no risk) */
-    private function senderMatches(string $entry, string $sender, ?array $account, bool $block): bool
-    {
-        $address = $this->resolve(self::untagged($sender));
-        // the sender address of a local entry only counts when logged in: Postfix checked that the login may use it
-        $useAddress = $account !== null || $block;
-        if (str_starts_with($entry, 'group:')) {
-            $id = (int)substr($entry, 6);
-            return ($account !== null && $this->cached('isGroupMember', $id, $account['username']))
-                || ($useAddress && ($this->cached('groupAddress', $id) === $address
-                    || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address))));
-        }
-        if (str_contains($entry, '@')) {
-            return $this->cached('isLocalDomain', self::domainOf($entry))
-                ? ($account !== null && $account['username'] === $this->resolve($entry)) || ($useAddress && $address === $this->resolve($entry))
-                : self::untagged($sender) === $entry;
-        }
-        return $this->cached('isLocalDomain', $entry)
-            ? ($account !== null && $account['domain'] === $this->resolveDomain($entry))
-                || ($useAddress && $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry))
-            : self::domainOf($sender) === $entry;
-    }
-
-    /**
-     * Stored spelling of an address of this mailcow: Postfix finds chéf@ as chef@ (MySQL ignores accents). Only
-     * addresses with non-ASCII characters can differ, as they are already in lower case.
-     */
-    private function stored(string $address): string
-    {
-        if (!preg_match('/[\x80-\xff]/', $address) || !str_contains($address, '@')) {
-            return $address;
-        }
-        foreach (array_unique([$address, self::untagged($address)]) as $key) {
-            $stored = $this->cached('storedAddress', $key);
-            if ($stored !== null) {
-                return $stored;
-            }
-        }
-        $domain = $this->cached('storedDomain', self::domainOf($address));
-        return ($domain === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $domain;
-    }
-
-    /** Address in an alias domain as the address in its target domain */
-    private function resolve(string $address): string
-    {
-        $domain = self::domainOf($address);
-        $target = ($domain !== '') ? $this->cached('aliasDomainTarget', $domain) : null;
-        return ($target === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $target;
-    }
-
-    private function resolveDomain(string $domain): string
-    {
-        return ($domain === '') ? '' : ($this->cached('aliasDomainTarget', $domain) ?? $domain);
-    }
-
-    /** A lookup, done once per check: long lists ask the same domains and groups again */
-    private function cached(string $method, int|string ...$args): mixed
-    {
-        $key = $method . "\0" . implode("\0", $args);
-        if (!array_key_exists($key, $this->cache)) {
-            $this->cache[$key] = $this->lookup->$method(...$args);
-        }
-        return $this->cache[$key];
-    }
-
-    /** @param Policy[] $policies @return Policy[] without repeated policies */
-    private static function unique(array $policies): array
-    {
-        $unique = [];
-        foreach ($policies as $policy) {
-            $unique[$policy->id ?? 'internal:' . $policy->domain] = $policy;
-        }
-        return array_values($unique);
-    }
-
-    /** Lower case, without angle brackets, quotes and trailing dot of the domain, as Postfix resolves it; '' = null sender */
-    private static function address(string $value): string
-    {
-        $value = strtolower(trim($value, " \t\r\n<>"));
-        $at = strrpos($value, '@');
-        return ($at === false) ? $value : str_replace('"', '', substr($value, 0, $at)) . '@' . rtrim(substr($value, $at + 1), '.');
-    }
-
-    /**
-     * Postfix would deliver the recipient elsewhere: no domain, or routing through a local destination
-     * (localhost, the IP of the server) with %, ! or a second @ in the local part
-     */
-    private static function isRouted(string $address): bool
-    {
-        $at = strrpos($address, '@');
-        if ($at === false || $at === 0 || $at === strlen($address) - 1) {
-            return true;
-        }
-        $domain = substr($address, $at + 1);
-        return (in_array($domain, self::LOCAL_DESTINATIONS, true) || str_starts_with($domain, '['))
-            && strpbrk(substr($address, 0, $at), '@%!') !== false;
-    }
-
-    private static function untagged(string $address): string
-    {
-        $at = strrpos($address, '@');
-        $pos = ($at === false) ? false : strpos(substr($address, 0, $at), '+');
-        return ($pos === false || $pos === 0) ? $address : substr($address, 0, $pos) . substr($address, $at);
-    }
-
-    private static function domainOf(string $address): string
-    {
-        $at = strrpos($address, '@');
-        return ($at === false) ? '' : substr($address, $at + 1);
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyService.php

@@ -1,351 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\MailPolicy\Model\Decision;
-use Mailcow\MailPolicy\Model\Policy;
-use Mailcow\MailPolicy\Repository\PolicyRepository;
-
-/**
- * Use cases of mail policies.
- *
- * Read: admins all policies (also read-only API keys); domain admins with the `mail_policies` ACL the policies
- *       of their domains.
- * Edit: admins; domain admins with the ACL for policies of their domains. A policy is assigned to mailboxes and
- *       groups of its own domain; `group:<id>` entries name groups of that domain. Groups need the `mailbox_groups`
- *       ACL as well: without it a domain admin sees no groups and keeps the group entries and assignments.
- */
-class PolicyService
-{
-    private const MAX_ENTRIES = 500;
-    /** The reject text goes into the SMTP reply: printable ASCII */
-    private const MAX_REJECT_TEXT = 200;
-    private const LISTS = ['send_allow' => 'sendAllow', 'send_block' => 'sendBlock', 'receive_allow' => 'receiveAllow', 'receive_block' => 'receiveBlock'];
-
-    public function __construct(private PolicyRepository $repo, private PolicyEngine $engine, private Context $context)
-    {
-    }
-
-    public function canRead(): bool
-    {
-        return $this->context->hasAdminRights()
-            || ($this->context->isDomainAdmin() && $this->context->hasAcl('mail_policies'));
-    }
-
-    /** @throws AccessDenied unless the caller may edit policies at all */
-    public function requireEditor(): void
-    {
-        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
-    }
-
-    public function get(int $id): ?Policy
-    {
-        $policy = $this->canRead() ? $this->repo->find($id) : null;
-        return ($policy !== null && $this->visible($policy)) ? $policy : null;
-    }
-
-    /** @return int[] */
-    public function visibleIds(array $domains = []): array
-    {
-        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
-    }
-
-    public function canEdit(Policy $policy): bool
-    {
-        return $this->canRead() && !$this->context->isReadOnlyApi() && $this->visible($policy);
-    }
-
-    public function toArray(Policy $policy): array
-    {
-        return $policy->toArray() + ['can_edit' => (int)$this->canEdit($policy)];
-    }
-
-    /** @throws UserError */
-    public function create(array $input): Policy
-    {
-        $this->requireEditor();
-        $policy = new Policy();
-        $policy->domain = $this->domainFor($input['domain'] ?? null);
-        $this->apply($policy, $input);
-        $this->repo->save($policy);
-        return $policy;
-    }
-
-    /** Attributes missing in $input keep their value @throws UserError */
-    public function update(int $id, array $input): Policy
-    {
-        $policy = $this->editable($id);
-        $this->apply($policy, $input);
-        $this->repo->save($policy);
-        return $policy;
-    }
-
-    /** @throws UserError */
-    public function delete(int $id): Policy
-    {
-        $policy = $this->editable($id);
-        $this->repo->delete($id);
-        return $policy;
-    }
-
-    /** Address entries of the lists follow a renamed mailbox or alias; called after the rename, without permission check */
-    public function addressRenamed(string $old, string $new): void
-    {
-        $this->repo->renameAddress(strtolower($old), strtolower($new));
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupChoices(array $domains): array
-    {
-        return ($this->canRead() && $this->canUseGroups()) ? $this->repo->groupsIn($domains) : [];
-    }
-
-    private function canUseGroups(): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->hasAcl('mailbox_groups');
-    }
-
-    /**
-     * "Test a rule": would the mail be accepted? A sender that is a mailbox counts as logged in when $loggedIn.
-     *
-     * @return array{allowed: int, side: string, policy: ?string, internal_alias: int, text: string}
-     * @throws UserError
-     */
-    public function test(mixed $sender, mixed $recipient, bool $loggedIn): array
-    {
-        if (!$this->canRead()) {
-            throw new AccessDenied();
-        }
-        $sender = strtolower((string)Input::text($sender));
-        $recipient = strtolower((string)Input::text($recipient));
-        if (($sender !== '' && !filter_var($sender, FILTER_VALIDATE_EMAIL)) || !filter_var($recipient, FILTER_VALIDATE_EMAIL)) {
-            throw new UserError('mail_policy_test_invalid');
-        }
-        // domain admins test mail of their domains; a sender of another domain counts as not logged in
-        $ownSender = $this->context->hasAdminRights() || $this->context->canAccessDomain(self::domainOf($sender));
-        if (!$ownSender && !$this->context->canAccessDomain(self::domainOf($recipient))) {
-            throw new AccessDenied();
-        }
-        $login = ($loggedIn && $ownSender && $this->repo->mailboxDomain($sender) !== null) ? $sender : '';
-        $decision = $this->engine->check($sender, $recipient, $login);
-        $policy = $decision->policy;
-        $hidden = $policy !== null && $policy->id !== null && !$this->visible($policy);
-        return [
-            'allowed' => (int)$decision->allowed,
-            'side' => $decision->side,
-            'policy' => ($policy !== null && $policy->id !== null && !$hidden) ? $policy->name : null,
-            'internal_alias' => (int)($policy !== null && $policy->id === null),
-            'logged_in' => (int)($login !== ''),
-            // the text of a policy of another domain stays hidden
-            'text' => $decision->allowed ? '' : ($hidden ? Decision::defaultText($decision->side) : $decision->text()),
-        ];
-    }
-
-    private function visible(Policy $policy): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->canAccessDomain($policy->domain);
-    }
-
-    /** @throws UserError */
-    private function editable(int $id): Policy
-    {
-        $this->requireEditor();
-        $policy = $this->get($id);
-        if ($policy === null || !$this->canEdit($policy)) {
-            throw new AccessDenied();
-        }
-        return $policy;
-    }
-
-    /** @throws UserError */
-    private function domainFor(mixed $value): string
-    {
-        $domain = strtolower((string)Input::text($value));
-        if ($domain === '' || !$this->repo->isDomain($domain)) {
-            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
-        }
-        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        return $domain;
-    }
-
-    /** @throws UserError */
-    private function apply(Policy $policy, array $input): void
-    {
-        if (array_key_exists('name', $input) || $policy->id === null) {
-            $name = Input::text($input['name'] ?? null);
-            if ($name === null || $name === '' || mb_strlen($name) > 100 || preg_match('/[\x00-\x1f\x7f]/', $name)) {
-                throw new UserError('mail_policy_name_invalid');
-            }
-            if ($this->repo->nameExists($policy->domain, $name, $policy->id)) {
-                throw new UserError('object_exists', [htmlspecialchars($name)]);
-            }
-            $policy->name = $name;
-        }
-        if (array_key_exists('description', $input)) {
-            $description = Input::text($input['description']);
-            if ($description === null || mb_strlen($description) > 255 || preg_match('/[\x00-\x1f\x7f]/', $description)) {
-                throw new UserError('mail_policy_description_invalid');
-            }
-            $policy->description = $description;
-        }
-        foreach (['send_mode' => 'sendMode', 'receive_mode' => 'receiveMode'] as $key => $property) {
-            if (array_key_exists($key, $input)) {
-                $mode = Input::text($input[$key]);
-                $policy->$property = in_array($mode, Policy::MODES, true) ? $mode : throw new UserError('mail_policy_mode_invalid');
-            }
-        }
-        $groupAccess = $this->canUseGroups();
-        foreach (self::LISTS as $key => $property) {
-            // the UI sends the groups of a list separately (<list>_groups); the API may name them as group:<id>
-            if (!array_key_exists($key, $input) && !array_key_exists($key . '_groups', $input)) {
-                continue;
-            }
-            $entries = array_key_exists($key, $input) ? self::values($input[$key]) : self::withoutGroups($policy->$property);
-            $kept = [];
-            if (!$groupAccess) {
-                // without access to groups the group entries stay (also when sent back unchanged); the form shows no group choices
-                $kept = array_values(array_diff($policy->$property, self::withoutGroups($policy->$property)));
-                $entries = array_values(array_diff(array_map('strtolower', $entries), $kept));
-            } elseif (array_key_exists($key . '_groups', $input)) {
-                $entries = array_merge(self::withoutGroups($entries), array_map(fn(string $id): string => 'group:' . $id, self::values($input[$key . '_groups'])));
-            }
-            $entries = array_values(array_unique(array_merge($this->entries($entries, $policy->domain, $groupAccess), $kept)));
-            if (count($entries) > self::MAX_ENTRIES) {
-                throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
-            }
-            $policy->$property = $entries;
-        }
-        if (array_key_exists('reject_text', $input)) {
-            $text = Input::text($input['reject_text']);
-            if ($text === null || strlen($text) > self::MAX_REJECT_TEXT || preg_match('/[^\x20-\x7e]/', $text)) {
-                throw new UserError('mail_policy_reject_text_invalid');
-            }
-            $policy->rejectText = $text;
-        }
-        if (array_key_exists('active', $input)) {
-            $policy->active = (int)self::flag($input['active']);
-        }
-        if (array_key_exists('mailboxes', $input)) {
-            $policy->mailboxes = [];
-            foreach (self::limited($input['mailboxes']) as $username) {
-                $username = strtolower($username);
-                if ($this->repo->mailboxDomain($username) !== $policy->domain) {
-                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($username)]);
-                }
-                $policy->mailboxes[$username] = $username;
-            }
-            $policy->mailboxes = array_values($policy->mailboxes);
-        }
-        // without access to groups the assignments stay; only sending them back unchanged is accepted
-        if (array_key_exists('groups', $input) && !$groupAccess && self::values($input['groups']) !== []) {
-            $sent = array_map('intval', self::values($input['groups']));
-            $stored = $policy->groups;
-            sort($sent);
-            sort($stored);
-            if (array_values(array_unique($sent)) !== $stored) {
-                throw new AccessDenied();
-            }
-        }
-        if (array_key_exists('groups', $input) && $groupAccess) {
-            $policy->groups = [];
-            foreach (self::limited($input['groups']) as $value) {
-                $id = Input::intInRange($value, 1, PHP_INT_MAX);
-                if ($id === null || $this->repo->groupDomain($id) !== $policy->domain) {
-                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($value)]);
-                }
-                $policy->groups[$id] = $id;
-            }
-            $policy->groups = array_values($policy->groups);
-        }
-    }
-
-    /**
-     * Entries of a list: addresses, domains (a leading @ is dropped) and group:<id> of a group of the policy's
-     * domain, lower case, without repeats.
-     *
-     * @param string[] $values
-     * @return string[]
-     * @throws UserError
-     */
-    private function entries(array $values, string $domain, bool $groupAccess): array
-    {
-        $entries = [];
-        foreach (self::limited(array_map('strtolower', $values)) as $entry) {
-            if (str_starts_with($entry, 'group:')) {
-                $id = Input::intInRange(substr($entry, 6), 1, PHP_INT_MAX);
-                if (!$groupAccess || $id === null || $this->repo->groupDomain($id) !== $domain) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-                $entry = 'group:' . $id;
-            } elseif (str_contains(ltrim($entry, '@'), '@')) {
-                if (!filter_var($entry, FILTER_VALIDATE_EMAIL)) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-            } else {
-                $entry = ltrim($entry, '@');
-                if (!str_contains($entry, '.') || !filter_var($entry, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME)) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-            }
-            $entries[$entry] = $entry;
-        }
-        return array_values($entries);
-    }
-
-    /**
-     * Values of a list field without repeats, at most MAX_ENTRIES (checked before anything is looked up)
-     *
-     * @return string[]
-     * @throws UserError
-     */
-    private static function limited(mixed $value): array
-    {
-        $values = array_values(array_unique(self::values($value)));
-        if (count($values) > self::MAX_ENTRIES) {
-            throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
-        }
-        return $values;
-    }
-
-    /** @return string[] entries without group:<id> */
-    private static function withoutGroups(array $entries): array
-    {
-        return array_values(array_filter($entries, fn(string $entry): bool => !str_starts_with(strtolower($entry), 'group:')));
-    }
-
-    /**
-     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
-     *
-     * @return string[]
-     */
-    private static function values(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
-        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
-    }
-
-    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
-    private static function flag(mixed $value): bool
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
-    }
-
-    private static function domainOf(string $address): string
-    {
-        $at = strrpos($address, '@');
-        return ($at === false) ? '' : substr($address, $at + 1);
-    }
-}

data/web/inc/lib/mailcow/MailboxGroup/Model/Group.php

@@ -42,8 +42,6 @@ class Group
     public array $sendAs = [];
     /** @var array[] {owner, rights, subscribe, state, last_error}: members may open these mailboxes (ACL group=mcg_<id>) */
     public array $imapGrants = [];
-    /** @var string[] owners of removed IMAP grants whose ACL Dovecot still has, not part of the grants to write */
-    public array $removingImapOwners = [];
     /** @var array[] {owner, type, folder, access, subscribe, state, last_error}: SOGo folders shared with the members */
     public array $sogoGrants = [];
     /** @var array[] warnings of the last write (msg values), not stored */

data/web/inc/lib/mailcow/MailboxGroup/Repository/GroupRepository.php

@@ -501,8 +501,6 @@ class GroupRepository extends Repository
         $group->imapGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
             $this->query("SELECT `owner`, `rights`, `subscribe`, `state`, `last_error` FROM `mailbox_group_imap_grant`
               WHERE `group_id` = :id AND `state` != 'removing' ORDER BY `owner`", $id)->fetchAll(PDO::FETCH_ASSOC));
-        $group->removingImapOwners = $this->query("SELECT `owner` FROM `mailbox_group_imap_grant` WHERE `group_id` = :id AND `state` = 'removing'",
-            $id)->fetchAll(PDO::FETCH_COLUMN);
         $group->sogoGrants = array_map(fn(array $grant): array => array_merge($grant, ['subscribe' => (int)$grant['subscribe']]),
             $this->query("SELECT `owner`, `folder_type` AS `type`, `folder`, `access`, `subscribe`, `state`, `last_error`
               FROM `mailbox_group_sogo_grant` WHERE `group_id` = :id ORDER BY `owner`, `folder_type`, `folder`", $id)->fetchAll(PDO::FETCH_ASSOC));

data/web/inc/lib/mailcow/MailboxGroup/Service/GroupService.php

@@ -79,12 +79,6 @@ class GroupService
                 return false;
             }
         }
-        // A removed grant still opens the mailbox until Dovecot dropped its ACL
-        foreach ($group->removingImapOwners as $owner) {
-            if (!$this->context->canAccessMailbox((string)$owner)) {
-                return false;
-            }
-        }
         return true;
     }
 
@@ -119,7 +113,7 @@ class GroupService
         $current = $this->editable($id);
         $group = clone $current;
         $this->apply($group, $input, $current);
-        $snapshot = $this->effects->snapshot($this->inScope(array_merge($current->usernames(), $group->usernames())));
+        $snapshot = $this->effects->snapshot(array_merge($current->usernames(), $group->usernames()));
         $sogo = $this->repo->update($group);
         $group->warnings = array_merge($this->sizeWarnings($group), $this->effects->apply($sogo, $snapshot),
             $this->sharing->applyImap((int)$group->id), $this->sharing->applySogo((int)$group->id));
@@ -138,7 +132,7 @@ class GroupService
     {
         $this->requireEditor();
         $group = $this->editable($id);
-        $snapshot = $this->effects->snapshot($this->inScope($group->usernames()));
+        $snapshot = $this->effects->snapshot($group->usernames());
         $this->sharing->removeImap($group);
         $sogo = $this->repo->delete($id);
         $group->warnings = $this->effects->apply($sogo, $snapshot);
@@ -156,30 +150,13 @@ class GroupService
         if (!$own && !($this->canRead() && $this->context->canAccessMailbox($username))) {
             return [];
         }
-        $groups = $this->lookup->membershipsOf($username);
-        // Domain admins see the mailbox's groups in their domains, not those of other domains
-        if (!$own && !$this->context->hasAdminRights()) {
-            $groups = array_values(array_filter($groups, fn(array $group): bool => $this->context->canAccessDomain((string)$group['domain'])));
-        }
         return [
-            'groups' => $groups,
+            'groups' => $this->lookup->membershipsOf($username),
             'template' => $this->templates->effective([$username])[$username] ?? null,
             'send_as' => $this->lookup->sendAsOf($username),
         ];
     }
 
-    /**
-     * Members whose templates a change of the caller may apply: a domain admin never changes mailboxes outside
-     * his domains, their new effective template applies with the next admin action or IdP login.
-     */
-    private function inScope(array $usernames): array
-    {
-        if ($this->context->hasAdminRights()) {
-            return $usernames;
-        }
-        return array_values(array_filter($usernames, fn(string $username): bool => $this->context->canAccessMailbox($username)));
-    }
-
     /** @throws AccessDenied when the caller cannot see or edit the group */
     private function editable(int $id): Group
     {

data/web/inc/lib/mailcow/Retention/Controller/RetentionApiController.php

@@ -1,168 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention\Controller;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\Retention\Service\Planner;
-use Mailcow\Retention\Service\RuleService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php, edit.php and user.php (routes `retention-rule`, `logs/retention`), and the
- * plan and results of the nightly run (data/conf/rspamd/dynmaps/retention.php). Messages are logged as
- * ['retention', action, 'rule', data].
- */
-class RetentionApiController
-{
-    public function __construct(private RuleService $rules, private ApiResponse $response)
-    {
-    }
-
-    /** POST add/retention-rule */
-    public function addRule(array $attr): bool
-    {
-        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
-            $rule = $this->rules->create($attr);
-            return $this->response->success(['retention_rule_added', htmlspecialchars($rule->name)], $log);
-        });
-    }
-
-    /** POST edit/retention-rule: one message per id */
-    public function editRules(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
-            $this->rules->requireEditor();
-            return $this->batch((array)($data['id'] ?? []), $log,
-                fn(int $id): array => ['retention_rule_modified', htmlspecialchars($this->rules->update($id, $attr)->name)]);
-        });
-    }
-
-    /** POST delete/retention-rule: one message per id */
-    public function deleteRules(array $items): bool
-    {
-        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
-            $this->rules->requireEditor();
-            return $this->batch($items, $log, fn(int $id): array => ['retention_rule_removed', htmlspecialchars($this->rules->delete($id)->name)]);
-        });
-    }
-
-    /** GET retention-rule/all[/<domain,domain>]: with the summary of the last nightly run */
-    public function getRules(?string $domains = null): array
-    {
-        $ids = $this->rules->visibleIds(self::domains($domains));
-        $runs = $this->rules->lastRuns($ids);
-        $rules = [];
-        foreach ($ids as $id) {
-            $rule = $this->rules->get($id);
-            if ($rule !== null) {
-                $rules[] = $this->rules->toArray($rule, $runs[$id] ?? null);
-            }
-        }
-        return $rules;
-    }
-
-    /** GET retention-rule/<id>, also edit.php */
-    public function getRule(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $rule = ($id === null) ? null : $this->rules->get($id);
-        return ($rule === null) ? [] : $this->rules->toArray($rule);
-    }
-
-    /**
-     * GET retention-rule/effective/<username>: what the rules delete in the mailbox; [] when not allowed to ask.
-     * $describe: for people (user.php), see Planner::describe()
-     */
-    public function getEffective(mixed $username, bool $describe = false): array
-    {
-        try {
-            $actions = $this->rules->effective($username);
-            return $describe ? Planner::describe($actions) : $actions;
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /** GET logs/retention[/<count>]: mailboxes with found mail or errors in the nightly runs, newest first */
-    public function getLog(int $limit = 0): array
-    {
-        return $this->rules->log($limit);
-    }
-
-    /** A page of the plan of the nightly run (retention.php): mailboxes with their actions, next: after for the next page */
-    public function plan(string $after = '', int $limit = 0): array
-    {
-        $page = $this->rules->plan($after, $limit);
-        $plan = [];
-        foreach ($page['mailboxes'] as $username => $actions) {
-            $plan[] = ['username' => (string)$username, 'actions' => $actions];
-        }
-        return ['mailboxes' => $plan, 'next' => $page['next']];
-    }
-
-    /** Results of the nightly run (retention.php): mailboxes with found mail or errors, a summary per rule, the run */
-    public function report(array $results, array $summary, ?array $run = null): int
-    {
-        return $this->rules->report($results, $summary, $run);
-    }
-
-    /** The domain was deleted (functions.mailbox.inc.php) */
-    public function domainDeleted(string $domain): void
-    {
-        $this->rules->domainDeleted($domain);
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, mixed $data, callable $work): bool
-    {
-        $log = ['retention', $action, 'rule', $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** One message per id: the success message returned by $work(id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id)), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** @return string[] domains of a comma separated filter */
-    private static function domains(?string $domains): array
-    {
-        return array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/Retention/Model/Rule.php

@@ -1,96 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention\Model;
-
-/**
- * A retention rule (table `retention_rule`): mail is deleted from its folders once it was saved there `days` days
- * ago. Folders are special folders with all their names (trash: Trash, Papierkorb, Deleted Items, ...), the INBOX,
- * all folders, or own folder names (`folder:<name>`, wildcard *). A rule applies to all mailboxes of its domain or
- * to mailboxes and groups of it. In mode `simulate` the nightly run only counts what it would delete.
- */
-class Rule
-{
-    /** Special folders, the INBOX and all folders; own folders are folder:<name> */
-    public const FOLDERS = ['trash', 'junk', 'sent', 'drafts', 'archive', 'inbox', 'all'];
-    public const MODES = ['simulate', 'delete'];
-    public const OWN_FOLDER = 'folder:';
-
-    public ?int $id = null;
-    public string $domain = '';
-    public string $name = '';
-    public string $description = '';
-    /** @var string[] keys of FOLDERS and folder:<name> */
-    public array $folders = [];
-    public int $days = 30;
-    /** Flagged mail stays */
-    public int $keepFlagged = 0;
-    public string $mode = 'simulate';
-    /** Applies to all mailboxes of the domain */
-    public int $allMailboxes = 0;
-    public int $active = 1;
-    /** @var string[] usernames the rule is assigned to */
-    public array $mailboxes = [];
-    /** @var int[] groups the rule is assigned to (their members of the rule's domain) */
-    public array $groups = [];
-    public ?string $created = null;
-    public ?string $modified = null;
-
-    /** Row of `retention_rule`; assignments are set by the repository */
-    public static function fromRow(array $row): self
-    {
-        $rule = new self();
-        $rule->id = (int)$row['id'];
-        $rule->domain = (string)$row['domain'];
-        $rule->name = (string)$row['name'];
-        $rule->description = (string)($row['description'] ?? '');
-        $folders = is_string($row['folders'] ?? null) ? json_decode($row['folders'], true) : null;
-        $rule->folders = is_array($folders) ? array_values(array_filter($folders, 'is_string')) : [];
-        $rule->days = (int)$row['days'];
-        $rule->keepFlagged = (int)$row['keep_flagged'];
-        $rule->mode = (string)$row['mode'];
-        $rule->allMailboxes = (int)$row['all_mailboxes'];
-        $rule->active = (int)$row['active'];
-        $rule->created = $row['created'] ?? null;
-        $rule->modified = $row['modified'] ?? null;
-        return $rule;
-    }
-
-    /** Column values for `retention_rule` */
-    public function toRow(): array
-    {
-        return [
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'folders' => json_encode($this->folders),
-            'days' => $this->days,
-            'keep_flagged' => $this->keepFlagged,
-            'mode' => $this->mode,
-            'all_mailboxes' => $this->allMailboxes,
-            'active' => $this->active,
-        ];
-    }
-
-    /** Entry of the API and the edit page */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'folders' => $this->folders,
-            'days' => $this->days,
-            'keep_flagged' => $this->keepFlagged,
-            'mode' => $this->mode,
-            'all_mailboxes' => $this->allMailboxes,
-            'active' => $this->active,
-            'mailboxes' => $this->mailboxes,
-            'groups' => $this->groups,
-            'created' => $this->created,
-            'modified' => $this->modified,
-        ];
-    }
-}

data/web/inc/lib/mailcow/Retention/Repository/RuleRepository.php

@@ -1,351 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention\Repository;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\Retention\Model\Rule;
-use Mailcow\Retention\Service\Planner;
-use PDO;
-
-/**
- * Tables `retention_rule`, `retention_rule_member` (the mailboxes and groups a rule is assigned to) and
- * `retention_log` (results of the nightly run), plus the lookups the editing needs. Assignments go with their
- * mailbox, group or domain (foreign keys, ON DELETE CASCADE).
- */
-class RuleRepository extends Repository
-{
-    /** Results of the nightly run are kept this long */
-    private const LOG_DAYS = 90;
-
-    public function find(int $id): ?Rule
-    {
-        $row = $this->query("SELECT * FROM `retention_rule` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? $this->withMembers(Rule::fromRow($row)) : null;
-    }
-
-    /**
-     * @param string[] $domains limit to these domains, [] = all visible
-     * @return int[] ids of the rules visible to the caller, ordered by domain and name
-     */
-    public function visibleIds(Context $context, array $domains = []): array
-    {
-        [$where, $params] = $this->visibleDomains($context, $domains);
-        if ($where === null) {
-            return [];
-        }
-        return array_map('intval', $this->query("SELECT `id` FROM `retention_rule` WHERE $where ORDER BY `domain`, `name`", $params)
-            ->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `retention_rule` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
-            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
-    }
-
-    /** Inserts or updates the rule with its assignments; returns its id */
-    public function save(Rule $rule): int
-    {
-        return $this->transaction(function () use ($rule): int {
-            if ($rule->id === null) {
-                $rule->id = $this->insertRow('retention_rule', $rule->toRow());
-            } else {
-                $this->updateRow('retention_rule', $rule->id, $rule->toRow());
-                $this->query("DELETE FROM `retention_rule_member` WHERE `rule_id` = :id", ['id' => $rule->id]);
-            }
-            foreach ($rule->mailboxes as $username) {
-                $this->insertRow('retention_rule_member', ['rule_id' => $rule->id, 'username' => $username]);
-            }
-            foreach ($rule->groups as $groupId) {
-                $this->insertRow('retention_rule_member', ['rule_id' => $rule->id, 'group_id' => $groupId]);
-            }
-            return $rule->id;
-        });
-    }
-
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `retention_rule` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /** A primary domain (rules never belong to alias domains) */
-    public function isDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** Domain of a regular mailbox, null if there is none */
-    public function mailboxDomain(string $username): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = ''", ['username' => $username])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    public function groupDomain(int $id): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox_group` WHERE `id` = :id", ['id' => $id])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupsIn(array $domains): array
-    {
-        if (empty($domains)) {
-            return [];
-        }
-        [$in, $params] = self::in('dom', $domains);
-        return $this->query("SELECT `id`, `name`, `domain`, `address` FROM `mailbox_group` WHERE `domain` IN ($in) ORDER BY `domain`, `name`", $params)
-            ->fetchAll();
-    }
-
-    /**
-     * A page of the mailboxes the nightly run looks at: Dovecot knows them (regular, not disabled), their domain is
-     * active and has active rules.
-     *
-     * @return string[] usernames after $after, in order
-     */
-    public function mailboxesWithRules(string $after, int $limit): array
-    {
-        return $this->query("SELECT `m`.`username` FROM `mailbox` `m` JOIN `domain` `d` ON `d`.`domain` = `m`.`domain` AND `d`.`active` = '1'
-          WHERE `m`.`kind` = '' AND `m`.`active` IN ('1', '2') AND `m`.`username` > :after
-            AND EXISTS (SELECT 1 FROM `retention_rule` `r` WHERE `r`.`domain` = `m`.`domain` AND `r`.`active` = '1')
-          ORDER BY `m`.`username` LIMIT " . max(1, $limit), ['after' => $after])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /**
-     * Active rules of these mailboxes with the level of their assignment: the whole domain, a group of the mailbox
-     * (members of the rule's domain only) or the mailbox itself. Only mailboxes Dovecot knows (regular, not
-     * disabled) in active domains.
-     *
-     * @param string[] $usernames
-     * @return array<string, array<array{rule: Rule, level: int}>> username => candidates
-     */
-    public function candidates(array $usernames): array
-    {
-        if (empty($usernames)) {
-            return [];
-        }
-        $params = [];
-        $mailbox = [];
-        foreach ([1, 2, 3] as $part) {
-            [$in, $inParams] = self::in('u' . $part . '_', $usernames);
-            $params += $inParams;
-            $mailbox[$part] = "`m`.`kind` = '' AND `m`.`active` IN ('1', '2') AND `m`.`username` IN ($in)
-              AND `m`.`domain` IN (SELECT `domain` FROM `domain` WHERE `active` = '1')";
-        }
-        $rows = $this->query("SELECT `r`.*, `m`.`username` AS `member`, " . Planner::LEVEL_DOMAIN . " AS `level` FROM `retention_rule` `r`
-            JOIN `mailbox` `m` ON `m`.`domain` = `r`.`domain` AND {$mailbox[1]}
-            WHERE `r`.`active` = '1' AND `r`.`all_mailboxes` = '1'
-          UNION ALL SELECT `r`.*, `m`.`username`, " . Planner::LEVEL_GROUP . " FROM `retention_rule` `r`
-            JOIN `retention_rule_member` `rm` ON `rm`.`rule_id` = `r`.`id`
-            JOIN `mailbox_group` `g` ON `g`.`id` = `rm`.`group_id` AND `g`.`active` = '1'
-            JOIN `mailbox_group_member` `gm` ON `gm`.`group_id` = `g`.`id`
-            JOIN `mailbox` `m` ON `m`.`username` = `gm`.`username` AND `m`.`domain` = `r`.`domain` AND {$mailbox[2]}
-            WHERE `r`.`active` = '1'
-          UNION ALL SELECT `r`.*, `m`.`username`, " . Planner::LEVEL_MAILBOX . " FROM `retention_rule` `r`
-            JOIN `retention_rule_member` `rm` ON `rm`.`rule_id` = `r`.`id`
-            JOIN `mailbox` `m` ON `m`.`username` = `rm`.`username` AND `m`.`domain` = `r`.`domain` AND {$mailbox[3]}
-            WHERE `r`.`active` = '1'
-          ORDER BY `member`, `id`", $params)->fetchAll();
-        $candidates = [];
-        foreach ($rows as $row) {
-            $candidates[strtolower((string)$row['member'])][] = ['rule' => Rule::fromRow($row), 'level' => (int)$row['level']];
-        }
-        return $candidates;
-    }
-
-    /**
-     * Stores results of the nightly run. Domain and rule name come from the database, not from the runner; results
-     * of a rule from another domain than the mailbox are dropped. With the summary of a rule, the results its
-     * earlier runs found while simulating are dropped: only the last simulation is kept. After a run without error,
-     * the active rules that decided about no folder get an empty summary.
-     *
-     * @param array[] $results username, rule_id, folder (names), days, matched, deleted, simulated, error: mailboxes
-     *                         with found mail or an error
-     * @param array[] $summary rule_id, mailboxes, matched, deleted, errors, simulated: one per rule and run
-     * @param array|null $run  started (unix time), mailboxes, matched, deleted, errors, error: the whole run, sent at its end
-     */
-    public function addLog(array $results, array $summary, ?array $run = null): int
-    {
-        $stored = $this->transaction(function () use ($results, $summary, $run): int {
-            $stored = 0;
-            foreach ($results as $result) {
-                $domain = $this->mailboxDomain((string)$result['username']);
-                $rule = $this->ruleName((int)$result['rule_id']);
-                if ($domain === null || $rule === null || $rule['domain'] !== $domain) {
-                    continue;
-                }
-                $this->insertRow('retention_log', [
-                    'domain' => $domain,
-                    'username' => (string)$result['username'],
-                    'rule_id' => (int)$result['rule_id'],
-                    'rule_name' => $rule['name'],
-                    'folder' => mb_substr((string)$result['folder'], 0, 10000),
-                    'days' => (int)($result['days'] ?? 0),
-                    'matched' => (int)($result['matched'] ?? 0),
-                    'deleted' => (int)($result['deleted'] ?? 0),
-                    'errors' => (int)(($result['error'] ?? '') !== ''),
-                    'simulated' => (int)(bool)($result['simulated'] ?? false),
-                    'error' => ($result['error'] ?? '') === '' ? null : mb_substr((string)$result['error'], 0, 1000),
-                ]);
-                $stored++;
-            }
-            $summarized = [];
-            foreach ($summary as $runOfRule) {
-                $rule = $this->ruleName((int)$runOfRule['rule_id']);
-                if ($rule === null || isset($summarized[(int)$runOfRule['rule_id']])) {
-                    continue;
-                }
-                $summarized[(int)$runOfRule['rule_id']] = true;
-                $this->addSummary((int)$runOfRule['rule_id'], $rule, $runOfRule);
-                $stored++;
-            }
-            if ($run !== null && ($run['error'] ?? '') === '') {
-                // rules created during the run may have missed its first pages
-                foreach ($this->query("SELECT `r`.`id`, `r`.`domain`, `r`.`name`, `r`.`mode` FROM `retention_rule` `r`
-                  JOIN `domain` `d` ON `d`.`domain` = `r`.`domain` AND `d`.`active` = '1'
-                  WHERE `r`.`active` = '1' AND `r`.`created` < FROM_UNIXTIME(:started)", ['started' => (int)($run['started'] ?? 0)])->fetchAll() as $row) {
-                    if (!isset($summarized[(int)$row['id']])) {
-                        $this->addSummary((int)$row['id'], ['domain' => (string)$row['domain'], 'name' => (string)$row['name']],
-                            ['simulated' => $row['mode'] !== 'delete']);
-                        $stored++;
-                    }
-                }
-            }
-            if ($run !== null) {
-                $this->insertRow('retention_log', [
-                    'domain' => '',
-                    'mailboxes' => (int)($run['mailboxes'] ?? 0),
-                    'matched' => (int)($run['matched'] ?? 0),
-                    'deleted' => (int)($run['deleted'] ?? 0),
-                    'errors' => (int)($run['errors'] ?? 0),
-                    'error' => ($run['error'] ?? '') === '' ? null : mb_substr((string)$run['error'], 0, 1000),
-                ]);
-                $stored++;
-            }
-            return $stored;
-        });
-        if ($run !== null) {
-            $this->prune();
-        }
-        return $stored;
-    }
-
-    /** Summary of a rule's run; drops what its earlier runs found while simulating */
-    private function addSummary(int $ruleId, array $rule, array $values): void
-    {
-        $previous = $this->query("SELECT MAX(`id`) FROM `retention_log` WHERE `rule_id` = :id AND `username` = ''", ['id' => $ruleId])->fetchColumn();
-        if ($previous !== null && $previous !== false) {
-            $this->query("DELETE FROM `retention_log` WHERE `rule_id` = :id AND `username` != '' AND `simulated` = '1' AND `id` < :previous",
-                ['id' => $ruleId, 'previous' => (int)$previous]);
-        }
-        $this->insertRow('retention_log', [
-            'domain' => $rule['domain'],
-            'rule_id' => $ruleId,
-            'rule_name' => $rule['name'],
-            'mailboxes' => (int)($values['mailboxes'] ?? 0),
-            'matched' => (int)($values['matched'] ?? 0),
-            'deleted' => (int)($values['deleted'] ?? 0),
-            'errors' => (int)($values['errors'] ?? 0),
-            'simulated' => (int)(bool)($values['simulated'] ?? false),
-        ]);
-    }
-
-    /** Drops the results of the runs in a domain (the domain was deleted) */
-    public function deleteLog(string $domain): void
-    {
-        $this->query("DELETE FROM `retention_log` WHERE `domain` = :domain", ['domain' => $domain]);
-    }
-
-    /** @return array<int, array> rule id => the summary of its last run (created, mailboxes, matched, deleted, errors, simulated) */
-    public function lastRuns(array $ruleIds): array
-    {
-        if (empty($ruleIds)) {
-            return [];
-        }
-        [$in, $params] = self::in('rule', array_map('intval', $ruleIds));
-        $runs = [];
-        foreach ($this->query("SELECT `l`.`rule_id`, `l`.`created`, `l`.`mailboxes`, `l`.`matched`, `l`.`deleted`, `l`.`errors`, `l`.`simulated`
-          FROM `retention_log` `l` WHERE `l`.`username` = '' AND `l`.`rule_id` IN ($in) AND `l`.`id` = (SELECT MAX(`id`) FROM `retention_log`
-            WHERE `rule_id` = `l`.`rule_id` AND `username` = '')", $params)->fetchAll() as $row) {
-            $runs[(int)$row['rule_id']] = $row;
-        }
-        return $runs;
-    }
-
-    /**
-     * @return array[] mailboxes with found mail or errors in the nightly runs visible to the caller, and for admins the
-     *                 whole runs (username '', rule_id null), newest first
-     */
-    public function log(Context $context, array $domains, int $limit): array
-    {
-        [$where, $params] = $this->visibleDomains($context, $domains);
-        if ($where === null) {
-            return [];
-        }
-        return $this->query("SELECT UNIX_TIMESTAMP(`created`) AS `time`, `domain`, `username`, `rule_id`, `rule_name`, `folder`, `days`, `mailboxes`, `matched`, `deleted`, `errors`, `simulated`, `error`
-          FROM `retention_log` WHERE (`username` != '' OR `rule_id` IS NULL) AND $where ORDER BY `created` DESC, `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll();
-    }
-
-    /** Drops results older than LOG_DAYS, in parts so a large log does not hold the table */
-    private function prune(): void
-    {
-        do {
-            $deleted = $this->query("DELETE FROM `retention_log` WHERE `created` < NOW() - INTERVAL " . self::LOG_DAYS . " DAY LIMIT 10000")->rowCount();
-        } while ($deleted >= 10000);
-    }
-
-    /** @return array{domain: string, name: string}|null */
-    private function ruleName(int $id): ?array
-    {
-        $row = $this->query("SELECT `domain`, `name` FROM `retention_rule` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? ['domain' => (string)$row['domain'], 'name' => (string)$row['name']] : null;
-    }
-
-    /** @return array{0: ?string, 1: array} condition on `domain` for the caller and the domain filter, null = nothing visible */
-    private function visibleDomains(Context $context, array $domains): array
-    {
-        if ($context->hasAdminRights()) {
-            [$where, $params] = ['1=1', []];
-        } elseif ($context->isDomainAdmin()) {
-            [$where, $params] = ["`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
-                ['vis_da' => $context->username]];
-        } else {
-            return [null, []];
-        }
-        if (!empty($domains)) {
-            [$in, $inParams] = self::in('dom', $domains);
-            $where .= " AND `domain` IN ($in)";
-            $params += $inParams;
-        }
-        return [$where, $params];
-    }
-
-    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
-    private static function in(string $prefix, array $values): array
-    {
-        $names = [];
-        $params = [];
-        foreach (array_values($values) as $i => $value) {
-            $names[] = ':' . $prefix . $i;
-            $params[$prefix . $i] = $value;
-        }
-        return [implode(', ', $names), $params];
-    }
-
-    private function withMembers(Rule $rule): Rule
-    {
-        foreach ($this->query("SELECT `username`, `group_id` FROM `retention_rule_member` WHERE `rule_id` = :id ORDER BY `username`, `group_id`",
-            ['id' => $rule->id])->fetchAll() as $row) {
-            if ($row['username'] !== null) {
-                $rule->mailboxes[] = strtolower((string)$row['username']);
-            } elseif ($row['group_id'] !== null) {
-                $rule->groups[] = (int)$row['group_id'];
-            }
-        }
-        return $rule;
-    }
-}

data/web/inc/lib/mailcow/Retention/RetentionModule.php

@@ -1,40 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Retention\Controller\RetentionApiController;
-use Mailcow\Retention\Repository\RuleRepository;
-use Mailcow\Retention\Service\RuleService;
-use PDO;
-
-/**
- * Composition root of the retention rules: which mail is deleted from which folders after how many days. The
- * nightly run in the dovecot container (retention_runner.py) gets its plan from data/conf/rspamd/dynmaps/retention.php.
- *
- *   process_get_return(RetentionModule::fromGlobals()->api->getRules());
- *   echo json_encode(RetentionModule::create($pdo, Context::system())->api->plan());  // nightly run
- */
-class RetentionModule
-{
-    public function __construct(
-        public readonly RuleService $rules,
-        public readonly RetentionApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context): self
-    {
-        $rules = new RuleService(new RuleRepository($pdo), $context);
-        return new self($rules, new RetentionApiController($rules, new ApiResponse()));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
-    }
-}

data/web/inc/lib/mailcow/Retention/Service/Planner.php

@@ -1,112 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention\Service;
-
-use Mailcow\Retention\Model\Rule;
-
-/**
- * What the rules of one mailbox delete: the most specific rule wins, per folder of the mailbox. The folders of the
- * rules become entries in order: a rule assigned to the mailbox before one of its groups before one of the whole
- * domain; then by what an entry matches (MATCHES, the folder itself first); then the longer period, then the lower
- * id. The runner gives every folder to the first entry that matches it.
- */
-class Planner
-{
-    public const LEVEL_DOMAIN = 1;
-    public const LEVEL_GROUP = 2;
-    public const LEVEL_MAILBOX = 3;
-
-    /**
-     * What an entry matches, most specific first: the folder itself (a special folder, the inbox, an own folder
-     * without *), an own folder name with *, the subfolders of a special folder or the inbox, all folders
-     */
-    public const MATCHES = ['folder' => 3, 'pattern' => 2, 'subfolders' => 1, 'all' => 0];
-
-    /**
-     * @param array<array{rule: Rule, level: int}> $candidates active rules of one mailbox with their assignment level
-     * @return array[] in order: folder (key), match, days, keep_flagged, mode, rule_id, rule_name. Entries that never
-     *                 win are left out: one matching the same folders as an entry before and everything after all folders.
-     */
-    public static function actions(array $candidates): array
-    {
-        $entries = [];
-        foreach ($candidates as $candidate) {
-            foreach ($candidate['rule']->folders as $folder) {
-                foreach (self::matches($folder) as $match) {
-                    // the lower id decides between equal rules, so the result does not depend on the order
-                    $entries[] = [
-                        'score' => [$candidate['level'], self::MATCHES[$match], $candidate['rule']->days, -(int)$candidate['rule']->id],
-                        'folder' => $folder,
-                        'match' => $match,
-                        'rule' => $candidate['rule'],
-                    ];
-                }
-            }
-        }
-        usort($entries, fn(array $a, array $b): int => $b['score'] <=> $a['score']);
-        $actions = [];
-        foreach ($entries as $entry) {
-            if (isset($actions[$entry['folder'] . "\n" . $entry['match']])) {
-                continue;
-            }
-            $actions[$entry['folder'] . "\n" . $entry['match']] = self::action($entry['folder'], $entry['match'], $entry['rule']);
-            if ($entry['match'] === 'all') {
-                break;
-            }
-        }
-        return array_values($actions);
-    }
-
-    /**
-     * The plan for people: the subfolders of a special folder or the inbox go with the folder (subfolders 1) when the
-     * same rule decides both and no entry with * comes between them.
-     *
-     * @return array[] the entries of actions() with subfolders 0 or 1
-     */
-    public static function describe(array $actions): array
-    {
-        $described = [];
-        $folders = [];
-        foreach ($actions as $action) {
-            $key = $action['folder'] . "\n" . $action['rule_id'];
-            if ($action['match'] === 'subfolders' && isset($folders[$key])) {
-                $described[$folders[$key]]['subfolders'] = 1;
-                continue;
-            }
-            if ($action['match'] === 'pattern') {
-                $folders = [];
-            } elseif ($action['match'] === 'folder') {
-                $folders[$key] = count($described);
-            }
-            $described[] = $action + ['subfolders' => (int)($action['match'] === 'subfolders')];
-        }
-        return $described;
-    }
-
-    /** @return string[] what the folder key of a rule matches */
-    private static function matches(string $folder): array
-    {
-        if ($folder === 'all') {
-            return ['all'];
-        }
-        if (str_starts_with($folder, Rule::OWN_FOLDER)) {
-            return [str_contains($folder, '*') ? 'pattern' : 'folder'];
-        }
-        return ['folder', 'subfolders'];
-    }
-
-    private static function action(string $folder, string $match, Rule $rule): array
-    {
-        return [
-            'folder' => $folder,
-            'match' => $match,
-            'days' => $rule->days,
-            'keep_flagged' => $rule->keepFlagged,
-            'mode' => $rule->mode,
-            'rule_id' => $rule->id,
-            'rule_name' => $rule->name,
-        ];
-    }
-}

data/web/inc/lib/mailcow/Retention/Service/RuleService.php

@@ -1,359 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Retention\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\Retention\Model\Rule;
-use Mailcow\Retention\Repository\RuleRepository;
-
-/**
- * Use cases of retention rules.
- *
- * Read: admins all rules (also read-only API keys); domain admins with the `retention` ACL the rules of their
- *       domains. Mailbox users see the rules that apply to their mailbox (effective()).
- * Edit: admins; domain admins with the ACL for rules of their domains. A rule is assigned to all mailboxes of its
- *       domain or to mailboxes and groups of it. Groups need the `mailbox_groups` ACL as well: without it a domain
- *       admin sees no groups and keeps the group assignments.
- */
-class RuleService
-{
-    private const MAX_FOLDERS = 50;
-    private const MAX_MEMBERS = 500;
-    private const MAX_DAYS = 36500;
-    private const LOG_LIMIT = 1000;
-    /** Mailboxes per page of the plan, and results per report of the runner */
-    private const PLAN_PAGE = 500;
-    private const MAX_RESULTS = 1000;
-
-    public function __construct(private RuleRepository $repo, private Context $context)
-    {
-    }
-
-    public function canRead(): bool
-    {
-        return $this->context->hasAdminRights()
-            || ($this->context->isDomainAdmin() && $this->context->hasAcl('retention'));
-    }
-
-    /** @throws AccessDenied unless the caller may edit rules at all */
-    public function requireEditor(): void
-    {
-        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
-    }
-
-    public function get(int $id): ?Rule
-    {
-        $rule = $this->canRead() ? $this->repo->find($id) : null;
-        return ($rule !== null && $this->visible($rule)) ? $rule : null;
-    }
-
-    /** @return int[] */
-    public function visibleIds(array $domains = []): array
-    {
-        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
-    }
-
-    public function canEdit(Rule $rule): bool
-    {
-        return $this->canRead() && !$this->context->isReadOnlyApi() && $this->visible($rule);
-    }
-
-    /** $lastRun: summary of the last nightly run, null = not run yet */
-    public function toArray(Rule $rule, ?array $lastRun = null): array
-    {
-        return $rule->toArray() + ['can_edit' => (int)$this->canEdit($rule), 'last_run' => $lastRun];
-    }
-
-    /** @return array<int, array> rule id => summary of its last nightly run */
-    public function lastRuns(array $ruleIds): array
-    {
-        return $this->canRead() ? $this->repo->lastRuns($ruleIds) : [];
-    }
-
-    /** @throws UserError */
-    public function create(array $input): Rule
-    {
-        $this->requireEditor();
-        $rule = new Rule();
-        $rule->domain = $this->domainFor($input['domain'] ?? null);
-        $this->apply($rule, $input);
-        $this->repo->save($rule);
-        return $rule;
-    }
-
-    /** Attributes missing in $input keep their value @throws UserError */
-    public function update(int $id, array $input): Rule
-    {
-        $rule = $this->editable($id);
-        $this->apply($rule, $input);
-        $this->repo->save($rule);
-        return $rule;
-    }
-
-    /** @throws UserError */
-    public function delete(int $id): Rule
-    {
-        $rule = $this->editable($id);
-        $this->repo->delete($id);
-        return $rule;
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupChoices(array $domains): array
-    {
-        return ($this->canRead() && $this->canUseGroups()) ? $this->repo->groupsIn($domains) : [];
-    }
-
-    /**
-     * What the rules delete in a mailbox, per folder (see Planner): for the mailbox user, or for those who may read
-     * the rules and access the mailbox.
-     *
-     * @return array[] actions of Planner::actions()
-     * @throws AccessDenied
-     */
-    public function effective(mixed $username): array
-    {
-        $username = strtolower((string)Input::text($username));
-        $allowed = $this->context->isUser() ? $username === strtolower((string)$this->context->username)
-            : ($this->canRead() && ($this->context->hasAdminRights() || $this->context->canAccessMailbox($username)));
-        if ($username === '' || !$allowed) {
-            throw new AccessDenied();
-        }
-        return Planner::actions($this->repo->candidates([$username])[$username] ?? []);
-    }
-
-    /** @return array[] results of the nightly runs in the caller's domains, newest first; $limit 0 = LOG_LIMIT */
-    public function log(int $limit = 0): array
-    {
-        return $this->canRead() ? $this->repo->log($this->context, [], ($limit > 0) ? min($limit, self::LOG_LIMIT * 10) : self::LOG_LIMIT) : [];
-    }
-
-    /**
-     * A page of the plan of the nightly run (system context), mailboxes after $after in order
-     *
-     * @return array{mailboxes: array<string, array[]>, next: ?string} username => actions of Planner::actions() for
-     *         the mailboxes with rules; next: the $after of the next page, null after the last
-     */
-    public function plan(string $after = '', int $limit = 0): array
-    {
-        $limit = ($limit > 0) ? min($limit, self::PLAN_PAGE) : self::PLAN_PAGE;
-        $usernames = $this->repo->mailboxesWithRules($after, $limit);
-        $plan = array_filter(array_map(fn(array $candidates): array => Planner::actions($candidates), $this->repo->candidates($usernames)));
-        return ['mailboxes' => $plan, 'next' => (count($usernames) < $limit) ? null : (string)end($usernames)];
-    }
-
-    /**
-     * Results of the nightly run (system context): mailboxes with found mail or errors, a summary per rule and at
-     * the end the whole run
-     *
-     * @throws UserError for more than MAX_RESULTS results
-     */
-    public function report(array $results, array $summary, ?array $run = null): int
-    {
-        if (count($results) > self::MAX_RESULTS || count($summary) > self::MAX_RESULTS * 10) {
-            throw new UserError('retention_entries_exceeded', [self::MAX_RESULTS]);
-        }
-        $results = array_filter($results, fn(mixed $result): bool => is_array($result) && is_string($result['username'] ?? null)
-            && is_numeric($result['rule_id'] ?? null) && is_string($result['folder'] ?? null));
-        $summary = array_filter($summary, fn(mixed $runOfRule): bool => is_array($runOfRule) && is_numeric($runOfRule['rule_id'] ?? null));
-        return $this->repo->addLog(array_values($results), array_values($summary), $run);
-    }
-
-    /** The domain was deleted: its rules went with it (foreign key), the results of its runs go now */
-    public function domainDeleted(string $domain): void
-    {
-        $this->repo->deleteLog($domain);
-    }
-
-    private function visible(Rule $rule): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->canAccessDomain($rule->domain);
-    }
-
-    private function canUseGroups(): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->hasAcl('mailbox_groups');
-    }
-
-    /** @throws UserError */
-    private function editable(int $id): Rule
-    {
-        $this->requireEditor();
-        $rule = $this->get($id);
-        if ($rule === null || !$this->canEdit($rule)) {
-            throw new AccessDenied();
-        }
-        return $rule;
-    }
-
-    /** @throws UserError */
-    private function domainFor(mixed $value): string
-    {
-        $domain = strtolower((string)Input::text($value));
-        if ($domain === '' || !$this->repo->isDomain($domain)) {
-            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
-        }
-        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        return $domain;
-    }
-
-    /** @throws UserError */
-    private function apply(Rule $rule, array $input): void
-    {
-        if (array_key_exists('name', $input) || $rule->id === null) {
-            $name = Input::text($input['name'] ?? null);
-            if ($name === null || $name === '' || mb_strlen($name) > 100 || preg_match('/[\x00-\x1f\x7f]/', $name)) {
-                throw new UserError('retention_name_invalid');
-            }
-            if ($this->repo->nameExists($rule->domain, $name, $rule->id)) {
-                throw new UserError('object_exists', [htmlspecialchars($name)]);
-            }
-            $rule->name = $name;
-        }
-        if (array_key_exists('description', $input)) {
-            $description = Input::text($input['description']);
-            if ($description === null || mb_strlen($description) > 255 || preg_match('/[\x00-\x1f\x7f]/', $description)) {
-                throw new UserError('retention_description_invalid');
-            }
-            $rule->description = $description;
-        }
-        // the UI sends the special folders and the own folder names apart (own_folders, one per line or a list)
-        if (array_key_exists('folders', $input) || array_key_exists('own_folders', $input)) {
-            $folders = array_key_exists('folders', $input) ? self::values($input['folders'])
-                : array_filter($rule->folders, fn(string $folder): bool => in_array($folder, Rule::FOLDERS, true));
-            if (array_key_exists('own_folders', $input)) {
-                $folders = array_filter($folders, fn(string $folder): bool => !str_starts_with($folder, Rule::OWN_FOLDER));
-                $names = is_array($input['own_folders']) ? $input['own_folders'] : preg_split('/\r?\n/', (string)Input::text($input['own_folders']));
-                foreach ($names as $name) {
-                    if (!is_string($name)) {
-                        throw new UserError('retention_folder_invalid', ['']);
-                    }
-                    if (trim($name) !== '') {
-                        $folders[] = Rule::OWN_FOLDER . trim($name);
-                    }
-                }
-            }
-            $rule->folders = self::folders($folders);
-        }
-        if ($rule->folders === []) {
-            throw new UserError('retention_folders_invalid');
-        }
-        if (array_key_exists('days', $input) || $rule->id === null) {
-            $days = Input::intInRange($input['days'] ?? null, 1, self::MAX_DAYS);
-            $rule->days = $days ?? throw new UserError('retention_days_invalid', [self::MAX_DAYS]);
-        }
-        if (array_key_exists('mode', $input)) {
-            $mode = Input::text($input['mode']);
-            $rule->mode = in_array($mode, Rule::MODES, true) ? $mode : throw new UserError('retention_mode_invalid');
-        }
-        foreach (['keep_flagged' => 'keepFlagged', 'all_mailboxes' => 'allMailboxes', 'active' => 'active'] as $key => $property) {
-            if (array_key_exists($key, $input)) {
-                $rule->$property = (int)self::flag($input[$key]);
-            }
-        }
-        if (array_key_exists('mailboxes', $input)) {
-            $rule->mailboxes = [];
-            foreach (self::limited($input['mailboxes'], self::MAX_MEMBERS) as $username) {
-                $username = strtolower($username);
-                if ($this->repo->mailboxDomain($username) !== $rule->domain) {
-                    throw new UserError('retention_member_invalid', [htmlspecialchars($username)]);
-                }
-                $rule->mailboxes[$username] = $username;
-            }
-            $rule->mailboxes = array_values($rule->mailboxes);
-        }
-        // without access to groups the assignments stay; only sending them back unchanged is accepted
-        $groupAccess = $this->canUseGroups();
-        if (array_key_exists('groups', $input) && !$groupAccess && self::values($input['groups']) !== []) {
-            $sent = array_values(array_unique(array_map('intval', self::values($input['groups']))));
-            $stored = $rule->groups;
-            sort($sent);
-            sort($stored);
-            if ($sent !== $stored) {
-                throw new AccessDenied();
-            }
-        }
-        if (array_key_exists('groups', $input) && $groupAccess) {
-            $rule->groups = [];
-            foreach (self::limited($input['groups'], self::MAX_MEMBERS) as $value) {
-                $id = Input::intInRange($value, 1, PHP_INT_MAX);
-                if ($id === null || $this->repo->groupDomain($id) !== $rule->domain) {
-                    throw new UserError('retention_member_invalid', [htmlspecialchars($value)]);
-                }
-                $rule->groups[$id] = $id;
-            }
-            $rule->groups = array_values($rule->groups);
-        }
-    }
-
-    /**
-     * Folders of a rule: keys of Rule::FOLDERS and folder:<name>, without repeats. Names are Dovecot folder names
-     * (separator /, wildcard *), printable and at most 255 characters; INBOX is written in capitals like Dovecot does.
-     *
-     * @param string[] $values
-     * @return string[]
-     * @throws UserError
-     */
-    private static function folders(array $values): array
-    {
-        $folders = [];
-        foreach (self::limited($values, self::MAX_FOLDERS) as $folder) {
-            if (str_starts_with($folder, Rule::OWN_FOLDER)) {
-                $name = trim(substr($folder, strlen(Rule::OWN_FOLDER)), " \t/");
-                if ($name === '' || mb_strlen($name) > 255 || preg_match('/[\x00-\x1f\x7f]/u', $name) || !mb_check_encoding($name, 'UTF-8')) {
-                    throw new UserError('retention_folder_invalid', [htmlspecialchars($folder)]);
-                }
-                $name = preg_replace('~^inbox(?=/|$)~i', 'INBOX', $name);
-                $folder = Rule::OWN_FOLDER . $name;
-            } elseif (!in_array($folder, Rule::FOLDERS, true)) {
-                throw new UserError('retention_folder_invalid', [htmlspecialchars($folder)]);
-            }
-            $folders[$folder] = $folder;
-        }
-        return array_values($folders);
-    }
-
-    /**
-     * Values of a list field without repeats, at most $max (checked before anything is looked up)
-     *
-     * @return string[]
-     * @throws UserError
-     */
-    private static function limited(mixed $value, int $max): array
-    {
-        $values = array_values(array_unique(self::values($value)));
-        if (count($values) > $max) {
-            throw new UserError('retention_entries_exceeded', [$max]);
-        }
-        return $values;
-    }
-
-    /**
-     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks or commas.
-     *
-     * @return string[]
-     */
-    private static function values(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\r\n,;]+/', (string)Input::text($value));
-        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
-    }
-
-    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
-    private static function flag(mixed $value): bool
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
-    }
-}

data/web/inc/restore.inc.php

@@ -1,19 +0,0 @@
-<?php
-// E-Mail > Restore of admins and domain admins (admin/restore.php, domainadmin/restore.php set $role after protect_route):
-// restores from the snapshots of the backup, also of deleted mailboxes (and domains for admins)
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/header.inc.php';
-$_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
-
-$js_minifier->add('/web/js/site/backup.js');
-
-$template = 'restore.twig';
-$template_data = [
-  'acl' => $_SESSION['acl'],
-  'role' => $role,
-  'lang_datatables' => json_encode($lang['datatables']),
-  'backup_configured' => \Mailcow\Backup\BackupModule::fromGlobals()->settings->load()->configured(),
-  'backup_timezone' => getenv('TZ') ?: 'UTC',
-  'backup_demo_mode' => $DEMO_MODE,
-];
-
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/inc/triggers.user.inc.php

@@ -19,10 +19,8 @@ if ($iam_provider){
       header('Location: ' . $redirect_uri);
       die();
     }
-  } elseif (!empty($_GET['code']) && !empty($_SESSION['oauth2state']) && is_string($_GET['state'] ?? null)
+  } elseif ($_GET['code'] && $_GET['state'] === $_SESSION['oauth2state']) {
-    && hash_equals((string)$_SESSION['oauth2state'], $_GET['state'])) {
+    // Check given state against previously stored one to mitigate CSRF attack
-    // Check given state against previously stored one to mitigate CSRF attack, a state is used once
-    unset($_SESSION['oauth2state']);
     // Received access token in $_GET['code']
     // extract info and verify user
     identity_provider('verify-sso');

data/web/inc/vars.inc.php

@@ -169,12 +169,6 @@ $SESSION_NAME = "MCSESSID";
 // Label for OTP devices
 $OTP_LABEL = "mailcow UI";
 
-// Failed second factor logins (TOTP, Yubico OTP, WebAuthn) of an account before its second factor is locked
-$TFA_MAX_FAILS = 10;
-
-// Time (in s) from the first failed second factor login until the counter and the lock are reset
-$TFA_FAIL_WINDOW = 900;
-
 // How long to wait (in s) for cURL Docker requests
 $DOCKER_TIMEOUT = 60;
 
@@ -297,13 +291,13 @@ $RSPAMD_MAPS = array(
 //   true   = option requires a non-empty value
 //   false  = pure flag, must NOT have a value
 //   string = PCRE (no delimiters) the whole non-empty value must match
-// Anything not listed here is rejected when a job is saved; the runner passes stored options as they are, except the blocked ones below.
+// Anything not listed here is rejected when a job is saved (the runner passes stored options as they are).
 // Override in vars.local.inc.php per option, e.g. $IMAPSYNC_OPTIONS['someoption'] = true; or unset($IMAPSYNC_OPTIONS['debug']);
 // The old format $IMAPSYNC_OPTIONS['whitelist'] (list of names) is no longer supported and ignored (logged).
 // NEVER add (RCE): 'delete2foldersonly', 'delete2foldersbutnot', 'regexflag', 'regexmess', 'pipemess', 'regextrans2', 'maxlinelengthcmd', 'oauthrefreshcmd1', 'oauthrefreshcmd2'
 // NEVER add options taking local file paths (e.g. 'logfile', 'logdir', 'pidfile', 'passfile1', 'sslargs1' with SSL_ca_file)
-// These, the tests* options (print the environment), 'showpasswords' and 'abort' (stops other runs) are also blocked in code
 $IMAPSYNC_OPTIONS_DEFAULT = array(
+  'abort'                  => false,
   'authmd51'               => false,
   'authuser1'              => true,
   'debug'                  => false,
@@ -379,6 +373,9 @@ $IMAPSYNC_OPTIONS_DEFAULT = array(
   'syncduplicates'         => false,
   'syncinternaldates'      => false,
   'synclabels'             => false,
+  'tests'                  => false,
+  'testslive'              => false,
+  'testslive6'             => false,
   'truncmess'              => true,
   'usecache'               => false,
   'useheader'              => true,

data/web/js/site/dashboard.js

@@ -875,93 +875,6 @@ jQuery(function($){
       hideTableExpandCollapseBtn('#tab-cron-logs', '#cron_log');
     });
   }
-  // results of the nightly retention runs: mailboxes with found mail or errors, and the runs (username '', no rule)
-  function draw_retention_logs() {
-    // just recalc width if instance already exists
-    if ($.fn.DataTable.isDataTable('#retention_log') ) {
-      $('#retention_log').DataTable().columns.adjust().responsive.recalc();
-      return;
-    }
-    var L = function(key) { return (lang.retention && lang.retention[key]) || ''; };
-    $('#retention_log').DataTable({
-      responsive: true,
-      processing: true,
-      serverSide: false,
-      stateSave: true,
-      pageLength: log_pagination_size,
-      dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" +
-           "tr" +
-           "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>",
-      language: lang_datatables,
-      order: [[0, 'desc']],
-      ajax: {
-        type: "GET",
-        url: "/api/v1/get/logs/retention",
-        dataSrc: function(data){
-          return Array.isArray(data) ? data : [];
-        }
-      },
-      columns: [
-        {
-          title: lang.time,
-          data: 'time',
-          defaultContent: '',
-          createdCell: function(td, cellData) {
-            createSortableDate(td, cellData)
-          }
-        },
-        {
-          title: L('mailbox'),
-          data: 'username',
-          defaultContent: '',
-          render: function(data, type, row) {
-            return row.rule_id === null ? '<i>' + escapeHtml(L('run')) + '</i>' : escapeHtml(data);
-          }
-        },
-        {
-          title: L('rule'),
-          data: 'rule_name',
-          defaultContent: '',
-          render: function(data) {
-            return escapeHtml(data);
-          }
-        },
-        {
-          title: L('folder'),
-          data: 'folder',
-          defaultContent: '',
-          className: 'text-break',
-          render: function(data, type, row) {
-            if (row.rule_id === null) return escapeHtml(L('run_mailboxes').replace('%s', row.mailboxes));
-            return (data || '').split('\n').map(escapeHtml).join('<br>') + ' <span class="text-muted small">(' + escapeHtml(L('days_short').replace('%s', row.days)) + ')</span>';
-          }
-        },
-        {
-          title: L('found'),
-          data: 'matched',
-          defaultContent: ''
-        },
-        {
-          title: L('deleted'),
-          data: 'deleted',
-          defaultContent: '',
-          render: function(data, type, row) {
-            return row.simulated == 1 ? '<span class="badge bg-secondary">' + escapeHtml(L('simulated')) + '</span>' : escapeHtml(data);
-          }
-        },
-        {
-          title: L('error'),
-          data: 'error',
-          defaultContent: '',
-          className: 'dtr-col-md text-break',
-          render: function(data, type, row) {
-            if (!data && row.rule_id === null && row.errors > 0) data = L('errors').replace('%s', row.errors);
-            return data ? '<span class="text-danger">' + escapeHtml(data) + '</span>' : '';
-          }
-        }
-      ]
-    });
-  }
   function rspamd_pie_graph() {
     $.ajax({
       url: '/api/v1/get/rspamd/actions',
@@ -1382,7 +1295,6 @@ jQuery(function($){
   onVisible("[id^=sasl_logs]", () => draw_sasl_logs());
   onVisible("[id^=netfilter_log]", () => draw_netfilter_logs());
   onVisible("[id^=cron_log]", () => draw_cron_logs());
-  onVisible("[id^=retention_log]", () => draw_retention_logs());
   onVisible("[id^=rspamd_history]", () => draw_rspamd_history());
   onVisible("[id^=rspamd_donut]", () => rspamd_pie_graph());