[Postfix][Rspamd] Send archive copies as SMTP journal through Postfix

data/Dockerfiles/postfix/postfix.sh

@@ -15,6 +15,21 @@ until dig +short mailcow.email > /dev/null; do
   sleep 1
 done
 
+# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates
+DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+DBV_WAIT=0
+while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
+  echo "Waiting for schema update..."
+  sleep 5
+  DBV_WAIT=$((DBV_WAIT + 1))
+  DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
+done
+if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
+  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
+else
+  echo "DB schema is ${DBV_NOW}"
+fi
+
 cat <<EOF > /etc/aliases
 # Autogenerated by mailcow
 null: /dev/null
@@ -163,6 +178,45 @@ query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM tra
   AND destination = '%s';
 EOF
 
+# Archive copies (journal) to the archive set in the admin UI: on port 591 the sender archive-bounce@localhost
+# sends the whole mail through archive_smtp (master.cf), whatever the recipient domain
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT('FILTER archive_smtp:', nexthop) AS action FROM archive_transport
+  WHERE active = '1'
+  AND '%s' = 'archive-bounce@localhost'
+  LIMIT 1;
+EOF
+
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM archive_transport
+  WHERE nexthop = '%s'
+  AND active = '1'
+  AND username != ''
+  LIMIT 1;
+EOF
+
+cat <<EOF > /opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
+# Autogenerated by mailcow
+user = ${DBUSER}
+password = ${DBPASS}
+hosts = unix:/var/run/mysqld/mysqld.sock
+dbname = ${DBNAME}
+query = SELECT tls_policy FROM archive_transport
+  WHERE nexthop = '%s'
+  AND active = '1'
+  LIMIT 1;
+EOF
+
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_resource_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}

data/conf/postfix/local_transport

@@ -1,2 +1,3 @@
 /watchdog@localhost$/   watchdog_discard:
+/^archive-bounce@localhost$/  archive_discard:
 /localhost$/  local:

data/conf/postfix/main.cf

@@ -50,6 +50,9 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf,
+  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   $sender_dependent_default_transport_maps,

data/conf/postfix/master.cf

@@ -61,13 +61,14 @@ submission inet n       -       n       -       -       smtpd
   -o non_smtpd_milters=
   -o syslog_name=postfix/quarantine
 
-# used to send bcc mails
+# used to send bcc mails and archive copies (sender archive-bounce@localhost goes to the archive)
 591 inet n      -       n       -       -       smtpd
   -o smtpd_helo_restrictions=
   -o smtpd_client_restrictions=permit_mynetworks,reject
   -o smtpd_tls_auth_only=no
   -o smtpd_milters=
   -o non_smtpd_milters=
+  -o smtpd_sender_restrictions=check_sender_access,proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,permit_mynetworks,reject
   -o syslog_name=postfix/bcc
 
 # enforced smtp connector
@@ -81,6 +82,27 @@ smtp_enforced_tls      unix  -       -       n       -       -       smtp
 smtp_via_transport_maps      unix  -       -       n       -       -       smtp
   -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf
 
+# archive copies (journal) to the archive set in the admin UI: only mails from archive-bounce@localhost on 591 get here
+# (FILTER of mysql_archive_sender_filter_maps), sent unchanged, TLS by default, refusals are retried (soft_bounce)
+# until maximal_queue_lifetime
+archive_smtp      unix  -       -       n       -       -       smtp
+  -o soft_bounce=yes
+  -o smtp_header_checks=
+  -o smtp_mime_header_checks=
+  -o smtp_nested_header_checks=
+  -o smtp_body_checks=
+  -o smtp_line_length_limit=0
+  -o disable_mime_output_conversion=yes
+  -o smtp_tls_security_level=encrypt
+  -o smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
+  -o smtp_sasl_auth_enable=yes
+  -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
+  -o smtp_sasl_tls_security_options=noanonymous
+  -o syslog_name=postfix/archive
+# notices about archive copies (delays, bounces) go to archive-bounce@localhost and are dropped here
+archive_discard   unix  -       -       n       -       -       discard
+  -o syslog_name=postfix/archive
+
 tlsproxy   unix  -       -       n       -       0       tlsproxy
 dnsblog    unix  -       -       n       -       0       dnsblog
 pickup     fifo  n       -       n       60      1       pickup

data/conf/rspamd/dynmaps/archive.php

@@ -0,0 +1,47 @@
+<?php
+// Archive decision for an accepted mail, asked by the MAILCOW_ARCHIVE symbol of rspamd
+// Request: JSON {from, rcpt[], user, bcc, qid, received, sha256}; answer: JSON {archive, rcpt, sender, headers}
+header('Content-Type: application/json');
+require_once "vars.inc.php";
+require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
+// Do not show errors, we log to using error_log
+ini_set('error_reporting', 0);
+// Init database
+$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
+$opt = [
+    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
+    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
+    PDO::ATTR_EMULATE_PREPARES   => false,
+];
+try {
+  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
+}
+catch (PDOException $e) {
+  error_log("ARCHIVE: " . $e . PHP_EOL);
+  http_response_code(501);
+  exit;
+}
+// Init Redis, only to mirror the switch rspamd reads
+$redis = new Redis();
+try {
+  $redis->connect('redis-mailcow', 6379);
+  $redis->auth(getenv("REDISPASS"));
+}
+catch (Exception $e) {
+  $redis = null;
+}
+
+$request = json_decode(file_get_contents('php://input'), true);
+if (!is_array($request)) {
+  http_response_code(400);
+  exit;
+}
+
+try {
+  $archive = \Mailcow\Archive\ArchiveModule::create($pdo, \Mailcow\Core\Context::system(), $redis);
+  echo json_encode($archive->journal->prepare($request));
+}
+catch (Throwable $e) {
+  error_log("ARCHIVE: " . $e->getMessage() . PHP_EOL);
+  http_response_code(502);
+}

data/conf/rspamd/lua/rspamd.local.lua

@@ -956,3 +956,138 @@ rspamd_config:register_symbol({
   end,
   priority = 1
 })
+
+-- Archive: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through Postfix (port 591)
+rspamd_config:register_symbol({
+  name = 'MAILCOW_ARCHIVE',
+  type = 'idempotent',
+  flags = 'empty,explicit_disable,ignore_passthrough',
+  callback = function(task)
+    local rspamd_http = require "rspamd_http"
+    local rspamd_logger = require "rspamd_logger"
+    local rspamd_text = require "rspamd_text"
+    local rspamd_util = require "rspamd_util"
+    local hash = require "rspamd_cryptobox_hash"
+    local lua_smtp = require "lua_smtp"
+    local cjson = require "cjson"
+
+    -- accepted mails only: no reject, soft reject (greylisting included), discard or quarantine
+    local action = task:get_metric_action('default')
+    local accepted = (action == 'no action' or action == 'add header' or action == 'rewrite subject' or action == 'greylist')
+    local pre_accept = false
+    if task.has_pre_result then
+      local has_pre, pre_action = task:has_pre_result()
+      pre_accept = (has_pre and pre_action == 'accept')
+    end
+    if not accepted and not pre_accept then
+      return
+    end
+
+    local rcpts = {}
+    for _, r in ipairs(task:get_recipients('smtp') or {}) do
+      if r.addr and r.addr ~= '' then
+        table.insert(rcpts, r.addr)
+      end
+    end
+    if #rcpts == 0 then
+      return
+    end
+    local from = task:get_from('smtp')
+    local from_addr = (from and from[1] and from[1].addr) or ''
+    local qid = task:get_queue_id() or ''
+    -- the BCC symbol sends copies under the same condition, they bypass rspamd
+    local bcc = (pre_accept or (action ~= 'greylist' and accepted)) and not task:has_symbol('ENCRYPTED_CHAT')
+    local redis_params = rspamd_parse_redis_server('archive')
+
+    -- not archived: counted and listed for the archive status in the UI
+    -- the Message-ID finds the mail in the archive log of the UI
+    local message_id = task:get_message_id()
+    if not message_id or message_id == 'undef' then
+      message_id = ''
+    end
+    local function dropped(reason)
+      rspamd_logger.errx(task, "archive: mail %s is not archived: %s", qid, reason)
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED', true, function() end, 'INCR', {'ARCHIVE_DROPPED'})
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LPUSH',
+        {'ARCHIVE_DROPPED_LOG', cjson.encode({time = os.time(), qid = qid, reason = tostring(reason), message_id = message_id})})
+      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LTRIM', {'ARCHIVE_DROPPED_LOG', '0', '99'})
+    end
+
+    local function archive()
+      -- the archive gets the mail as accepted, with CRLF line ends and a final CRLF (also what the hash covers)
+      local content = rspamd_text.fromstring(tostring(task:get_content()))
+      content:normalize_newlines('smtp')
+      if #content < 2 or content:at(#content) ~= 10 or content:at(#content - 1) ~= 13 then
+        content = rspamd_text.fromtable({content, '\r\n'})
+      end
+      local body = {
+        from = from_addr,
+        rcpt = rcpts,
+        user = task:get_user() or '',
+        bcc = bcc,
+        qid = qid,
+        received = rspamd_util.get_time(),
+        sha256 = hash.create_specific('sha256', content):hex()
+      }
+
+      local function sent_cb(ok, err)
+        if not ok then
+          dropped(err)
+        end
+      end
+
+      local function journal_cb(err, code, res_body)
+        if err or code ~= 200 then
+          dropped(string.format('owner lookup failed (%s, %s)', err, code))
+          return
+        end
+        local ok, res = pcall(cjson.decode, tostring(res_body))
+        if not ok or type(res) ~= 'table' then
+          dropped('invalid answer of the owner lookup')
+          return
+        end
+        if res.archive ~= true then
+          return
+        end
+        -- lua_smtp ends the data with CRLF.CRLF, so the last CRLF of the content is left to it
+        lua_smtp.sendmail({
+          task = task,
+          host = os.getenv("IPV4_NETWORK") .. '.253',
+          port = 591,
+          from = res.sender,
+          recipients = res.rcpt,
+          helo = 'archive',
+          timeout = 20,
+        }, {res.headers or '', content:span(1, #content - 2)}, sent_cb)
+      end
+
+      rspamd_http.request({
+        task = task,
+        url = 'http://nginx:8081/archive.php',
+        body = cjson.encode(body),
+        timeout = (#rcpts > 100) and 10.0 or 2.0,
+        callback = journal_cb,
+        headers = {['Content-Type'] = 'application/json'},
+      })
+    end
+
+    local function enabled_cb(err, data)
+      if not err and type(data) == 'string' and data == '0' then
+        return
+      end
+      archive()
+    end
+
+    local ret = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'ARCHIVE_ENABLED', -- hash key
+      false, -- is write
+      enabled_cb, --callback
+      'GET', -- command
+      {'ARCHIVE_ENABLED'} -- arguments
+    )
+    if not ret then
+      archive()
+    end
+  end
+})

data/web/inc/init_db.inc.php

@@ -242,6 +242,7 @@ function init_db_schema()
           "gal" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "relay_all_recipients" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "relay_unknown_only" => "TINYINT(1) NOT NULL DEFAULT '0'",
+          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
           "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
           "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
@@ -1053,7 +1054,8 @@ function init_db_schema()
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'",
+          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1277,6 +1279,42 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
+      "archive_settings" => array(
+        "cols" => array(
+          "name" => "VARCHAR(64) NOT NULL",
+          "value" => "TEXT NOT NULL",
+          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
+          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("name")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
+      // Archive target the Postfix maps read (FILTER of the copies on port 591, SASL, TLS policy), one row kept in line with archive_settings
+      "archive_transport" => array(
+        "cols" => array(
+          "id" => "INT NOT NULL AUTO_INCREMENT",
+          "address" => "VARCHAR(255) NOT NULL",
+          "nexthop" => "VARCHAR(255) NOT NULL",
+          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "password" => "VARCHAR(255) NOT NULL DEFAULT ''",
+          "tls_policy" => "VARCHAR(64) NOT NULL DEFAULT 'encrypt'",
+          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
+        ),
+        "keys" => array(
+          "primary" => array(
+            "" => array("id")
+          ),
+          "key" => array(
+            "address" => array("address"),
+            "nexthop" => array("nexthop")
+          )
+        ),
+        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
+      ),
       "bcc_maps" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -2009,6 +2047,7 @@ function init_db_schema()
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.relayhost', \"0\") WHERE JSON_VALUE(`attributes`, '$.relayhost') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_pw_update', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_pw_update') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
+    $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.archive', \"inherit\") WHERE JSON_VALUE(`attributes`, '$.archive') IS NULL;");
     // admin attributes
     $pdo->query("UPDATE `admin` SET `attributes` = '{}' WHERE `attributes` = '' OR `attributes` IS NULL;");
     $pdo->query("UPDATE `admin` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
@@ -2056,6 +2095,7 @@ function init_db_schema()
         "backupmx" => 0,
         "relay_all_recipients" => 0,
         "relay_unknown_only" => 0,
+        "archive" => 0,
         "dkim_selector" => "dkim",
         "key_size" => 2048,
         "max_quota_for_domain" => 10240 * 1048576,
@@ -2073,6 +2113,7 @@ function init_db_schema()
         "rl_value" => "",
         "force_pw_update" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_pw_update']),
         "force_tfa" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_tfa']),
+        "archive" => strval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['archive']),
         "sogo_access" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['sogo_access']),
         "active" => 1,
         "tls_enforce_in" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['tls_enforce_in']),

data/web/inc/vars.inc.php

@@ -205,6 +205,9 @@ $MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update'] = false;
 // Force 2FA enrollment at next login
 $MAILBOX_DEFAULT_ATTRIBUTES['force_tfa'] = false;
 
+// Archive mails of the mailbox (archivecow): "inherit" follows the domain, "1" always, "0" never
+$MAILBOX_DEFAULT_ATTRIBUTES['archive'] = 'inherit';
+
 // Enable SOGo access - Users will be redirected to SOGo after login (set to false to disable redirect by default)
 $MAILBOX_DEFAULT_ATTRIBUTES['sogo_access'] = true;
 

docker-compose.yml

@@ -1,5 +1,5 @@
 # Version of the database schema
-x-db-version: &db_version DB_VERSION=02112026_1600
+x-db-version: &db_version DB_VERSION=06112026_1800
 
 services:
 
@@ -348,7 +348,7 @@ services:
             - dovecot
 
     postfix-mailcow:
-      image: ghcr.io/mailcow/postfix:3.10.12-1
+      image: ghcr.io/mailcow/postfix:3.10.12-2
       depends_on:
         mysql-mailcow:
           condition: service_started
@@ -375,6 +375,7 @@ services:
         - REDISPASS=${REDISPASS}
         - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
         - SPAMHAUS_DQS_KEY=${SPAMHAUS_DQS_KEY:-}
+        - *db_version
       cap_add:
         - NET_BIND_SERVICE
       ports: