From fb85a9eb5ca6b73919272f299e072a7d68d9a4fe Mon Sep 17 00:00:00 2001 From: FreddleSpl0it <75116288+FreddleSpl0it@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:26:17 +0100 Subject: [PATCH] [Postfix][Rspamd] Send archive copies as SMTP journal through Postfix --- data/Dockerfiles/postfix/postfix.sh | 54 +++++++++++ data/conf/postfix/local_transport | 1 + data/conf/postfix/main.cf | 3 + data/conf/postfix/master.cf | 24 ++++- data/conf/rspamd/dynmaps/archive.php | 47 +++++++++ data/conf/rspamd/lua/rspamd.local.lua | 135 ++++++++++++++++++++++++++ data/web/inc/init_db.inc.php | 43 +++++++- data/web/inc/vars.inc.php | 3 + docker-compose.yml | 5 +- 9 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 data/conf/rspamd/dynmaps/archive.php diff --git a/data/Dockerfiles/postfix/postfix.sh b/data/Dockerfiles/postfix/postfix.sh index 51927ea11..a2d82e0e3 100755 --- a/data/Dockerfiles/postfix/postfix.sh +++ b/data/Dockerfiles/postfix/postfix.sh @@ -15,6 +15,21 @@ until dig +short mailcow.email > /dev/null; do sleep 1 done +# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates +DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null) +DBV_WAIT=0 +while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do + echo "Waiting for schema update..." + sleep 5 + DBV_WAIT=$((DBV_WAIT + 1)) + DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null) +done +if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then + echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m" +else + echo "DB schema is ${DBV_NOW}" +fi + cat < /etc/aliases # Autogenerated by mailcow null: /dev/null @@ -163,6 +178,45 @@ query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM tra AND destination = '%s'; EOF +# Archive copies (journal) to the archive set in the admin UI: on port 591 the sender archive-bounce@localhost +# sends the whole mail through archive_smtp (master.cf), whatever the recipient domain +cat < /opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT('FILTER archive_smtp:', nexthop) AS action FROM archive_transport + WHERE active = '1' + AND '%s' = 'archive-bounce@localhost' + LIMIT 1; +EOF + +cat < /opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM archive_transport + WHERE nexthop = '%s' + AND active = '1' + AND username != '' + LIMIT 1; +EOF + +cat < /opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT tls_policy FROM archive_transport + WHERE nexthop = '%s' + AND active = '1' + LIMIT 1; +EOF + cat < /opt/postfix/conf/sql/mysql_virtual_resource_maps.cf # Autogenerated by mailcow user = ${DBUSER} diff --git a/data/conf/postfix/local_transport b/data/conf/postfix/local_transport index 6dd210111..feab66c6a 100644 --- a/data/conf/postfix/local_transport +++ b/data/conf/postfix/local_transport @@ -1,2 +1,3 @@ /watchdog@localhost$/ watchdog_discard: +/^archive-bounce@localhost$/ archive_discard: /localhost$/ local: diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 4329f5487..5c6d9afc7 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -50,6 +50,9 @@ postscreen_greet_wait = 3s postscreen_non_smtp_command_enable = no postscreen_pipelining_enable = no proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf, $sender_dependent_default_transport_maps, diff --git a/data/conf/postfix/master.cf b/data/conf/postfix/master.cf index fb49f2df2..6e2c93e2d 100644 --- a/data/conf/postfix/master.cf +++ b/data/conf/postfix/master.cf @@ -61,13 +61,14 @@ submission inet n - n - - smtpd -o non_smtpd_milters= -o syslog_name=postfix/quarantine -# used to send bcc mails +# used to send bcc mails and archive copies (sender archive-bounce@localhost goes to the archive) 591 inet n - n - - smtpd -o smtpd_helo_restrictions= -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_tls_auth_only=no -o smtpd_milters= -o non_smtpd_milters= + -o smtpd_sender_restrictions=check_sender_access,proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,permit_mynetworks,reject -o syslog_name=postfix/bcc # enforced smtp connector @@ -81,6 +82,27 @@ smtp_enforced_tls unix - - n - - smtp smtp_via_transport_maps unix - - n - - smtp -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf +# archive copies (journal) to the archive set in the admin UI: only mails from archive-bounce@localhost on 591 get here +# (FILTER of mysql_archive_sender_filter_maps), sent unchanged, TLS by default, refusals are retried (soft_bounce) +# until maximal_queue_lifetime +archive_smtp unix - - n - - smtp + -o soft_bounce=yes + -o smtp_header_checks= + -o smtp_mime_header_checks= + -o smtp_nested_header_checks= + -o smtp_body_checks= + -o smtp_line_length_limit=0 + -o disable_mime_output_conversion=yes + -o smtp_tls_security_level=encrypt + -o smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf + -o smtp_sasl_auth_enable=yes + -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf + -o smtp_sasl_tls_security_options=noanonymous + -o syslog_name=postfix/archive +# notices about archive copies (delays, bounces) go to archive-bounce@localhost and are dropped here +archive_discard unix - - n - - discard + -o syslog_name=postfix/archive + tlsproxy unix - - n - 0 tlsproxy dnsblog unix - - n - 0 dnsblog pickup fifo n - n 60 1 pickup diff --git a/data/conf/rspamd/dynmaps/archive.php b/data/conf/rspamd/dynmaps/archive.php new file mode 100644 index 000000000..afb24900e --- /dev/null +++ b/data/conf/rspamd/dynmaps/archive.php @@ -0,0 +1,47 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false, +]; +try { + $pdo = new PDO($dsn, $database_user, $database_pass, $opt); +} +catch (PDOException $e) { + error_log("ARCHIVE: " . $e . PHP_EOL); + http_response_code(501); + exit; +} +// Init Redis, only to mirror the switch rspamd reads +$redis = new Redis(); +try { + $redis->connect('redis-mailcow', 6379); + $redis->auth(getenv("REDISPASS")); +} +catch (Exception $e) { + $redis = null; +} + +$request = json_decode(file_get_contents('php://input'), true); +if (!is_array($request)) { + http_response_code(400); + exit; +} + +try { + $archive = \Mailcow\Archive\ArchiveModule::create($pdo, \Mailcow\Core\Context::system(), $redis); + echo json_encode($archive->journal->prepare($request)); +} +catch (Throwable $e) { + error_log("ARCHIVE: " . $e->getMessage() . PHP_EOL); + http_response_code(502); +} diff --git a/data/conf/rspamd/lua/rspamd.local.lua b/data/conf/rspamd/lua/rspamd.local.lua index 5642fba2c..d75c513a1 100644 --- a/data/conf/rspamd/lua/rspamd.local.lua +++ b/data/conf/rspamd/lua/rspamd.local.lua @@ -956,3 +956,138 @@ rspamd_config:register_symbol({ end, priority = 1 }) + +-- Archive: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through Postfix (port 591) +rspamd_config:register_symbol({ + name = 'MAILCOW_ARCHIVE', + type = 'idempotent', + flags = 'empty,explicit_disable,ignore_passthrough', + callback = function(task) + local rspamd_http = require "rspamd_http" + local rspamd_logger = require "rspamd_logger" + local rspamd_text = require "rspamd_text" + local rspamd_util = require "rspamd_util" + local hash = require "rspamd_cryptobox_hash" + local lua_smtp = require "lua_smtp" + local cjson = require "cjson" + + -- accepted mails only: no reject, soft reject (greylisting included), discard or quarantine + local action = task:get_metric_action('default') + local accepted = (action == 'no action' or action == 'add header' or action == 'rewrite subject' or action == 'greylist') + local pre_accept = false + if task.has_pre_result then + local has_pre, pre_action = task:has_pre_result() + pre_accept = (has_pre and pre_action == 'accept') + end + if not accepted and not pre_accept then + return + end + + local rcpts = {} + for _, r in ipairs(task:get_recipients('smtp') or {}) do + if r.addr and r.addr ~= '' then + table.insert(rcpts, r.addr) + end + end + if #rcpts == 0 then + return + end + local from = task:get_from('smtp') + local from_addr = (from and from[1] and from[1].addr) or '' + local qid = task:get_queue_id() or '' + -- the BCC symbol sends copies under the same condition, they bypass rspamd + local bcc = (pre_accept or (action ~= 'greylist' and accepted)) and not task:has_symbol('ENCRYPTED_CHAT') + local redis_params = rspamd_parse_redis_server('archive') + + -- not archived: counted and listed for the archive status in the UI + -- the Message-ID finds the mail in the archive log of the UI + local message_id = task:get_message_id() + if not message_id or message_id == 'undef' then + message_id = '' + end + local function dropped(reason) + rspamd_logger.errx(task, "archive: mail %s is not archived: %s", qid, reason) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED', true, function() end, 'INCR', {'ARCHIVE_DROPPED'}) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LPUSH', + {'ARCHIVE_DROPPED_LOG', cjson.encode({time = os.time(), qid = qid, reason = tostring(reason), message_id = message_id})}) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LTRIM', {'ARCHIVE_DROPPED_LOG', '0', '99'}) + end + + local function archive() + -- the archive gets the mail as accepted, with CRLF line ends and a final CRLF (also what the hash covers) + local content = rspamd_text.fromstring(tostring(task:get_content())) + content:normalize_newlines('smtp') + if #content < 2 or content:at(#content) ~= 10 or content:at(#content - 1) ~= 13 then + content = rspamd_text.fromtable({content, '\r\n'}) + end + local body = { + from = from_addr, + rcpt = rcpts, + user = task:get_user() or '', + bcc = bcc, + qid = qid, + received = rspamd_util.get_time(), + sha256 = hash.create_specific('sha256', content):hex() + } + + local function sent_cb(ok, err) + if not ok then + dropped(err) + end + end + + local function journal_cb(err, code, res_body) + if err or code ~= 200 then + dropped(string.format('owner lookup failed (%s, %s)', err, code)) + return + end + local ok, res = pcall(cjson.decode, tostring(res_body)) + if not ok or type(res) ~= 'table' then + dropped('invalid answer of the owner lookup') + return + end + if res.archive ~= true then + return + end + -- lua_smtp ends the data with CRLF.CRLF, so the last CRLF of the content is left to it + lua_smtp.sendmail({ + task = task, + host = os.getenv("IPV4_NETWORK") .. '.253', + port = 591, + from = res.sender, + recipients = res.rcpt, + helo = 'archive', + timeout = 20, + }, {res.headers or '', content:span(1, #content - 2)}, sent_cb) + end + + rspamd_http.request({ + task = task, + url = 'http://nginx:8081/archive.php', + body = cjson.encode(body), + timeout = (#rcpts > 100) and 10.0 or 2.0, + callback = journal_cb, + headers = {['Content-Type'] = 'application/json'}, + }) + end + + local function enabled_cb(err, data) + if not err and type(data) == 'string' and data == '0' then + return + end + archive() + end + + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + 'ARCHIVE_ENABLED', -- hash key + false, -- is write + enabled_cb, --callback + 'GET', -- command + {'ARCHIVE_ENABLED'} -- arguments + ) + if not ret then + archive() + end + end +}) diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index ff95a0b0a..eda3f28f6 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -242,6 +242,7 @@ function init_db_schema() "gal" => "TINYINT(1) NOT NULL DEFAULT '1'", "relay_all_recipients" => "TINYINT(1) NOT NULL DEFAULT '0'", "relay_unknown_only" => "TINYINT(1) NOT NULL DEFAULT '0'", + "archive" => "TINYINT(1) NOT NULL DEFAULT '0'", "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP", "active" => "TINYINT(1) NOT NULL DEFAULT '1'" @@ -1053,7 +1054,8 @@ function init_db_schema() "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'", "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'", "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'", - "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'" + "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'", + "archive" => "TINYINT(1) NOT NULL DEFAULT '0'" ), "keys" => array( "primary" => array( @@ -1277,6 +1279,42 @@ function init_db_schema() ), "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" ), + "archive_settings" => array( + "cols" => array( + "name" => "VARCHAR(64) NOT NULL", + "value" => "TEXT NOT NULL", + "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", + "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP" + ), + "keys" => array( + "primary" => array( + "" => array("name") + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + // Archive target the Postfix maps read (FILTER of the copies on port 591, SASL, TLS policy), one row kept in line with archive_settings + "archive_transport" => array( + "cols" => array( + "id" => "INT NOT NULL AUTO_INCREMENT", + "address" => "VARCHAR(255) NOT NULL", + "nexthop" => "VARCHAR(255) NOT NULL", + "username" => "VARCHAR(255) NOT NULL DEFAULT ''", + "password" => "VARCHAR(255) NOT NULL DEFAULT ''", + "tls_policy" => "VARCHAR(64) NOT NULL DEFAULT 'encrypt'", + "active" => "TINYINT(1) NOT NULL DEFAULT '1'" + ), + "keys" => array( + "primary" => array( + "" => array("id") + ), + "key" => array( + "address" => array("address"), + "nexthop" => array("nexthop") + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), "bcc_maps" => array( "cols" => array( "id" => "INT NOT NULL AUTO_INCREMENT", @@ -2009,6 +2047,7 @@ function init_db_schema() $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.relayhost', \"0\") WHERE JSON_VALUE(`attributes`, '$.relayhost') IS NULL;"); $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.force_pw_update', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_pw_update') IS NULL;"); $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;"); + $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.archive', \"inherit\") WHERE JSON_VALUE(`attributes`, '$.archive') IS NULL;"); // admin attributes $pdo->query("UPDATE `admin` SET `attributes` = '{}' WHERE `attributes` = '' OR `attributes` IS NULL;"); $pdo->query("UPDATE `admin` SET `attributes` = JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;"); @@ -2056,6 +2095,7 @@ function init_db_schema() "backupmx" => 0, "relay_all_recipients" => 0, "relay_unknown_only" => 0, + "archive" => 0, "dkim_selector" => "dkim", "key_size" => 2048, "max_quota_for_domain" => 10240 * 1048576, @@ -2073,6 +2113,7 @@ function init_db_schema() "rl_value" => "", "force_pw_update" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_pw_update']), "force_tfa" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_tfa']), + "archive" => strval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['archive']), "sogo_access" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['sogo_access']), "active" => 1, "tls_enforce_in" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['tls_enforce_in']), diff --git a/data/web/inc/vars.inc.php b/data/web/inc/vars.inc.php index 0f0dc154a..7da5634d4 100644 --- a/data/web/inc/vars.inc.php +++ b/data/web/inc/vars.inc.php @@ -205,6 +205,9 @@ $MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update'] = false; // Force 2FA enrollment at next login $MAILBOX_DEFAULT_ATTRIBUTES['force_tfa'] = false; +// Archive mails of the mailbox (archivecow): "inherit" follows the domain, "1" always, "0" never +$MAILBOX_DEFAULT_ATTRIBUTES['archive'] = 'inherit'; + // Enable SOGo access - Users will be redirected to SOGo after login (set to false to disable redirect by default) $MAILBOX_DEFAULT_ATTRIBUTES['sogo_access'] = true; diff --git a/docker-compose.yml b/docker-compose.yml index c6e07d7c5..83f57ca66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # Version of the database schema -x-db-version: &db_version DB_VERSION=02112026_1600 +x-db-version: &db_version DB_VERSION=06112026_1800 services: @@ -348,7 +348,7 @@ services: - dovecot postfix-mailcow: - image: ghcr.io/mailcow/postfix:3.10.12-1 + image: ghcr.io/mailcow/postfix:3.10.12-2 depends_on: mysql-mailcow: condition: service_started @@ -375,6 +375,7 @@ services: - REDISPASS=${REDISPASS} - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME} - SPAMHAUS_DQS_KEY=${SPAMHAUS_DQS_KEY:-} + - *db_version cap_add: - NET_BIND_SERVICE ports: