diff --git a/data/Dockerfiles/postfix/postfix.sh b/data/Dockerfiles/postfix/postfix.sh index 51927ea11..a2d82e0e3 100755 --- a/data/Dockerfiles/postfix/postfix.sh +++ b/data/Dockerfiles/postfix/postfix.sh @@ -15,6 +15,21 @@ until dig +short mailcow.email > /dev/null; do sleep 1 done +# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates +DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null) +DBV_WAIT=0 +while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do + echo "Waiting for schema update..." + sleep 5 + DBV_WAIT=$((DBV_WAIT + 1)) + DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null) +done +if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then + echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m" +else + echo "DB schema is ${DBV_NOW}" +fi + cat < /etc/aliases # Autogenerated by mailcow null: /dev/null @@ -163,6 +178,45 @@ query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM tra AND destination = '%s'; EOF +# Archive copies (journal) to the archive set in the admin UI: on port 591 the sender archive-bounce@localhost +# sends the whole mail through archive_smtp (master.cf), whatever the recipient domain +cat < /opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT('FILTER archive_smtp:', nexthop) AS action FROM archive_transport + WHERE active = '1' + AND '%s' = 'archive-bounce@localhost' + LIMIT 1; +EOF + +cat < /opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM archive_transport + WHERE nexthop = '%s' + AND active = '1' + AND username != '' + LIMIT 1; +EOF + +cat < /opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf +# Autogenerated by mailcow +user = ${DBUSER} +password = ${DBPASS} +hosts = unix:/var/run/mysqld/mysqld.sock +dbname = ${DBNAME} +query = SELECT tls_policy FROM archive_transport + WHERE nexthop = '%s' + AND active = '1' + LIMIT 1; +EOF + cat < /opt/postfix/conf/sql/mysql_virtual_resource_maps.cf # Autogenerated by mailcow user = ${DBUSER} diff --git a/data/conf/phpfpm/crons/identity-sync.php b/data/conf/phpfpm/crons/identity-sync.php index 11c55bff2..7483b3103 100644 --- a/data/conf/phpfpm/crons/identity-sync.php +++ b/data/conf/phpfpm/crons/identity-sync.php @@ -60,6 +60,7 @@ $_SESSION['acl']['sogo_access'] = "1"; $_SESSION['acl']['protocol_access'] = "1"; $_SESSION['acl']['mailbox_relayhost'] = "1"; $_SESSION['acl']['unlimited_quota'] = "1"; +$_SESSION['acl']['archive'] = "1"; // One run at a time, released when the process ends $lock = fopen('/tmp/identity-sync.lock', 'c'); diff --git a/data/conf/postfix/local_transport b/data/conf/postfix/local_transport index 6dd210111..feab66c6a 100644 --- a/data/conf/postfix/local_transport +++ b/data/conf/postfix/local_transport @@ -1,2 +1,3 @@ /watchdog@localhost$/ watchdog_discard: +/^archive-bounce@localhost$/ archive_discard: /localhost$/ local: diff --git a/data/conf/postfix/main.cf b/data/conf/postfix/main.cf index 4329f5487..5c6d9afc7 100644 --- a/data/conf/postfix/main.cf +++ b/data/conf/postfix/main.cf @@ -50,6 +50,9 @@ postscreen_greet_wait = 3s postscreen_non_smtp_command_enable = no postscreen_pipelining_enable = no proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf, + proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf, proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf, $sender_dependent_default_transport_maps, diff --git a/data/conf/postfix/master.cf b/data/conf/postfix/master.cf index fb49f2df2..7c9b22cec 100644 --- a/data/conf/postfix/master.cf +++ b/data/conf/postfix/master.cf @@ -61,13 +61,15 @@ submission inet n - n - - smtpd -o non_smtpd_milters= -o syslog_name=postfix/quarantine -# used to send bcc mails +# used to send bcc mails and archive copies (sender archive-bounce@localhost goes to the archive) +# the archive FILTER comes after the one of MX-based transports, Postfix uses the last FILTER 591 inet n - n - - smtpd -o smtpd_helo_restrictions= -o smtpd_client_restrictions=permit_mynetworks,reject -o smtpd_tls_auth_only=no -o smtpd_milters= -o non_smtpd_milters= + -o smtpd_recipient_restrictions=check_recipient_mx_access,proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,check_sender_access,proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,permit_mynetworks,reject -o syslog_name=postfix/bcc # enforced smtp connector @@ -81,6 +83,27 @@ smtp_enforced_tls unix - - n - - smtp smtp_via_transport_maps unix - - n - - smtp -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf +# archive copies (journal) to the archive set in the admin UI: only mails from archive-bounce@localhost on 591 get here +# (FILTER of mysql_archive_sender_filter_maps), sent unchanged, TLS by default, refusals are retried (soft_bounce) +# until maximal_queue_lifetime +archive_smtp unix - - n - - smtp + -o soft_bounce=yes + -o smtp_header_checks= + -o smtp_mime_header_checks= + -o smtp_nested_header_checks= + -o smtp_body_checks= + -o smtp_line_length_limit=0 + -o disable_mime_output_conversion=yes + -o smtp_tls_security_level=encrypt + -o smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf + -o smtp_sasl_auth_enable=yes + -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf + -o smtp_sasl_tls_security_options=noanonymous + -o syslog_name=postfix/archive +# notices about archive copies (delays, bounces) go to archive-bounce@localhost and are dropped here +archive_discard unix - - n - - discard + -o syslog_name=postfix/archive + tlsproxy unix - - n - 0 tlsproxy dnsblog unix - - n - 0 dnsblog pickup fifo n - n 60 1 pickup diff --git a/data/conf/rspamd/dynmaps/archive.php b/data/conf/rspamd/dynmaps/archive.php new file mode 100644 index 000000000..fc41f3e89 --- /dev/null +++ b/data/conf/rspamd/dynmaps/archive.php @@ -0,0 +1,47 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false, +]; +try { + $pdo = new PDO($dsn, $database_user, $database_pass, $opt); +} +catch (PDOException $e) { + error_log("ARCHIVE: " . $e . PHP_EOL); + http_response_code(501); + exit; +} +// Init Redis, it keeps the global switch; without it nothing is archived +$redis = new Redis(); +try { + $redis->connect('redis-mailcow', 6379); + $redis->auth(getenv("REDISPASS")); +} +catch (Exception $e) { + $redis = null; +} + +$request = json_decode(file_get_contents('php://input'), true); +if (!is_array($request)) { + http_response_code(400); + exit; +} + +try { + $archive = \Mailcow\Archive\ArchiveModule::create($pdo, \Mailcow\Core\Context::system(), $redis); + echo json_encode($archive->journal->prepare($request)); +} +catch (Throwable $e) { + error_log("ARCHIVE: " . $e->getMessage() . PHP_EOL); + http_response_code(502); +} diff --git a/data/conf/rspamd/lua/rspamd.local.lua b/data/conf/rspamd/lua/rspamd.local.lua index 5642fba2c..b0903a372 100644 --- a/data/conf/rspamd/lua/rspamd.local.lua +++ b/data/conf/rspamd/lua/rspamd.local.lua @@ -956,3 +956,134 @@ rspamd_config:register_symbol({ end, priority = 1 }) + +-- Archive: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through Postfix (port 591) +rspamd_config:register_symbol({ + name = 'MAILCOW_ARCHIVE', + type = 'idempotent', + flags = 'empty,explicit_disable,ignore_passthrough', + callback = function(task) + local rspamd_http = require "rspamd_http" + local rspamd_logger = require "rspamd_logger" + local rspamd_text = require "rspamd_text" + local rspamd_util = require "rspamd_util" + local hash = require "rspamd_cryptobox_hash" + local lua_smtp = require "lua_smtp" + local cjson = require "cjson" + + -- accepted mails only: no reject, soft reject (greylisting included), discard or quarantine + local action = task:get_metric_action('default') + local accepted = (action == 'no action' or action == 'add header' or action == 'rewrite subject' or action == 'greylist') + local pre_accept = false + if task.has_pre_result then + local has_pre, pre_action = task:has_pre_result() + pre_accept = (has_pre and pre_action == 'accept') + end + if not accepted and not pre_accept then + return + end + + local rcpts = {} + for _, r in ipairs(task:get_recipients('smtp') or {}) do + if r.addr and r.addr ~= '' then + table.insert(rcpts, r.addr) + end + end + if #rcpts == 0 then + return + end + local from = task:get_from('smtp') + local from_addr = (from and from[1] and from[1].addr) or '' + local qid = task:get_queue_id() or '' + -- the BCC symbol sends copies under the same condition, they bypass rspamd + local bcc = (pre_accept or (action ~= 'greylist' and accepted)) and not task:has_symbol('ENCRYPTED_CHAT') + local redis_params = rspamd_parse_redis_server('archive') + + -- not archived: counted and listed for the archive status in the UI + -- the Message-ID finds the mail in the archive log of the UI + local message_id = task:get_message_id() + if not message_id or message_id == 'undef' then + message_id = '' + end + local function dropped(reason) + rspamd_logger.errx(task, "archive: mail %s is not archived: %s", qid, reason) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED', true, function() end, 'INCR', {'ARCHIVE_DROPPED'}) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LPUSH', + {'ARCHIVE_DROPPED_LOG', cjson.encode({time = os.time(), qid = qid, reason = tostring(reason), message_id = message_id})}) + rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LTRIM', {'ARCHIVE_DROPPED_LOG', '0', '99'}) + end + + local function archive() + -- the archive gets the mail as accepted, with CRLF line ends and a final CRLF (also what the hash covers) + local content = rspamd_text.fromstring(tostring(task:get_content())) + content:normalize_newlines('smtp') + if #content < 2 or content:at(#content) ~= 10 or content:at(#content - 1) ~= 13 then + content = rspamd_text.fromtable({content, '\r\n'}) + end + local body = { + from = from_addr, + rcpt = rcpts, + user = task:get_user() or '', + bcc = bcc, + qid = qid, + received = rspamd_util.get_time(), + sha256 = hash.create_specific('sha256', content):hex() + } + + local function sent_cb(ok, err) + if not ok then + dropped(err) + end + end + + local function journal_cb(err, code, res_body) + if err or code ~= 200 then + dropped(string.format('owner lookup failed (%s, %s)', err, code)) + return + end + local ok, res = pcall(cjson.decode, tostring(res_body)) + if not ok or type(res) ~= 'table' then + dropped('invalid answer of the owner lookup') + return + end + if res.archive ~= true then + return + end + -- lua_smtp ends the data with CRLF.CRLF, so the last CRLF of the content is left to it + lua_smtp.sendmail({ + task = task, + host = os.getenv("IPV4_NETWORK") .. '.253', + port = 591, + from = res.sender, + recipients = res.rcpt, + helo = 'archive', + timeout = 20, + }, {res.headers or '', content:span(1, #content - 2)}, sent_cb) + end + + rspamd_http.request({ + task = task, + url = 'http://nginx:8081/archive.php', + body = cjson.encode(body), + timeout = (#rcpts > 100) and 10.0 or 2.0, + callback = journal_cb, + headers = {['Content-Type'] = 'application/json'}, + }) + end + + local function enabled_cb(err, data) + if not err and type(data) == 'string' and data == '1' then + archive() + end + end + + rspamd_redis_make_request(task, + redis_params, -- connect params + 'ARCHIVE_ENABLED', -- hash key + false, -- is write + enabled_cb, --callback + 'GET', -- command + {'ARCHIVE_ENABLED'} -- arguments + ) + end +}) diff --git a/data/web/admin/mailbox.php b/data/web/admin/mailbox.php index 05530e99e..4a1690bf4 100644 --- a/data/web/admin/mailbox.php +++ b/data/web/admin/mailbox.php @@ -46,6 +46,7 @@ $template_data = [ 'lang_edit' => json_encode($lang['edit']), 'lang_datatables' => json_encode($lang['datatables']), 'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(), + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(), ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php'; diff --git a/data/web/admin/system.php b/data/web/admin/system.php index 8353a8d2d..ed072ae12 100644 --- a/data/web/admin/system.php +++ b/data/web/admin/system.php @@ -13,6 +13,7 @@ $js_minifier->add('/web/js/site/admin.js'); $js_minifier->add('/web/js/presets/rspamd.js'); $js_minifier->add('/web/js/site/pwgen.js'); $js_minifier->add('/web/js/site/identity.js'); +$js_minifier->add('/web/js/site/archive.js'); // all domains $domains = mailbox('get', 'domains'); @@ -116,6 +117,7 @@ $template_data = [ 'ui_theme' => customize('get', 'ui_theme'), 'password_complexity' => password_complexity('get'), 'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(), + 'archive_settings' => \Mailcow\Archive\ArchiveModule::fromGlobals()->api->getSettings(), 'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'], 'cors_settings' => $cors_settings, 'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on', diff --git a/data/web/api/openapi.yaml b/data/web/api/openapi.yaml index b08747722..a8f4e285a 100644 --- a/data/web/api/openapi.yaml +++ b/data/web/api/openapi.yaml @@ -685,6 +685,11 @@ paths: is domain global address list active or not, it enables shared contacts accross domain in SOGo webmail type: boolean + archive: + description: >- + archive mails of the mailboxes of this domain by default (mailboxes set to `inherit`), + see GET /api/v1/get/archive/mailboxes + type: boolean mailboxes: description: limit count of mailboxes associated with this domain type: number @@ -1254,6 +1259,12 @@ paths: force_tfa: description: force 2FA enrollment at login type: boolean + archive: + description: >- + archiving of the mailbox: `inherit` follows the domain, `1` always, `0` never (admins, domain + admins with the archive permission; otherwise the template decides) + type: string + enum: [inherit, "1", "0"] tls_enforce_in: description: force inbound email tls encryption type: boolean @@ -3918,6 +3929,11 @@ paths: is domain global address list active or not, it enables shared contacts accross domain in SOGo webmail type: boolean + archive: + description: >- + archive mails of the mailboxes of this domain by default (admins, domain admins with the + archive permission) + type: boolean mailboxes: description: limit count of mailboxes associated with this domain type: number @@ -4222,6 +4238,12 @@ paths: force_tfa: description: force 2FA enrollment at login type: boolean + archive: + description: >- + archiving of the mailbox: `inherit` follows the domain, `1` always, `0` never (admins, domain + admins with the archive permission) + type: string + enum: [inherit, "1", "0"] name: description: Full name of the mailbox user type: string @@ -4546,6 +4568,104 @@ paths: default: run type: object summary: Request identity sync run + /api/v1/edit/archive: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - archive + - edit + - settings + - enabled: "1" + address: journal@archive.example.org + signing_secret: "*" + msg: archive_settings_saved + type: success + description: OK + headers: {} + tags: + - Archive + description: >- + Archive target, admins with a read-write key only. mailcow sends a copy of every accepted mail of archived + mailboxes by SMTP to `address`, through the Postfix queue (envelope sender `archive-bounce@localhost`, notices + to it are dropped: no bounce or delay warning reaches a sender). Attributes missing in the request keep their + value, an empty `password` or `signing_secret` keeps the stored one. Switching on needs the address, and the + signing key for the profile archivecow. With `action: test` the stored target is tested (connection, TLS, + AUTH, whether the address is accepted; no mail is sent), with `action: testmail` a test copy is sent through + the mail queue. + operationId: Edit archive settings + requestBody: + content: + application/json: + schema: + example: + items: + - archive + attr: + enabled: "1" + address: journal@archive.example.org + host: archive.example.org + port: 25 + tls: encrypt + profile: archivecow + signing_secret: secret-key + properties: + items: + type: array + default: ["archive"] + attr: + type: object + properties: + enabled: + description: copies of accepted mails go to the archive; copies already queued are delivered either way + type: boolean + address: + description: address the archive receives copies at + type: string + host: + description: SMTP server of the archive, empty for the MX of the address domain + type: string + port: + type: integer + default: 25 + tls: + description: "`encrypt` (TLS required), `verify` (TLS required, certificate verified), `may`" + type: string + enum: [encrypt, verify, may] + username: + description: SMTP AUTH user at the archive, empty for none + type: string + password: + description: SMTP AUTH password, never returned + type: string + profile: + description: >- + headers in front of each copy: `archivecow` (X-Archive-Owner, -Rcpt, -From, -Queue-ID, + -Received, -Direction, -Source, -SHA256 and X-Archive-Signature), `envelope` (sender and + recipients under `header_from` / `header_rcpt`), `none` + type: string + enum: [archivecow, envelope, none] + signing_secret: + description: key of X-Archive-Signature (HMAC-SHA256), never returned + type: string + header_from: + type: string + default: X-Envelope-From + header_rcpt: + type: string + default: X-Envelope-To + action: + type: string + enum: [test, testmail] + type: object + summary: Edit archive settings /api/v1/edit/mailq: post: responses: @@ -5758,6 +5878,7 @@ paths: description: Some description domain_name: domain.tld gal: "0" + archive: 1 max_new_mailbox_quota: 10737418240 max_num_aliases_for_domain: 400 max_num_mboxes_for_domain: 10 @@ -6312,7 +6433,9 @@ paths: response: value: - active: "1" + archive: 1 attributes: + archive: inherit force_pw_update: "0" force_tfa: "0" mailbox_format: "maildir:" @@ -6507,6 +6630,154 @@ paths: nothing was changed). operationId: Get identity sync status summary: Get identity sync status + /api/v1/get/archive/mailboxes: + get: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + enabled: 1 + mailboxes: + - username: anna@example.org + domain: example.org + name: Anna Example + active: 1 + source: domain + - username: board@example.net + domain: example.net + name: Board + active: 2 + source: mailbox + description: OK + headers: {} + tags: + - Archive + description: >- + Mailboxes whose mails go to the archive now (admins, read-only keys included). A mailbox is archived when + archiving is switched on (`enabled`) and its setting is `1`, or `inherit` while its domain archives by default + (`source` says which). Only mailboxes that receive mail are listed: `active` 1, or 2 (no login, mail is + delivered). `username` is the value of the X-Archive-Owner header. Mailboxes that drop out of the list + (archiving switched off, deleted, renamed) keep their mails in the archive. + operationId: Get archived mailboxes + summary: Get archived mailboxes + /api/v1/get/archive/status: + get: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + enabled: 1 + configured: 1 + queued: 2 + queued_bytes: 182044 + oldest_queued: 1762425600 + delay_reasons: + - "connect to archive.example.org[192.0.2.10]:25: Connection refused" + last_delivery: 1762425580 + dropped: 3 + dropped_24h: 1 + last_dropped: + time: 1762425500 + reason: "owner lookup failed (nil, 502)" + description: OK + headers: {} + tags: + - Archive + description: >- + Summary of the archive state (admins, read-only keys included): copies waiting in the Postfix queue with the + reasons Postfix gives, the time of the last delivery to the archive, and mails that were not archived because + their copy could not be handed to Postfix (`dropped` since ever, `dropped_24h` and `last_dropped` for the last + 24 hours). + operationId: Get archive status + summary: Get archive status + "/api/v1/get/logs/archive/{count}": + get: + parameters: + - description: Number of logs to return + in: path + name: count + required: true + schema: + type: number + - description: e.g. api-key-string + example: api-key-string + in: header + name: X-API-Key + required: false + schema: + type: string + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - time: 1762425580 + status: sent + qid: 4XyZ1abc + message_id: "c3d1e0f2@example.org" + text: "250 2.0.0 Ok: queued as 9F2C4" + - time: 1762425500 + status: dropped + qid: 3AbC2def + message_id: "a1b2c3@example.org" + text: "owner lookup failed (nil, 502)" + description: OK + headers: {} + tags: + - Logs + description: >- + Archive log, newest first (admins, read-only keys included): deliveries of copies from the Postfix log + (`status` sent, deferred, softbounce when the archive refused the copy and Postfix keeps it, or bounced), + copies discarded after the queue lifetime (expired) and mails that were not archived (dropped, `text` is the + reason). `message_id` is the Message-ID of the mail, `qid` the queue ID of the copy (of the mail for dropped). + + Tip: You can limit how many logs you want to get by using `/` at + the end of the api url. + operationId: Get archive log + summary: Get archive log + /api/v1/get/archive/settings: + get: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + enabled: 1 + address: journal@archive.example.org + host: archive.example.org + port: 25 + tls: encrypt + username: "" + has_password: 0 + profile: archivecow + has_signing_secret: 1 + header_from: X-Envelope-From + header_rcpt: X-Envelope-To + description: OK + headers: {} + tags: + - Archive + description: Archive target (admins, read-only keys included); password and signing key are never returned. + operationId: Get archive settings + summary: Get archive settings "/api/v1/get/identity-link/{username}": get: parameters: @@ -7597,7 +7868,9 @@ paths: response: value: - active: "1" + archive: 1 attributes: + archive: inherit force_pw_update: "0" force_tfa: "0" mailbox_format: "maildir:" @@ -8025,3 +8298,7 @@ tags: description: Manage Cross-Origin Resource Sharing (CORS) settings - name: Identity Provider description: Manage external Identity Provider settings + - name: Archive + description: >- + Archiving (SMTP journaling): a copy of every accepted incoming, outgoing and internal mail of archived + mailboxes goes by SMTP to the archive, through the Postfix queue. diff --git a/data/web/domainadmin/mailbox.php b/data/web/domainadmin/mailbox.php index 4b99579a0..020f84384 100644 --- a/data/web/domainadmin/mailbox.php +++ b/data/web/domainadmin/mailbox.php @@ -45,6 +45,7 @@ $template_data = [ 'lang_edit' => json_encode($lang['edit']), 'lang_datatables' => json_encode($lang['datatables']), 'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(), + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(), ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php'; \ No newline at end of file diff --git a/data/web/edit.php b/data/web/edit.php index 51d88a12d..67b188319 100644 --- a/data/web/edit.php +++ b/data/web/edit.php @@ -65,7 +65,8 @@ if (isset($_SESSION['mailcow_cc_role'])) { 'mta_sts' => $mta_sts, 'mailboxes' => mailbox('get', 'mailboxes', $_GET["domain"]), 'aliases' => mailbox('get', 'aliases', $_GET["domain"], 'address'), - 'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"]) + 'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"]), + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled() ]; } } @@ -75,6 +76,7 @@ if (isset($_SESSION['mailcow_cc_role'])) { $template_data = [ 'template' => $domain_template, 'rl' => ['frame' => $domain_template['attributes']['rl_frame']], + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(), ]; $template = 'edit/domain-templates.twig'; $result = true; @@ -85,6 +87,7 @@ if (isset($_SESSION['mailcow_cc_role'])) { $template_data = [ 'template' => $mailbox_template, 'rl' => ['frame' => $mailbox_template['attributes']['rl_frame']], + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(), ]; $template = 'edit/mailbox-templates.twig'; $result = true; @@ -162,6 +165,7 @@ if (isset($_SESSION['mailcow_cc_role'])) { 'user_acls' => acl('get', 'user', $mailbox), 'mailbox_details' => $result, 'iam_settings' => $iam_settings, + 'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(), ]; } } diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index 0c05293c4..c304e1b7c 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -539,6 +539,7 @@ function sys_mail($_data) { $mail->Body = $mass_text; } $mail->XMailer = 'MooMassMail'; + $archive = \Mailcow\Archive\ArchiveModule::fromGlobals(); foreach ($rcpts as $rcpt) { $mail->AddAddress($rcpt); if (!$mail->send()) { @@ -548,6 +549,10 @@ function sys_mail($_data) { 'msg' => 'Mailer error (RCPT "' . htmlspecialchars($rcpt) . '"): ' . str_replace('https://github.com/PHPMailer/PHPMailer/wiki/Troubleshooting', '', $mail->ErrorInfo) ); } + else { + // Delivered by LMTP without rspamd, so the archive gets it here + $archive->journal->submitDelivered($mail->getSentMIMEMessage(), $mass_from, array($rcpt), 'massmail'); + } $mail->ClearAllRecipients(); } } diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index 121d43a0c..075700804 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -366,6 +366,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $relay_unknown_only = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $DOMAIN_DEFAULT_ATTRIBUTES['relay_unknown_only']; $backupmx = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $DOMAIN_DEFAULT_ATTRIBUTES['backupmx']; $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $DOMAIN_DEFAULT_ATTRIBUTES['gal']; + $archive = (intval((isset($_data['archive'])) ? $_data['archive'] : ($DOMAIN_DEFAULT_ATTRIBUTES['archive'] ?? 0)) == 1) ? 1 : 0; if ($relay_all_recipients == 1) { $backupmx = '1'; } @@ -421,8 +422,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ':domain' => '%@' . $domain )); // save domain - $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`) - VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients)"); + $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`, `archive`) + VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients, :archive)"); $stmt->execute(array( ':domain' => $domain, ':description' => $description, @@ -435,7 +436,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ':gal' => $gal, ':active' => $active, ':relay_unknown_only' => $relay_unknown_only, - ':relay_all_recipients' => $relay_all_recipients + ':relay_all_recipients' => $relay_all_recipients, + ':archive' => $archive )); // save tags foreach($tags as $index => $tag){ @@ -940,6 +942,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : 0; // Shared mailbox: no login of its own, members open it through group shares $shared = (isset($_data['shared'])) ? intval($_data['shared']) : 0; + // Archiving comes from the template unless the caller may set it + $archive = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($MAILBOX_DEFAULT_ATTRIBUTES['archive'] ?? 'inherit')); $quarantine_notification = (isset($_data['quarantine_notification'])) ? strval($_data['quarantine_notification']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_notification']); $quarantine_category = (isset($_data['quarantine_category'])) ? strval($_data['quarantine_category']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_category']); // Validate quarantine_category @@ -974,6 +978,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { 'dav_access' => strval($dav_access), 'relayhost' => strval($relayhost), 'shared' => strval($shared), + 'archive' => $archive, 'passwd_update' => time(), 'mailbox_format' => strval($MAILBOX_DEFAULT_ATTRIBUTES['mailbox_format']), 'quarantine_notification' => strval($quarantine_notification), @@ -1525,6 +1530,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $attr['backupmx'] = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0; $attr['relay_all_recipients'] = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0; $attr['relay_unknown_only'] = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0; + $attr['archive'] = (isset($_data['archive']) && intval($_data['archive']) == 1) ? 1 : 0; $attr['dkim_selector'] = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim"; $attr['key_size'] = isset($_data['key_size']) ? intval($_data['key_size']) : 2048; @@ -1600,6 +1606,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $attr["rl_value"] = (!empty($_data['rl_value'])) ? $_data['rl_value'] : ""; $attr["force_pw_update"] = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update']); $attr["force_tfa"] = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_tfa']); + $attr["archive"] = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? $MAILBOX_DEFAULT_ATTRIBUTES['archive']); $attr["sogo_access"] = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['sogo_access']); $attr["active"] = isset($_data['active']) ? intval($_data['active']) : 1; $attr["tls_enforce_in"] = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_in']); @@ -2488,6 +2495,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal']; $description = (!empty($_data['description']) && isset($_SESSION['acl']['domain_desc']) && $_SESSION['acl']['domain_desc'] == "1") ? $_data['description'] : $is_now['description']; (int)$relayhost = (isset($_data['relayhost']) && isset($_SESSION['acl']['domain_relayhost']) && $_SESSION['acl']['domain_relayhost'] == "1") ? intval($_data['relayhost']) : intval($is_now['relayhost']); + $archive = (isset($_data['archive']) && hasACLAccess("archive")) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']); $tags = (is_array($_data['tags']) ? $_data['tags'] : array()); } else { @@ -2501,11 +2509,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $stmt = $pdo->prepare("UPDATE `domain` SET `description` = :description, - `gal` = :gal + `gal` = :gal, + `archive` = :archive WHERE `domain` = :domain"); $stmt->execute(array( ':description' => $description, ':gal' => $gal, + ':archive' => $archive, ':domain' => $domain )); // save tags @@ -2538,6 +2548,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active']; $backupmx = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $is_now['backupmx']; $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal']; + $archive = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']); $relay_all_recipients = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : $is_now['relay_all_recipients']; $relay_unknown_only = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $is_now['relay_unknown_only']; $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : $is_now['relayhost']; @@ -2652,6 +2663,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { `relay_unknown_only` = :relay_unknown_only, `backupmx` = :backupmx, `gal` = :gal, + `archive` = :archive, `active` = :active, `quota` = :quota, `defquota` = :defquota, @@ -2666,6 +2678,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ':relay_unknown_only' => $relay_unknown_only, ':backupmx' => $backupmx, ':gal' => $gal, + ':archive' => $archive, ':active' => $active, ':quota' => $quota, ':defquota' => $defquota, @@ -2753,6 +2766,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $attr['backupmx'] = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0; $attr['relay_all_recipients'] = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0; $attr['relay_unknown_only'] = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0; + $attr['archive'] = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['attributes']['archive'] ?? 0); $attr['dkim_selector'] = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim"; $attr['key_size'] = isset($_data['key_size']) ? intval($_data['key_size']) : 2048; @@ -2814,6 +2828,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { (int)$eas_access = (isset($_data['eas_access']) && hasACLAccess("protocol_access")) ? intval($_data['eas_access']) : intval($is_now['attributes']['eas_access']); (int)$dav_access = (isset($_data['dav_access']) && hasACLAccess("protocol_access")) ? intval($_data['dav_access']) : intval($is_now['attributes']['dav_access']); (int)$relayhost = (isset($_data['relayhost']) && hasACLAccess("mailbox_relayhost")) ? intval($_data['relayhost']) : intval($is_now['attributes']['relayhost']); + $archive = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($is_now['attributes']['archive'] ?? null)); (int)$shared = (isset($_data['shared'])) ? intval($_data['shared']) : intval($is_now['attributes']['shared'] ?? 0); (int)$quota_m = (isset_has_content($_data['quota'])) ? intval($_data['quota']) : ($is_now['quota'] / 1048576); $name = (!empty($_data['name'])) ? ltrim(rtrim($_data['name'], '>'), '<') : $is_now['name']; @@ -3114,6 +3129,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { `attributes` = JSON_SET(`attributes`, '$.smtp_access', :smtp_access), `attributes` = JSON_SET(`attributes`, '$.eas_access', :eas_access), `attributes` = JSON_SET(`attributes`, '$.dav_access', :dav_access), + `attributes` = JSON_SET(`attributes`, '$.archive', :archive), `attributes` = JSON_SET(`attributes`, '$.recovery_email', :recovery_email), `attributes` = JSON_SET(`attributes`, '$.attribute_hash', :attribute_hash) WHERE `username` = :username"); @@ -3131,6 +3147,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ':smtp_access' => $smtp_access, ':eas_access' => $eas_access, ':dav_access' => $dav_access, + ':archive' => $archive, ':recovery_email' => $pw_recovery_email, ':relayhost' => $relayhost, ':shared' => strval($shared), @@ -3534,6 +3551,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $attr["rl_value"] = (!empty($_data['rl_value'])) ? $_data['rl_value'] : $is_now['rl_value']; $attr["force_pw_update"] = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : $is_now['force_pw_update']; $attr["force_tfa"] = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : $is_now['force_tfa']; + $attr["archive"] = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? ($is_now['attributes']['archive'] ?? null)); $attr["sogo_access"] = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : $is_now['sogo_access']; $attr["active"] = isset($_data['active']) ? intval($_data['active']) : $is_now['active']; $attr["tls_enforce_in"] = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : $is_now['tls_enforce_in']; @@ -4682,6 +4700,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { `relay_unknown_only`, `backupmx`, `gal`, + `archive`, `active` FROM `domain` WHERE `domain`= :domain"); $stmt->execute(array( @@ -4742,6 +4761,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $domaindata['backupmx_int'] = $row['backupmx']; $domaindata['gal'] = $row['gal']; $domaindata['gal_int'] = $row['gal']; + $domaindata['archive'] = $row['archive']; $domaindata['rl'] = $rl; $domaindata['active'] = $row['active']; $domaindata['active_int'] = $row['active']; @@ -4829,6 +4849,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { if (preg_match('/y|yes/i', getenv('MASTER'))) { $stmt = $pdo->prepare("SELECT `domain`.`backupmx`, + `domain`.`active` AS `domain_active`, + `domain`.`archive` AS `domain_archive`, `mailbox`.`username`, `mailbox`.`name`, `mailbox`.`active`, @@ -4851,6 +4873,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { else { $stmt = $pdo->prepare("SELECT `domain`.`backupmx`, + `domain`.`active` AS `domain_active`, + `domain`.`archive` AS `domain_archive`, `mailbox`.`username`, `mailbox`.`name`, `mailbox`.`active`, @@ -4890,6 +4914,18 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $mailboxdata['created'] = $row['created']; $mailboxdata['modified'] = $row['modified']; $mailboxdata['authsource'] = ($row['authsource']) ? $row['authsource'] : 'mailcow'; + // Mails of the mailbox go to the archive: global switch, mailbox setting ("inherit" follows the domain), receives mail + static $archive_enabled = null; + if ($archive_enabled === null) { + try { + $archive_enabled = \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(); + } + catch (PDOException $e) { + $archive_enabled = false; + } + } + $mailboxdata['archive'] = ($archive_enabled && $row['active'] != 0 && $row['domain_active'] == 1 && $row['backupmx'] == 0 + && \Mailcow\Archive\Model\Mailbox::effective($mailboxdata['attributes']['archive'] ?? null, $row['domain_archive'])) ? 1 : 0; if ($mailboxdata['percent_in_use'] === '- ') { $mailboxdata['percent_class'] = "info"; diff --git a/data/web/inc/functions.quarantine.inc.php b/data/web/inc/functions.quarantine.inc.php index 55674e528..f45e057e5 100644 --- a/data/web/inc/functions.quarantine.inc.php +++ b/data/web/inc/functions.quarantine.inc.php @@ -154,6 +154,7 @@ function quarantine($_action, $_data = null) { $mail->addAttachment($msg_tmpf, $detail_row['qid'] . '.eml'); $mail->Body = sprintf($lang['quarantine']['release_body']); $mail->send(); + $released_msg = $mail->getSentMIMEMessage(); unlink($msg_tmpf); } catch (phpmailerException $e) { @@ -170,6 +171,7 @@ function quarantine($_action, $_data = null) { } elseif ($release_format == 'raw') { $detail_row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $detail_row['msg']); + $released_msg = $detail_row['msg']; // dot-stuffing per RFC5321 4.5.2: escape leading dots $detail_row['msg'] = preg_replace('~^\.~m', '..', $detail_row['msg']); $postfix_talk = array( @@ -213,6 +215,10 @@ function quarantine($_action, $_data = null) { } fclose($smtp_connection); } + // Port 590 bypasses rspamd, so the archive gets released mails here + if (isset($released_msg)) { + \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($detail_row['rcpt']), 'quarantine'); + } $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE id = :id"); $stmt->execute(array( ':id' => $row['id'] @@ -453,6 +459,7 @@ function quarantine($_action, $_data = null) { $mail->addAttachment($msg_tmpf, $row['qid'] . '.eml'); $mail->Body = sprintf($lang['quarantine']['release_body']); $mail->send(); + $released_msg = $mail->getSentMIMEMessage(); unlink($msg_tmpf); } catch (phpmailerException $e) { @@ -467,6 +474,7 @@ function quarantine($_action, $_data = null) { } elseif ($release_format == 'raw') { $row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $row['msg']); + $released_msg = $row['msg']; // dot-stuffing per RFC5321 4.5.2: escape leading dots $row['msg'] = preg_replace('~^\.~m', '..', $row['msg']); $postfix_talk = array( @@ -506,6 +514,11 @@ function quarantine($_action, $_data = null) { } fclose($smtp_connection); } + // Port 590 bypasses rspamd, so the archive gets released mails here + if (isset($released_msg)) { + \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($row['rcpt']), 'quarantine'); + unset($released_msg); + } $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE `id` = :id"); $stmt->execute(array( ':id' => $id diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index ff95a0b0a..eda3f28f6 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -242,6 +242,7 @@ function init_db_schema() "gal" => "TINYINT(1) NOT NULL DEFAULT '1'", "relay_all_recipients" => "TINYINT(1) NOT NULL DEFAULT '0'", "relay_unknown_only" => "TINYINT(1) NOT NULL DEFAULT '0'", + "archive" => "TINYINT(1) NOT NULL DEFAULT '0'", "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP", "active" => "TINYINT(1) NOT NULL DEFAULT '1'" @@ -1053,7 +1054,8 @@ function init_db_schema() "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'", "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'", "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'", - "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'" + "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'", + "archive" => "TINYINT(1) NOT NULL DEFAULT '0'" ), "keys" => array( "primary" => array( @@ -1277,6 +1279,42 @@ function init_db_schema() ), "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" ), + "archive_settings" => array( + "cols" => array( + "name" => "VARCHAR(64) NOT NULL", + "value" => "TEXT NOT NULL", + "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", + "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP" + ), + "keys" => array( + "primary" => array( + "" => array("name") + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + // Archive target the Postfix maps read (FILTER of the copies on port 591, SASL, TLS policy), one row kept in line with archive_settings + "archive_transport" => array( + "cols" => array( + "id" => "INT NOT NULL AUTO_INCREMENT", + "address" => "VARCHAR(255) NOT NULL", + "nexthop" => "VARCHAR(255) NOT NULL", + "username" => "VARCHAR(255) NOT NULL DEFAULT ''", + "password" => "VARCHAR(255) NOT NULL DEFAULT ''", + "tls_policy" => "VARCHAR(64) NOT NULL DEFAULT 'encrypt'", + "active" => "TINYINT(1) NOT NULL DEFAULT '1'" + ), + "keys" => array( + "primary" => array( + "" => array("id") + ), + "key" => array( + "address" => array("address"), + "nexthop" => array("nexthop") + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), "bcc_maps" => array( "cols" => array( "id" => "INT NOT NULL AUTO_INCREMENT", @@ -2009,6 +2047,7 @@ function init_db_schema() $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.relayhost', \"0\") WHERE JSON_VALUE(`attributes`, '$.relayhost') IS NULL;"); $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.force_pw_update', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_pw_update') IS NULL;"); $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;"); + $pdo->query("UPDATE `mailbox` SET `attributes` = JSON_SET(`attributes`, '$.archive', \"inherit\") WHERE JSON_VALUE(`attributes`, '$.archive') IS NULL;"); // admin attributes $pdo->query("UPDATE `admin` SET `attributes` = '{}' WHERE `attributes` = '' OR `attributes` IS NULL;"); $pdo->query("UPDATE `admin` SET `attributes` = JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;"); @@ -2056,6 +2095,7 @@ function init_db_schema() "backupmx" => 0, "relay_all_recipients" => 0, "relay_unknown_only" => 0, + "archive" => 0, "dkim_selector" => "dkim", "key_size" => 2048, "max_quota_for_domain" => 10240 * 1048576, @@ -2073,6 +2113,7 @@ function init_db_schema() "rl_value" => "", "force_pw_update" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_pw_update']), "force_tfa" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_tfa']), + "archive" => strval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['archive']), "sogo_access" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['sogo_access']), "active" => 1, "tls_enforce_in" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['tls_enforce_in']), diff --git a/data/web/inc/lib/mailcow/Archive/ArchiveModule.php b/data/web/inc/lib/mailcow/Archive/ArchiveModule.php new file mode 100644 index 000000000..96896d598 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/ArchiveModule.php @@ -0,0 +1,65 @@ +api->getMailboxes()); + * ArchiveModule::create($pdo, Context::system(), $redis)->journal->prepare($request); // rspamd dynmap + * ArchiveModule::fromGlobals()->journal->submitDelivered($message, $from, $rcpts, 'quarantine'); + */ +class ArchiveModule +{ + /** Request keys whose values never reach a log */ + private const SECRET_KEYS = ['password', 'signing_secret']; + + public function __construct( + public readonly SettingsService $settings, + public readonly JournalService $journal, + public readonly StatusService $status, + public readonly MailboxListService $mailboxes, + public readonly ArchiveApiController $api + ) { + } + + /** @param ?object $redis Redis connection; without it nothing is read from or written to Redis */ + public static function create(PDO $pdo, Context $context, ?object $redis = null): self + { + $settingsRepo = new SettingsRepository($pdo); + $lookup = new LookupRepository($pdo); + $state = new ArchiveState($redis); + $smtp = new SmtpClient(); + $settings = new SettingsService($settingsRepo, $lookup, $state, $smtp, $context); + $journal = new JournalService($settingsRepo, new OwnerResolver($lookup), new HeaderBuilder(), $smtp, $state, $context); + $status = new StatusService($settingsRepo, $state, new LegacyQueue(), $context); + $mailboxes = new MailboxListService($lookup, $state, $context); + $api = new ArchiveApiController($settings, $status, $journal, $mailboxes, new ApiResponse(self::SECRET_KEYS)); + return new self($settings, $journal, $status, $mailboxes, $api); + } + + /** Legacy entry point for procedural callers: reads $pdo, $redis and $_SESSION; $context overrides the session caller */ + public static function fromGlobals(?Context $context = null): self + { + $redis = $GLOBALS['redis'] ?? null; + return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), is_object($redis) ? $redis : null); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Controller/ArchiveApiController.php b/data/web/inc/lib/mailcow/Archive/Controller/ArchiveApiController.php new file mode 100644 index 000000000..201805a2c --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Controller/ArchiveApiController.php @@ -0,0 +1,103 @@ +read(fn(): array => $this->settings->get()->toArray()); + } + + /** GET archive/status */ + public function getStatus(): array + { + return $this->read(fn(): array => $this->status->status()); + } + + /** GET logs/archive[/count] */ + public function getLog(int $count = 0): array + { + return $this->read(fn(): array => $this->status->log($count)); + } + + /** GET archive/mailboxes */ + public function getMailboxes(): array + { + return $this->read(fn(): array => $this->mailboxes->archived()); + } + + /** POST edit/archive: the settings; `action: test` tests the connection, `action: testmail` sends a test copy */ + public function edit(array $attr): bool + { + return match ($attr['action'] ?? null) { + 'test' => $this->handle('edit', 'test', [], fn(array $log): bool => $this->testResult($this->settings->test(), $log)), + 'testmail' => $this->handle('edit', 'testmail', [], + fn(array $log): bool => $this->response->success(['archive_testmail_queued', htmlspecialchars($this->journal->testMail())], $log)), + default => $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool { + $this->settings->save($attr); + return $this->response->success('archive_settings_saved', $log); + }), + }; + } + + /** A missing extension is a warning: the archive works, but may refuse some mails */ + private function testResult(string|array $msg, array $log): bool + { + if (is_array($msg)) { + $this->response->warning($msg, $log); + return true; + } + return $this->response->success($msg, $log); + } + + /** A GET answer, empty when the caller may not read it */ + private function read(callable $work): array + { + try { + return $work(); + } catch (UserError) { + return []; + } + } + + /** + * Runs one write call: $work gets the log entry and returns the result for process_*_return(). + * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message. + */ + private function handle(string $action, string $object, mixed $data, callable $work): bool + { + $log = ['archive', $action, $object, $data]; + try { + $result = $work($log); + } catch (UserError $e) { + $result = $this->response->error($e, $log); + } catch (PDOException $e) { + $result = $this->response->danger(['mysql_error', $e->getMessage()], $log); + } + $this->response->flush(); + return $result; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Legacy/LegacyQueue.php b/data/web/inc/lib/mailcow/Archive/Legacy/LegacyQueue.php new file mode 100644 index 000000000..184996434 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Legacy/LegacyQueue.php @@ -0,0 +1,27 @@ + is_array($entry) && ($entry['sender'] ?? '') === $sender)); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Model/Journal.php b/data/web/inc/lib/mailcow/Archive/Model/Journal.php new file mode 100644 index 000000000..c3c8b45d1 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Model/Journal.php @@ -0,0 +1,32 @@ +archive, $this->domainArchive); + } + + /** Why the mailbox is archived: its own setting or the domain default */ + public function source(): string + { + return ($this->archive === '1') ? 'mailbox' : 'domain'; + } + + /** Entry of GET archive/mailboxes */ + public function toArray(): array + { + return [ + 'username' => $this->username, + 'domain' => $this->domain, + 'name' => $this->name, + 'active' => $this->active, + 'source' => $this->source(), + ]; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Model/Settings.php b/data/web/inc/lib/mailcow/Archive/Model/Settings.php new file mode 100644 index 000000000..50fca3d17 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Model/Settings.php @@ -0,0 +1,72 @@ +address !== '' && ($this->profile !== 'archivecow' || $this->signingKey !== ''); + } + + /** Next hop for Postfix: [host]:port, else the domain of the address (MX lookup), with the port when not 25 */ + public function nexthop(): string + { + if ($this->host !== '') { + $host = filter_var($this->host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6:' . $this->host : $this->host; + return '[' . $host . ']:' . $this->port; + } + $domain = substr($this->address, (int)strrpos($this->address, '@') + 1); + return ($this->port === 25) ? $domain : $domain . ':' . $this->port; + } + + /** API representation (GET archive/settings), without secrets */ + public function toArray(): array + { + return [ + 'enabled' => (int)$this->enabled, + 'address' => $this->address, + 'host' => $this->host, + 'port' => $this->port, + 'tls' => $this->tls, + 'username' => $this->username, + 'has_password' => (int)($this->password !== ''), + 'profile' => $this->profile, + 'has_signing_secret' => (int)($this->signingKey !== ''), + 'header_from' => $this->headerFrom, + 'header_rcpt' => $this->headerRcpt, + ]; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Redis/ArchiveState.php b/data/web/inc/lib/mailcow/Archive/Redis/ArchiveState.php new file mode 100644 index 000000000..ea6846009 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Redis/ArchiveState.php @@ -0,0 +1,118 @@ +redis?->set(self::ENABLED, $enabled ? '1' : '0'); + } + + /** The global switch: on only while ARCHIVE_ENABLED is '1', as rspamd reads it (missing or unreadable is off) */ + public function enabled(): bool + { + try { + return $this->redis?->get(self::ENABLED) === '1'; + } catch (RuntimeException) { + return false; + } + } + + /** A mail was not archived: its copy could not be handed to Postfix */ + public function dropped(string $qid, string $reason, string $messageId = ''): void + { + if ($this->redis === null) { + return; + } + $this->redis->incr(self::DROPPED); + $this->redis->lPush(self::DROPPED_LOG, json_encode(['time' => time(), 'qid' => $qid, 'reason' => $reason, 'message_id' => $messageId])); + $this->redis->lTrim(self::DROPPED_LOG, 0, 99); + } + + /** @return array{count: int, recent: array[]} count since ever; recent: time, qid, reason, message_id; newest first */ + public function droppedStats(int $limit = 100): array + { + if ($this->redis === null) { + return ['count' => 0, 'recent' => []]; + } + $recent = []; + foreach ((array)$this->redis->lRange(self::DROPPED_LOG, 0, $limit - 1) as $line) { + $entry = is_string($line) ? json_decode($line, true) : null; + if (is_array($entry)) { + $recent[] = ['time' => (int)($entry['time'] ?? 0), 'qid' => (string)($entry['qid'] ?? ''), + 'reason' => (string)($entry['reason'] ?? ''), 'message_id' => (string)($entry['message_id'] ?? '')]; + } + } + return ['count' => (int)$this->redis->get(self::DROPPED), 'recent' => $recent]; + } + + /** + * Archive log, newest first: deliveries of copies (status sent|deferred|softbounce|bounced, text = answer of the + * archive), expired copies (expired) and mails not archived (dropped, text = reason). + * + * @return array[] time, status, qid (of the copy, of the mail for dropped), message_id, text + */ + public function events(int $limit = 1000): array + { + if ($this->redis === null) { + return []; + } + $events = []; + $messageIds = []; + foreach ((array)$this->redis->lRange(self::MAILLOG, 0, -1) as $line) { + $entry = is_string($line) ? json_decode($line, true) : null; + if (!is_array($entry)) { + continue; + } + $program = (string)($entry['program'] ?? ''); + $message = (string)($entry['message'] ?? ''); + $time = (int)($entry['time'] ?? 0); + if ($program === 'postfix/cleanup' && preg_match('/^([0-9A-Za-z]+): message-id=(\S*)/', $message, $m)) { + $messageIds[$m[1]] = trim($m[2], '<>'); + } + // the archive_discard lines are the dropped notices, not deliveries + elseif ($program === 'postfix/archive/smtp' && preg_match('/^([0-9A-Za-z]+): to=<[^>]*>.*\bstatus=(\w+) \((.*)\)$/', $message, $m)) { + $events[] = ['time' => $time, 'status' => strtolower($m[2]), 'qid' => $m[1], 'message_id' => '', 'text' => $m[3]]; + } + elseif ($program === 'postfix/qmgr' && preg_match('/^([0-9A-Za-z]+): from=<' . preg_quote(Settings::SENDER, '/') . '>, status=expired/', $message, $m)) { + $events[] = ['time' => $time, 'status' => 'expired', 'qid' => $m[1], 'message_id' => '', 'text' => '']; + } + } + foreach ($events as &$event) { + $event['message_id'] = $messageIds[$event['qid']] ?? ''; + } + unset($event); + foreach ($this->droppedStats()['recent'] as $dropped) { + $events[] = ['time' => $dropped['time'], 'status' => 'dropped', 'qid' => $dropped['qid'], + 'message_id' => $dropped['message_id'], 'text' => $dropped['reason']]; + } + usort($events, fn(array $a, array $b): int => $b['time'] <=> $a['time']); + return array_slice($events, 0, $limit); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Repository/LookupRepository.php b/data/web/inc/lib/mailcow/Archive/Repository/LookupRepository.php new file mode 100644 index 000000000..3aed0d38c --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Repository/LookupRepository.php @@ -0,0 +1,94 @@ +value("SELECT `new_dest` FROM `recipient_maps` WHERE `old_dest` = :key AND `active` = '1' LIMIT 1", $key); + } + + /** Targets of an alias row (aliases, catch-alls '@domain', groups, mailbox self rows) */ + public function aliasGoto(string $key): ?string + { + return $this->value("SELECT `goto` FROM `alias` WHERE `address` = :key AND (`active` = '1' OR `active` = '2') LIMIT 1", $key); + } + + /** Resources (location, thing, group) are discarded by Postfix */ + public function isResource(string $username): bool + { + return $this->value("SELECT `username` FROM `mailbox` WHERE `username` = :key AND `kind` REGEXP 'location|thing|group' LIMIT 1", $username) !== null; + } + + public function spamAliasGoto(string $key): ?string + { + return $this->value("SELECT `goto` FROM `spamalias` WHERE `address` = :key + AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0) LIMIT 1", $key); + } + + /** Mailbox local@target behind local@domain when domain is an alias domain */ + public function aliasDomainMailbox(string $local, string $domain): ?string + { + $username = $this->query("SELECT `mailbox`.`username` FROM `mailbox`, `alias_domain` + WHERE `alias_domain`.`alias_domain` = :domain AND `mailbox`.`username` = CONCAT(:local, '@', `alias_domain`.`target_domain`) + AND (`mailbox`.`active` = '1' OR `mailbox`.`active` = '2') AND `alias_domain`.`active` = '1' LIMIT 1", + ['domain' => $domain, 'local' => $local])->fetchColumn(); + return is_string($username) ? $username : null; + } + + /** BCC destination of an address or '@domain' as rspamd's bcc.php finds it; $type 'rcpt' or 'sender' */ + public function bccDestination(string $key, string $type): ?string + { + $dest = $this->query("SELECT `bcc_dest` FROM `bcc_maps` WHERE `type` = :type AND `local_dest` = :key AND `active` = '1' LIMIT 1", + ['type' => $type, 'key' => $key])->fetchColumn(); + return is_string($dest) ? $dest : null; + } + + /** A domain or alias domain of this mailcow */ + public function isMailcowDomain(string $domain): bool + { + return $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain + UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :alias LIMIT 1", + ['domain' => $domain, 'alias' => $domain])->fetchColumn() !== false; + } + + /** The mailbox when it receives mail */ + public function mailbox(string $username): ?Mailbox + { + $row = $this->query(self::MAILBOX . " AND m.`username` = :username", ['username' => $username])->fetch(); + return is_array($row) ? Mailbox::fromRow($row) : null; + } + + /** @return Mailbox[] mailboxes archived by their setting or their domain (without the global switch) */ + public function archivedMailboxes(): array + { + $rows = $this->query(self::MAILBOX . " AND (JSON_VALUE(m.`attributes`, '$.archive') = '1' + OR (COALESCE(JSON_VALUE(m.`attributes`, '$.archive'), '') NOT IN ('0', '1') AND d.`archive` = '1')) + ORDER BY m.`username`")->fetchAll(); + return array_map(fn(array $row): Mailbox => Mailbox::fromRow($row), $rows); + } + + private function value(string $sql, string $key): ?string + { + $value = $this->query($sql, ['key' => $key])->fetchColumn(); + return is_string($value) ? $value : null; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Repository/SettingsRepository.php b/data/web/inc/lib/mailcow/Archive/Repository/SettingsRepository.php new file mode 100644 index 000000000..4b375d0e3 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Repository/SettingsRepository.php @@ -0,0 +1,71 @@ + value) and the row the Postfix maps + * read (table `archive_transport`: transport, SASL, TLS policy of the archive_smtp service). + */ +class SettingsRepository extends Repository +{ + /** Stored settings; keys that are not stored yet keep the defaults of the model */ + public function load(): Settings + { + $rows = $this->query("SELECT `name`, `value` FROM `archive_settings`")->fetchAll(PDO::FETCH_KEY_PAIR); + $settings = new Settings(); + $settings->address = (string)($rows['address'] ?? ''); + $settings->host = (string)($rows['host'] ?? ''); + $settings->port = (int)($rows['port'] ?? $settings->port); + $settings->tls = (string)($rows['tls'] ?? $settings->tls); + $settings->username = (string)($rows['username'] ?? ''); + $settings->password = (string)($rows['password'] ?? ''); + $settings->profile = (string)($rows['profile'] ?? $settings->profile); + $settings->signingKey = (string)($rows['signing_secret'] ?? ''); + $settings->headerFrom = (string)($rows['header_from'] ?? $settings->headerFrom); + $settings->headerRcpt = (string)($rows['header_rcpt'] ?? $settings->headerRcpt); + return $settings; + } + + /** + * Saves the settings and the transport row. The row stays while an address is set, also with archiving off, + * so copies still queued in Postfix are delivered. + */ + public function save(Settings $settings): void + { + $values = [ + 'address' => $settings->address, + 'host' => $settings->host, + 'port' => (string)$settings->port, + 'tls' => $settings->tls, + 'username' => $settings->username, + 'password' => $settings->password, + 'profile' => $settings->profile, + 'signing_secret' => $settings->signingKey, + 'header_from' => $settings->headerFrom, + 'header_rcpt' => $settings->headerRcpt, + ]; + $sql = "INSERT INTO `archive_settings` (`name`, `value`) VALUES (:name, :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)"; + $this->transaction(function () use ($sql, $values, $settings): void { + foreach ($values as $name => $value) { + $this->query($sql, ['name' => $name, 'value' => $value]); + } + $this->query("DELETE FROM `archive_transport`"); + if ($settings->address !== '') { + $this->insertRow('archive_transport', [ + 'address' => $settings->address, + 'nexthop' => $settings->nexthop(), + 'username' => $settings->username, + 'password' => $settings->password, + 'tls_policy' => $settings->tls, + 'active' => 1, + ]); + } + }); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/HeaderBuilder.php b/data/web/inc/lib/mailcow/Archive/Service/HeaderBuilder.php new file mode 100644 index 000000000..c79157292 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/HeaderBuilder.php @@ -0,0 +1,125 @@ +. The signature is HMAC-SHA256 with the signing key over + * the lines above it, each as ":", joined by "\n". + * The block ends with the signature line, everything after it is the mail as accepted (CRLF line ends). + * - envelope: SMTP sender and recipients under configurable names (default X-Envelope-From / X-Envelope-To). + * - none: no headers. + * + * Empty values are left out. Lists are folded after commas (CRLF + space). + */ +class HeaderBuilder +{ + /** Fold a list before a line would get longer */ + private const FOLD_AT = 78; + + public function build(Settings $settings, Journal $journal): string + { + return match ($settings->profile) { + 'archivecow' => $this->archivecow($settings, $journal), + 'envelope' => $this->envelope($settings, $journal), + default => '', + }; + } + + /** One line of the signed text */ + public static function canonical(string $name, string $value): string + { + return strtolower($name) . ':' . preg_replace('/\s+/', '', $value); + } + + private function archivecow(Settings $settings, Journal $journal): string + { + $values = [ + 'X-Archive-Owner' => $journal->owners, + 'X-Archive-Rcpt' => $journal->rcpts, + 'X-Archive-From' => $journal->from, + 'X-Archive-Queue-ID' => $journal->qid, + 'X-Archive-Received' => self::time($journal->received), + 'X-Archive-Direction' => $journal->direction, + 'X-Archive-Source' => $journal->source, + 'X-Archive-SHA256' => $journal->sha256, + ]; + $lines = []; + $signed = []; + foreach ($values as $name => $value) { + $value = is_array($value) ? self::cleanList($value) : self::clean($value); + if ($value === '' || $value === []) { + continue; + } + $lines[] = self::line($name, $value); + $signed[] = self::canonical($name, is_array($value) ? implode(',', $value) : $value); + } + $lines[] = 'X-Archive-Signature: v=1; a=hmac-sha256; s=' . hash_hmac('sha256', implode("\n", $signed), $settings->signingKey); + return implode("\r\n", $lines) . "\r\n"; + } + + private function envelope(Settings $settings, Journal $journal): string + { + $from = self::clean($journal->from); + $lines = [self::line($settings->headerFrom, ($from === '') ? '<>' : $from)]; + $rcpts = self::cleanList($journal->rcpts); + if ($rcpts !== []) { + $lines[] = self::line($settings->headerRcpt, $rcpts); + } + return implode("\r\n", $lines) . "\r\n"; + } + + /** Header line; a list is folded after commas */ + private static function line(string $name, string|array $value): string + { + if (is_string($value)) { + return $name . ': ' . $value; + } + $out = $name . ': ' . array_shift($value); + $length = strlen($out); + foreach ($value as $item) { + if ($length + strlen($item) + 1 > self::FOLD_AT) { + $out .= ",\r\n " . $item; + $length = strlen($item) + 1; + } else { + $out .= ',' . $item; + $length += strlen($item) + 1; + } + } + return $out; + } + + /** Header value without control characters (no header injection) */ + private static function clean(string $value): string + { + return trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', $value)); + } + + /** @return string[] */ + private static function cleanList(array $values): array + { + $list = []; + foreach ($values as $value) { + $value = is_string($value) ? str_replace(',', '', self::clean($value)) : ''; + if ($value !== '') { + $list[] = $value; + } + } + return $list; + } + + /** RFC 3339 in UTC with milliseconds */ + private static function time(float $time): string + { + $date = DateTime::createFromFormat('U.u', sprintf('%.3F', $time)); + return $date ? $date->format('Y-m-d\TH:i:s.v\Z') : ''; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/JournalService.php b/data/web/inc/lib/mailcow/Archive/Service/JournalService.php new file mode 100644 index 000000000..08de41556 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/JournalService.php @@ -0,0 +1,144 @@ +settings->load(); + if (!$this->state->enabled() || !$settings->canDeliver()) { + return ['archive' => false]; + } + $from = self::text($request['from'] ?? ''); + $rcpts = is_array($request['rcpt'] ?? null) ? array_values(array_filter($request['rcpt'], 'is_string')) : []; + $resolved = $this->resolver->resolve($from, $rcpts, self::text($request['user'] ?? ''), ($request['bcc'] ?? false) === true); + if ($resolved['owners'] === []) { + return ['archive' => false]; + } + $sha256 = self::text($request['sha256'] ?? ''); + $journal = new Journal($resolved['owners'], $rcpts, $from, mb_strcut(self::text($request['qid'] ?? ''), 0, 64), + $resolved['direction'], 'smtp', is_numeric($request['received'] ?? null) ? (float)$request['received'] : microtime(true), + preg_match('/^[0-9a-f]{64}$/', $sha256) ? $sha256 : ''); + return ['archive' => true, 'rcpt' => $settings->address, 'sender' => Settings::SENDER, 'headers' => $this->headers->build($settings, $journal)]; + } + + /** + * Sends the copy of a mail that reached its mailbox without rspamd (quarantine release, mass mail) for its + * archived recipients. Errors are logged and counted, never thrown: the mail is delivered already. + * + * @param string[] $rcpts + */ + public function submitDelivered(string $message, string $from, array $rcpts, string $source): bool + { + try { + $settings = $this->settings->load(); + if (!$this->state->enabled() || !$settings->canDeliver()) { + return false; + } + $resolved = $this->resolver->resolve($from, $rcpts, '', false); + if ($resolved['owners'] === []) { + return false; + } + $this->send($settings, $message, $resolved['owners'], array_values($rcpts), $from, 'in', $source); + return true; + } catch (Throwable $e) { + error_log('ARCHIVE: cannot send the archive copy (' . $source . '): ' . $e->getMessage()); + $this->state->dropped('', $source . ': ' . $e->getMessage(), self::messageId($message)); + return false; + } + } + + /** + * Sends a test copy through Postfix to the archive and returns the Postfix queue id. + * + * @throws UserError + */ + public function testMail(): string + { + $this->context->requireAdmin(); + $settings = $this->settings->load(); + if (!$settings->canDeliver()) { + throw new UserError('archive_incomplete'); + } + $message = implode("\r\n", [ + 'From: mailcow <' . Settings::SENDER . '>', + 'To: <' . $settings->address . '>', + 'Subject: mailcow archive test', + 'Date: ' . date(DATE_RFC2822), + 'Message-ID: <' . bin2hex(random_bytes(12)) . '@archive-test.mailcow>', + 'MIME-Version: 1.0', + 'Content-Type: text/plain; charset=utf-8', + '', + 'Test copy from mailcow (System - Configuration - Archive).', + '', + ]); + try { + return $this->send($settings, $message, [], [$settings->address], Settings::SENDER, 'in', 'test'); + } catch (SmtpException $e) { + throw new UserError('archive_testmail_failed', [htmlspecialchars($e->describe())]); + } + } + + /** @throws SmtpException */ + private function send(Settings $settings, string $message, array $owners, array $rcpts, string $from, string $direction, string $source): string + { + $message = SmtpClient::crlf($message); + $journal = new Journal($owners, $rcpts, $from, '', $direction, $source, microtime(true), hash('sha256', $message)); + return $this->smtp->submit($this->postfixHost, self::SUBMIT_PORT, Settings::SENDER, $settings->address, + $this->headers->build($settings, $journal) . $message); + } + + /** Message-ID of a mail, for the archive log */ + private static function messageId(string $message): string + { + $headers = explode("\n\n", str_replace("\r\n", "\n", $message), 2)[0]; + return preg_match('/^Message-ID:\s*\s]+)>?/mi', $headers, $m) ? $m[1] : ''; + } + + private static function text(mixed $value): string + { + return is_scalar($value) ? trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', (string)$value)) : ''; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/MailboxListService.php b/data/web/inc/lib/mailcow/Archive/Service/MailboxListService.php new file mode 100644 index 000000000..0603f2d1b --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/MailboxListService.php @@ -0,0 +1,38 @@ +context->requireAdmin(false); + $enabled = $this->state->enabled(); + return [ + 'enabled' => (int)$enabled, + 'mailboxes' => $enabled ? array_map(fn(Mailbox $mailbox): array => $mailbox->toArray(), $this->lookup->archivedMailboxes()) : [], + ]; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/OwnerResolver.php b/data/web/inc/lib/mailcow/Archive/Service/OwnerResolver.php new file mode 100644 index 000000000..7ae419050 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/OwnerResolver.php @@ -0,0 +1,264 @@ +deliver($rcpt, $incoming); + } + } + if ($bcc) { + foreach ($this->bccDestinations($from, $rcpts) as $destination) { + $this->deliver($destination, $incoming); + } + } + $outgoing = $this->senders($from, self::address($user)); + $owners = []; + foreach (array_merge(array_values($incoming), array_values($outgoing)) as $mailbox) { + if ($mailbox->archived()) { + $owners[$mailbox->username] = true; + } + } + $owners = array_keys($owners); + sort($owners); + $direction = empty($outgoing) ? 'in' : (empty($incoming) ? 'out' : 'internal'); + return ['owners' => $owners, 'direction' => $direction]; + } + + /** Adds the local mailboxes a recipient ends in to $mailboxes (username => Mailbox) */ + private function deliver(string $address, array &$mailboxes): void + { + $seen = []; + $this->expand($this->canonical($address), 0, $mailboxes, $seen); + } + + /** recipient_maps: first matching key form, one rewrite */ + private function canonical(string $address): string + { + foreach (self::keys($address) as $key) { + $target = self::target($this->lookup->recipientMap($key)); + if ($target !== '') { + return self::rewrite($address, $key, $target); + } + } + return $address; + } + + private function expand(string $address, int $depth, array &$mailboxes, array &$seen): void + { + if ($depth > self::MAX_DEPTH || count($seen) >= self::MAX_ADDRESSES || isset($seen[$address])) { + return; + } + $seen[$address] = true; + $targets = $this->virtual($address); + if ($targets === null) { + $this->final($address, $mailboxes); + return; + } + foreach ($targets as $target) { + // an address that expands to itself is delivered as it is + if ($target === $address) { + $this->final($address, $mailboxes); + } else { + $this->expand($target, $depth + 1, $mailboxes, $seen); + } + } + } + + /** Targets from the first key form a virtual alias map knows, null if none does */ + private function virtual(string $address): ?array + { + if (self::split($address)[1] === 'localhost') { + // null@, spam@, ham@localhost: local delivery of Postfix, never a mailbox + return []; + } + foreach (self::keys($address) as $key) { + $targets = $this->virtualLookup($key); + if ($targets !== null) { + return array_map(fn(string $target): string => self::rewrite($address, $key, $target), $targets); + } + } + return null; + } + + /** virtual_alias_maps in the order of main.cf: alias, resource, spam alias, alias domain mailbox */ + private function virtualLookup(string $key): ?array + { + $domainKey = str_starts_with($key, '@'); + $targets = self::targets($this->lookup->aliasGoto($key)); + if ($targets === null && !$domainKey && $this->lookup->isResource($key)) { + $targets = ['null@localhost']; + } + $targets ??= self::targets($this->lookup->spamAliasGoto($key)); + if ($targets === null && !$domainKey) { + [$local, $domain] = self::split($key); + $targets = self::targets($this->lookup->aliasDomainMailbox($local, $domain)); + } + return $targets; + } + + /** A final address: a mailbox (as u+ext@d or u@d) or anything else, which is not ours */ + private function final(string $address, array &$mailboxes): void + { + foreach (array_unique([$address, self::untagged($address)]) as $candidate) { + $mailbox = $this->lookup->mailbox($candidate); + if ($mailbox !== null) { + $mailboxes[$mailbox->username] = $mailbox; + return; + } + } + } + + /** BCC destinations of rspamd: rcpt maps for every recipient and its domain, sender maps for MAIL FROM and its domain */ + private function bccDestinations(string $from, array $rcpts): array + { + $destinations = []; + foreach ($rcpts as $rcpt) { + $rcpt = self::address($rcpt); + if ($rcpt !== '') { + $destinations[] = $this->lookup->bccDestination(self::untagged($rcpt), 'rcpt'); + $destinations[] = $this->lookup->bccDestination('@' . self::split($rcpt)[1], 'rcpt'); + } + } + if ($from !== '') { + $destinations[] = $this->lookup->bccDestination(self::untagged($from), 'sender'); + $destinations[] = $this->lookup->bccDestination('@' . self::split($from)[1], 'sender'); + } + return array_values(array_unique(array_filter(array_map(fn(?string $address): string => self::address($address), $destinations)))); + } + + /** Mailboxes on the sending side: the SASL user and the mailbox the user sent as */ + private function senders(string $from, string $user): array + { + $mailbox = ($user !== '') ? $this->lookup->mailbox($user) : null; + if ($mailbox === null) { + return []; + } + $senders = [$mailbox->username => $mailbox]; + $sentAs = ($from !== '') ? $this->sentAs(self::untagged($from)) : null; + if ($sentAs !== null) { + $senders[$sentAs->username] = $sentAs; + } + return $senders; + } + + /** Mailbox a sender address belongs to: the mailbox itself or the mailbox behind an alias domain address, not aliases */ + private function sentAs(string $address): ?Mailbox + { + $mailbox = $this->lookup->mailbox($address); + if ($mailbox === null) { + [$local, $domain] = self::split($address); + $username = $this->lookup->aliasDomainMailbox($local, $domain); + $mailbox = ($username !== null) ? $this->lookup->mailbox(strtolower($username)) : null; + } + return $mailbox; + } + + /** Lookup keys in the order of Postfix: u+ext@d, u@d, @d */ + private static function keys(string $address): array + { + $keys = [$address, self::untagged($address), '@' . self::split($address)[1]]; + return array_values(array_unique($keys)); + } + + /** + * Result of a lookup for $address under $key: '@domain' keeps the local part, a match without the extension + * passes it on (propagate_unmatched_extensions). + */ + private static function rewrite(string $address, string $key, string $target): string + { + [$local] = self::split($address); + if (str_starts_with($target, '@')) { + return $local . $target; + } + $extension = self::extension($local); + if ($key === $address || $extension === '') { + return $target; + } + [$targetLocal, $targetDomain] = self::split($target); + return ($targetDomain === 'localhost' || self::extension($targetLocal) !== '') + ? $target + : $targetLocal . '+' . $extension . '@' . $targetDomain; + } + + /** Comma separated lookup result; null when empty (not found) */ + private static function targets(?string $value): ?array + { + $targets = array_values(array_filter(array_map(fn(string $target): string => self::target($target), explode(',', (string)$value)))); + return empty($targets) ? null : $targets; + } + + /** Address or '@domain' from a lookup result, lower case; '' if invalid */ + private static function target(?string $value): string + { + $value = strtolower(trim((string)$value)); + return preg_match('/^[^@\s,<>]*@[^@\s,<>]+$/', $value) ? $value : ''; + } + + /** Lower case address without angle brackets; '' unless local@domain */ + private static function address(mixed $value): string + { + $value = is_string($value) ? strtolower(trim($value, " \t\r\n<>")) : ''; + return preg_match('/^[^@\s,<>]+@[^@\s,<>]+$/', $value) ? $value : ''; + } + + private static function untagged(string $address): string + { + [$local, $domain] = self::split($address); + $pos = strpos($local, '+'); + return ($pos === false || $pos === 0) ? $address : substr($local, 0, $pos) . '@' . $domain; + } + + /** Address extension after the recipient delimiter "+" */ + private static function extension(string $local): string + { + $pos = strpos($local, '+'); + return ($pos === false || $pos === 0) ? '' : substr($local, $pos + 1); + } + + /** [local part, domain] of an address or '@domain' */ + private static function split(string $address): array + { + $at = strrpos($address, '@'); + return ($at === false) ? [$address, ''] : [substr($address, 0, $at), substr($address, $at + 1)]; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/SettingsService.php b/data/web/inc/lib/mailcow/Archive/Service/SettingsService.php new file mode 100644 index 000000000..6a8bbe636 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/SettingsService.php @@ -0,0 +1,164 @@ +context->requireAdmin(false); + $settings = $this->repo->load(); + $settings->enabled = $this->state->enabled(); + return $settings; + } + + /** Global switch for internal callers (page templates, mailbox details), no permission check */ + public function enabled(): bool + { + return $this->state->enabled(); + } + + /** + * Saves the settings: attributes missing in $input keep their value, an empty password or signing key keeps + * the stored one, `password_remove` deletes the password. Switching on needs an address (and a signing key for archivecow). + * + * @throws UserError + */ + public function save(array $input): Settings + { + $this->context->requireAdmin(); + $settings = $this->repo->load(); + $settings->enabled = $this->state->enabled(); + if (array_key_exists('address', $input)) { + $address = strtolower((string)Input::text($input['address'])); + if (!is_scalar($input['address']) || ($address !== '' && !filter_var($address, FILTER_VALIDATE_EMAIL))) { + throw new UserError('archive_address_invalid'); + } + // Postfix would treat an own address as local (aliases, unknown users) before the copy reaches the archive + if ($address !== '' && $this->lookup->isMailcowDomain(substr($address, strrpos($address, '@') + 1))) { + throw new UserError('archive_address_local'); + } + $settings->address = $address; + } + if (array_key_exists('host', $input)) { + $host = strtolower(trim((string)Input::text($input['host']), " []")); + if (!is_scalar($input['host']) || ($host !== '' && !filter_var($host, FILTER_VALIDATE_IP) + && !filter_var($host, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME))) { + throw new UserError('archive_host_invalid'); + } + $settings->host = $host; + } + if (array_key_exists('port', $input)) { + $settings->port = Input::intInRange($input['port'], 1, 65535) ?? throw new UserError('archive_port_invalid'); + } + if (array_key_exists('tls', $input)) { + $settings->tls = in_array($input['tls'], Settings::TLS_MODES, true) ? $input['tls'] : throw new UserError('archive_tls_invalid'); + } + if (array_key_exists('username', $input)) { + $username = Input::text($input['username']); + if ($username === null || strlen($username) > self::MAX_SECRET_LENGTH || preg_match('/[\s:]/', $username)) { + throw new UserError('archive_username_invalid'); + } + $settings->username = $username; + } + if (array_key_exists('password', $input)) { + $settings->password = self::secret($input['password'], $settings->password, 'archive_password_invalid'); + } + if (self::flag($input['password_remove'] ?? '0')) { + $settings->password = ''; + } + if (array_key_exists('profile', $input)) { + $settings->profile = in_array($input['profile'], Settings::PROFILES, true) ? $input['profile'] : throw new UserError('archive_profile_invalid'); + } + if (array_key_exists('signing_secret', $input)) { + $settings->signingKey = self::secret($input['signing_secret'], $settings->signingKey, 'archive_signing_secret_invalid'); + } + foreach (['header_from' => 'headerFrom', 'header_rcpt' => 'headerRcpt'] as $key => $property) { + if (array_key_exists($key, $input)) { + $name = Input::text($input[$key]); + if ($name === null || !preg_match('/^[A-Za-z][A-Za-z0-9-]{0,63}$/', $name)) { + throw new UserError('archive_header_invalid'); + } + $settings->$property = $name; + } + } + if (array_key_exists('enabled', $input)) { + $settings->enabled = self::flag($input['enabled']); + } + if ($settings->enabled && !$settings->canDeliver()) { + throw new UserError('archive_incomplete'); + } + $this->repo->save($settings); + $this->state->setEnabled($settings->enabled); + return $settings; + } + + /** + * Tests the stored target: connection, TLS, AUTH and whether the archive accepts the address (no mail is sent). + * Returns the success message, with the missing extension when the archive lacks 8BITMIME. + * + * @throws UserError + */ + public function test(): string|array + { + $this->context->requireAdmin(); + $settings = $this->repo->load(); + if ($settings->address === '') { + throw new UserError('archive_incomplete'); + } + $host = ($settings->host !== '') ? $settings->host : $this->smtp->mailHost(substr($settings->address, strrpos($settings->address, '@') + 1)); + try { + $extensions = $this->smtp->probe($host, $settings->port, $settings->tls, $settings->username, $settings->password, + Settings::SENDER, $settings->address); + } catch (SmtpException $e) { + throw new UserError('archive_test_failed', [htmlspecialchars($host . ':' . $settings->port . ' ' . $e->describe())]); + } + $missing = array_values(array_diff(self::WANTED_EXTENSIONS, $extensions)); + return empty($missing) ? 'archive_test_ok' : ['archive_test_ok_missing', implode(', ', $missing)]; + } + + /** A new secret, or the stored one when $value is empty */ + private static function secret(mixed $value, string $current, string $error): string + { + $secret = Input::text($value); + if ($secret === null || strlen($secret) > self::MAX_SECRET_LENGTH || preg_match('/[^\x21-\x7e]/', $secret)) { + throw new UserError($error); + } + return ($secret === '') ? $current : $secret; + } + + /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */ + private static function flag(mixed $value): bool + { + if (is_array($value)) { + $value = end($value); + } + return filter_var($value, FILTER_VALIDATE_BOOLEAN); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Service/StatusService.php b/data/web/inc/lib/mailcow/Archive/Service/StatusService.php new file mode 100644 index 000000000..c240c2482 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Service/StatusService.php @@ -0,0 +1,85 @@ +context->requireAdmin(false); + $settings = $this->settings->load(); + $copies = $this->queue->entries(Settings::SENDER); + $reasons = []; + $oldest = null; + $bytes = 0; + foreach ($copies as $copy) { + $arrival = (int)($copy['arrival_time'] ?? 0); + $oldest = ($oldest === null) ? $arrival : min($oldest, $arrival); + $bytes += (int)($copy['message_size'] ?? 0); + foreach ((array)($copy['recipients'] ?? []) as $rcpt) { + if (is_string($rcpt) && preg_match('/^\S+ \((.+)\)$/', $rcpt, $m)) { + $reasons[$m[1]] = true; + } + } + } + $dropped = $this->state->droppedStats(); + $recent = array_values(array_filter($dropped['recent'], fn(array $d): bool => $d['time'] >= time() - self::DROPPED_WINDOW)); + $lastDelivery = null; + foreach ($this->state->events() as $event) { + if ($event['status'] === 'sent') { + $lastDelivery = $event['time']; + break; + } + } + return [ + 'enabled' => (int)$this->state->enabled(), + 'configured' => (int)$settings->canDeliver(), + 'queued' => count($copies), + 'queued_bytes' => $bytes, + 'oldest_queued' => $oldest, + 'delay_reasons' => array_slice(array_keys($reasons), 0, 5), + 'last_delivery' => $lastDelivery, + 'dropped' => $dropped['count'], + 'dropped_24h' => count($recent), + 'last_dropped' => $recent ? ['time' => $recent[0]['time'], 'reason' => $recent[0]['reason']] : null, + ]; + } + + /** + * GET logs/archive: deliveries, expired copies and mails not archived, newest first (admins, read-only keys included); + * $count limits the entries, 0 keeps the default. + * + * @throws AccessDenied + */ + public function log(int $count = 0): array + { + $this->context->requireAdmin(false); + return ($count > 0) ? $this->state->events($count) : $this->state->events(); + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Smtp/SmtpClient.php b/data/web/inc/lib/mailcow/Archive/Smtp/SmtpClient.php new file mode 100644 index 000000000..2e2116bff --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Smtp/SmtpClient.php @@ -0,0 +1,190 @@ +open($host, $port); + try { + $this->expect('connect', 220); + $this->ehlo(); + $this->command('sender', 'MAIL FROM:<' . $from . '>', 250); + $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250); + $this->command('data', 'DATA', 354); + $this->write(self::dotStuff(self::crlf($data)) . ".\r\n"); + $reply = $this->expect('data', 250); + $this->write("QUIT\r\n"); + return preg_match('/queued as ([0-9A-Za-z]+)/', $reply, $m) ? $m[1] : ''; + } finally { + $this->close(); + } + } + + /** + * Connection test against the archive server: STARTTLS (unless $tls is 'may' and it is not offered), AUTH when + * a username is set, then MAIL and RCPT for the archive address, then RSET. + * + * @return string[] extensions the server announced (after STARTTLS) + * @throws SmtpException + */ + public function probe(string $host, int $port, string $tls, string $username, string $password, string $from, string $rcpt): array + { + $this->open($host, $port); + try { + $this->expect('connect', 220); + $extensions = $this->ehlo(); + if ($tls !== 'may' || in_array('STARTTLS', $extensions, true)) { + if (!in_array('STARTTLS', $extensions, true)) { + throw new SmtpException('tls', 'STARTTLS not offered'); + } + $this->command('tls', 'STARTTLS', 220); + $this->startTls($host, $tls === 'verify'); + $extensions = $this->ehlo(); + } + if ($username !== '') { + $this->command('auth', 'AUTH PLAIN ' . base64_encode("\0" . $username . "\0" . $password), 235); + } + $this->command('sender', 'MAIL FROM:<' . $from . '>', 250); + $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250); + $this->command('recipient', 'RSET', 250); + $this->write("QUIT\r\n"); + return $extensions; + } finally { + $this->close(); + } + } + + /** Mail server of a domain: the MX with the lowest preference, else the domain itself */ + public function mailHost(string $domain): string + { + if (@getmxrr($domain, $hosts, $weights) && !empty($hosts)) { + array_multisort($weights, SORT_NUMERIC, $hosts); + return rtrim($hosts[0], '.'); + } + return $domain; + } + + /** CRLF line ends, ending with CRLF */ + public static function crlf(string $data): string + { + $data = (string)preg_replace('/\r\n|\r|\n/', "\r\n", $data); + return str_ends_with($data, "\r\n") ? $data : $data . "\r\n"; + } + + /** RFC 5321 4.5.2: a line starting with "." gets another "." */ + public static function dotStuff(string $data): string + { + $data = str_replace("\r\n.", "\r\n..", $data); + return str_starts_with($data, '.') ? '.' . $data : $data; + } + + /** @throws SmtpException */ + protected function open(string $host, int $port): void + { + $target = filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? '[' . $host . ']' : $host; + $socket = @stream_socket_client('tcp://' . $target . ':' . $port, $errno, $error, self::TIMEOUT); + if ($socket === false) { + throw new SmtpException('connect', $error !== '' ? $error : 'connection failed'); + } + stream_set_timeout($socket, self::TIMEOUT); + $this->socket = $socket; + } + + protected function close(): void + { + if (is_resource($this->socket)) { + fclose($this->socket); + } + $this->socket = null; + } + + /** @throws SmtpException */ + protected function write(string $data): void + { + for ($written = 0; $written < strlen($data); $written += $count) { + $count = @fwrite($this->socket, substr($data, $written, 1048576)); + if ($count === false || $count === 0) { + throw new SmtpException('data', 'connection lost while sending'); + } + } + } + + /** One reply line without CRLF, null when the connection ended */ + protected function readLine(): ?string + { + $line = fgets($this->socket, 4096); + return ($line === false) ? null : rtrim($line, "\r\n"); + } + + /** @throws SmtpException */ + protected function startTls(string $host, bool $verify): void + { + stream_context_set_option($this->socket, 'ssl', 'verify_peer', $verify); + stream_context_set_option($this->socket, 'ssl', 'verify_peer_name', $verify); + stream_context_set_option($this->socket, 'ssl', 'allow_self_signed', !$verify); + stream_context_set_option($this->socket, 'ssl', 'peer_name', $host); + if (!@stream_socket_enable_crypto($this->socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT)) { + throw new SmtpException('tls', $verify ? 'TLS handshake failed or certificate not valid' : 'TLS handshake failed'); + } + } + + /** @return string[] extensions in upper case */ + private function ehlo(): array + { + $reply = $this->command('connect', 'EHLO ' . (gethostname() ?: 'mailcow'), 250); + $extensions = []; + foreach (array_slice(explode("\n", $reply), 1) as $line) { + $extensions[] = strtoupper((string)strtok(substr($line, 4), ' ')); + } + return $extensions; + } + + /** @throws SmtpException */ + private function command(string $step, string $command, int $code): string + { + $this->write($command . "\r\n"); + return $this->expect($step, $code); + } + + /** + * Reads a (multi-line) reply and returns it, lines joined by "\n". + * + * @throws SmtpException when the code differs + */ + private function expect(string $step, int $code): string + { + $lines = []; + do { + $line = $this->readLine(); + if ($line === null) { + throw new SmtpException($step, 'connection closed'); + } + $lines[] = $line; + } while (strlen($line) > 3 && $line[3] === '-'); + $reply = implode("\n", $lines); + if ((int)substr($reply, 0, 3) !== $code && !($code === 250 && (int)substr($reply, 0, 3) === 251)) { + throw new SmtpException($step, $reply); + } + return $reply; + } +} diff --git a/data/web/inc/lib/mailcow/Archive/Smtp/SmtpException.php b/data/web/inc/lib/mailcow/Archive/Smtp/SmtpException.php new file mode 100644 index 000000000..2a538eb16 --- /dev/null +++ b/data/web/inc/lib/mailcow/Archive/Smtp/SmtpException.php @@ -0,0 +1,27 @@ +: ", without angle brackets (the UI escapes message arguments twice) */ + public function describe(): string + { + $text = trim((string)preg_replace('/\s+/', ' ', $this->step . ': ' . str_replace(['<', '>'], '', $this->reply))); + return mb_substr($text, 0, self::MAX_TEXT_LENGTH); + } +} diff --git a/data/web/inc/vars.inc.php b/data/web/inc/vars.inc.php index 0f0dc154a..7da5634d4 100644 --- a/data/web/inc/vars.inc.php +++ b/data/web/inc/vars.inc.php @@ -205,6 +205,9 @@ $MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update'] = false; // Force 2FA enrollment at next login $MAILBOX_DEFAULT_ATTRIBUTES['force_tfa'] = false; +// Archive mails of the mailbox (archivecow): "inherit" follows the domain, "1" always, "0" never +$MAILBOX_DEFAULT_ATTRIBUTES['archive'] = 'inherit'; + // Enable SOGo access - Users will be redirected to SOGo after login (set to false to disable redirect by default) $MAILBOX_DEFAULT_ATTRIBUTES['sogo_access'] = true; diff --git a/data/web/js/site/archive.js b/data/web/js/site/archive.js new file mode 100644 index 000000000..3d830d0f1 --- /dev/null +++ b/data/web/js/site/archive.js @@ -0,0 +1,92 @@ +// Archive tab of the admin system page: header profile fields and, while archiving is enabled, the state of the archive +// under the switch (one line per problem; the details are in the archive log of the dashboard and in the queue manager). +var mailcowArchive = (function($) { + 'use strict'; + + function L(key) { + return (lang[key] !== undefined) ? lang[key] : key; + } + function esc(s) { + return escapeHtml(s === null || s === undefined ? '' : String(s)); + } + function time(ts) { + return ts ? new Date(ts * 1000).toLocaleString(LOCALE, DATETIME_FORMAT) : '-'; + } + // lang string with %s placeholders, values escaped + function text(key, values) { + var i = 0; + return esc(L(key)).replace(/%s/g, function() { return esc(values[i++]); }); + } + function notice(cls, html) { + return '
' + html + '
'; + } + var loaded = 0; + + function renderStatus(s) { + var html = ''; + if (s.enabled && !s.configured) { + html += notice('warning', esc(L('archive_state_unconfigured'))); + } + if (s.dropped_24h) { + html += notice('danger', text('archive_state_dropped', [s.dropped_24h, time(s.last_dropped.time), s.last_dropped.reason])); + } + if (s.queued) { + var reasons = $.map(s.delay_reasons || [], function(r) { return '
  • ' + esc(r) + '
    • '; }); + html += notice('warning', text('archive_state_queued', [s.queued, time(s.oldest_queued)]) + + (reasons.length ? '
      ' + reasons.join('') + '
    ' : '')); + } + if (!s.enabled) { + html += notice('secondary', esc(L('archive_state_off'))); + } else if (!html) { + html += notice('success', s.last_delivery ? text('archive_state_ok', [time(s.last_delivery)]) : esc(L('archive_state_ok_none'))); + } + $('#archive_state').html(html); + } + + function load() { + loaded = Date.now(); + $.get('/api/v1/get/archive/status', function(s) { renderStatus(s || {}); }, 'json').fail(function() { + $('#archive_state').html(notice('danger', esc(lang_danger.generic_server_error))); + }); + } + + // fields of the selected header profile only; hidden fields are not posted, so their stored values are kept + function showProfile() { + var profile = $('#archive_profile').val(); + $('.archive-profile').each(function() { + var on = $(this).hasClass('archive-profile-' + profile); + $(this).toggleClass('d-none', !on).find(':input').prop('disabled', !on); + }); + $('.archive-profile-info').each(function() { + $(this).toggleClass('d-none', !$(this).hasClass('archive-profile-info-' + profile)); + }); + } + + $(document).ready(function() { + $('#archive_profile').on('change', showProfile); + // the tests use the saved settings: not with unsaved changes in the form + $('form[data-id="archive"]').on('input change', ':input', function() { + $('.archive-test').prop('disabled', true); + $('#archive_test_info').text(L('archive_save_first')).removeClass('text-muted').addClass('text-warning'); + }); + // the state line is only shown while archiving is enabled + var tab = document.getElementById('collapse-tab-archive'); + if (!tab || !document.getElementById('archive_state')) return; + // the tab body is at the top: visible as soon as the tab (or on mobile its card) is opened; load again when shown later + new IntersectionObserver(function(entries) { + if (entries.some(function(e) { return e.intersectionRatio > 0; }) && Date.now() - loaded > 10000) load(); + }).observe(tab); + $('#archive_refresh').on('click', function(e) { + e.preventDefault(); + load(); + }); + // open the dashboard with the archive log (the dashboard restores the last tab) + $('#archive_log_link').on('click', function() { + try { + localStorage.setItem('lastTag', 'tab-archive-logs'); + } catch (e) {} + }); + }); + + return {load: load}; +})(jQuery); diff --git a/data/web/js/site/dashboard.js b/data/web/js/site/dashboard.js index 7c077e27b..f00b9ccd8 100644 --- a/data/web/js/site/dashboard.js +++ b/data/web/js/site/dashboard.js @@ -875,6 +875,81 @@ jQuery(function($){ hideTableExpandCollapseBtn('#tab-cron-logs', '#cron_log'); }); } + // archive copies from the Postfix log and mails not archived; the Message-ID finds a mail + function draw_archive_logs() { + // just recalc width if instance already exists + if ($.fn.DataTable.isDataTable('#archive_log') ) { + $('#archive_log').DataTable().columns.adjust().responsive.recalc(); + return; + } + // softbounce: the archive refused the copy, Postfix keeps it and tries again + var status_class = {sent: 'bg-success', deferred: 'bg-warning text-dark', softbounce: 'bg-danger', bounced: 'bg-danger', expired: 'bg-danger', dropped: 'bg-danger'}; + + $('#archive_log').DataTable({ + responsive: true, + processing: true, + serverSide: false, + stateSave: true, + pageLength: log_pagination_size, + dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" + + "tr" + + "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>", + language: lang_datatables, + order: [[0, 'desc']], + ajax: { + type: "GET", + url: "/api/v1/get/logs/archive", + dataSrc: function(data){ + return Array.isArray(data) ? data : []; + } + }, + columns: [ + { + title: lang.time, + data: 'time', + defaultContent: '', + createdCell: function(td, cellData) { + createSortableDate(td, cellData) + } + }, + { + title: lang.archive_delivery, + data: 'status', + defaultContent: '', + render: function(data) { + var label = lang['archive_delivery_' + data] || data; + return '' + escapeHtml(label) + ''; + } + }, + { + title: lang.archive_message_id, + data: 'message_id', + defaultContent: '', + className: 'text-break', + render: function(data) { + return escapeHtml(data); + } + }, + { + title: lang.archive_queue_id, + data: 'qid', + defaultContent: '', + render: function(data) { + return escapeHtml(data); + } + }, + { + title: lang.archive_details, + data: 'text', + defaultContent: '', + className: 'dtr-col-md text-break', + render: function(data) { + return escapeHtml(data); + } + } + ] + }); + } function rspamd_pie_graph() { $.ajax({ url: '/api/v1/get/rspamd/actions', @@ -1295,6 +1370,7 @@ jQuery(function($){ onVisible("[id^=sasl_logs]", () => draw_sasl_logs()); onVisible("[id^=netfilter_log]", () => draw_netfilter_logs()); onVisible("[id^=cron_log]", () => draw_cron_logs()); + onVisible("[id^=archive_log]", () => draw_archive_logs()); onVisible("[id^=rspamd_history]", () => draw_rspamd_history()); onVisible("[id^=rspamd_donut]", () => rspamd_pie_graph()); diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index 32de2a8d1..596bacbfd 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -232,6 +232,7 @@ $(document).ready(function() { } else { $('#addDomain_gal').prop('checked', false); } + $('#addDomain_archive').prop('checked', template.archive == 1); if (template.active == 1){ $('#addDomain_active').prop('checked', true); @@ -433,6 +434,7 @@ $(document).ready(function() { } else { $('#force_tfa').prop('checked', false); } + $('#addMailbox_archive').selectpicker('val', ['0', '1'].includes(String(template.archive)) ? String(template.archive) : 'inherit'); if (template.sogo_access == 1){ $('#sogo_access').prop('checked', true); } else { @@ -1187,6 +1189,18 @@ jQuery(function($){ return 1==data?'1':(0==data?'0':2==data&&'—'); } }, + { + title: lang.archive, + data: 'archive', + searchable: false, + defaultContent: '', + responsivePriority: 5, + // not shown while archiving is switched off + className: archive_enabled ? '' : 'never', + render: function (data, type) { + return 1==data?'1':'0'; + } + }, { title: lang.action, data: 'action', diff --git a/data/web/json_api.php b/data/web/json_api.php index b059ae8be..8757f16c4 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -858,6 +858,26 @@ if (isset($_GET['query'])) { } break; + case "archive": + switch ($object) { + case "settings": + process_get_return(\Mailcow\Archive\ArchiveModule::fromGlobals()->api->getSettings()); + break; + + case "status": + process_get_return(\Mailcow\Archive\ArchiveModule::fromGlobals()->api->getStatus()); + break; + + case "mailboxes": + process_get_return(\Mailcow\Archive\ArchiveModule::fromGlobals()->api->getMailboxes()); + break; + + default: + process_get_return(null); + break; + } + break; + case "logs": switch ($object) { case "dovecot": @@ -904,6 +924,11 @@ if (isset($_GET['query'])) { } echo (isset($logs) && !empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}'; break; + case "archive": + // deliveries of archive copies and mails not archived, newest first + $logs = \Mailcow\Archive\ArchiveModule::fromGlobals()->api->getLog(isset($extra) ? intval(preg_replace('/[^\d]/', '', $extra)) : 0); + echo (!empty($logs)) ? json_encode($logs, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) : '{}'; + break; case "postfix": // 0 is first record, so empty is fine if (isset($extra)) { @@ -1682,6 +1707,8 @@ if (isset($_GET['query'])) { ['db' => 'messages', 'dt' => 20, 'dummy' => true, 'order_subquery' => "SELECT SUM(messages) FROM `quota2` WHERE `quota2`.`username` = `m`.`username`"], ['db' => 'tags', 'dt' => 23, 'dummy' => true, 'search' => ['join' => 'LEFT JOIN `tags_mailbox` AS `tm` ON `tm`.`username` = `m`.`username`', 'where_column' => '`tm`.`tag_name`']], ['db' => 'active', 'dt' => 24], + // archived by the mailbox setting or ('inherit') the domain default + ['db' => 'archive', 'dt' => 25, 'dummy' => true, 'order_subquery' => "SELECT IF(JSON_VALUE(`m`.`attributes`, '$.archive') = '1' OR (COALESCE(JSON_VALUE(`m`.`attributes`, '$.archive'), '') NOT IN ('0', '1') AND `domain`.`archive` = '1'), 1, 0) FROM `domain` WHERE `domain`.`domain` = `m`.`domain`"], ]; if($_SESSION['mailcow_cc_role'] === 'admin') { @@ -1953,6 +1980,9 @@ if (isset($_GET['query'])) { case "mailbox-group": process_edit_return(\Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->editGroups($items, $attr)); break; + case "archive": + process_edit_return(\Mailcow\Archive\ArchiveModule::fromGlobals()->api->edit($attr)); + break; case "recipient_map": process_edit_return(recipient_map('edit', array_merge(array('id' => $items), $attr))); break; diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index 60c84cc34..02743959d 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -3,6 +3,7 @@ "alias_domains": "Alias-Domains hinzufügen", "alias_external_goto": "Aliase mit externen Ziel-Domains erlauben", "app_passwds": "App-Passwörter verwalten", + "archive": "Archivierung von Domains und Mailboxen ändern", "bcc_maps": "BCC-Maps", "delimiter_action": "Delimiter-Aktionen (tags)", "domain_desc": "Domainbeschreibung ändern", @@ -138,6 +139,57 @@ "app_links": "App-Links", "app_name": "App-Name", "apps_name": "\"mailcow Apps\" Name", + "archive": "Archivierung", + "archive_address": "Archivadresse", + "archive_address_info": "mailcow sendet eine Kopie jeder archivierten E-Mail an diese Adresse.", + "archive_delivery": "Ergebnis", + "archive_delivery_bounced": "Abgewiesen", + "archive_delivery_deferred": "Verzögert", + "archive_delivery_dropped": "Nicht archiviert", + "archive_delivery_expired": "Abgelaufen, verworfen", + "archive_delivery_sent": "Zugestellt", + "archive_delivery_softbounce": "Abgelehnt, wird wiederholt", + "archive_details": "Antwort des Archivs / Grund", + "archive_enabled": "Archivierung aktivieren", + "archive_enabled_info": "Archiviert die Domains und Mailboxen, bei denen die Archivierung aktiviert ist. Nach dem Deaktivieren entstehen keine neuen Kopien; Kopien in der Mailqueue werden noch zugestellt.", + "archive_header_from": "Header-Name für den Envelope-Absender", + "archive_header_rcpt": "Header-Name für die Envelope-Empfänger", + "archive_host": "Archivserver", + "archive_host_info": "Leer lassen, um den MX der Domain der Archivadresse zu verwenden.", + "archive_info": "mailcow sendet eine Kopie jeder angenommenen eingehenden, ausgehenden und internen E-Mail archivierter Mailboxen per SMTP an ein externes Archiv (Journaling). Die Kopien laufen über die Mailqueue und werden erneut zugestellt, solange das Archiv nicht erreichbar ist; Absender erhalten dafür nie einen Bounce oder eine Verzögerungswarnung.", + "archive_log": "Archiv-Protokoll", + "archive_log_info": "Kopien an das Archiv aus dem Postfix-Protokoll im Speicher und E-Mails, die nicht archiviert wurden. Suchen Sie nach der Message-ID einer E-Mail.", + "archive_message_id": "Message-ID", + "archive_password": "SMTP-Passwort", + "archive_password_remove": "Gespeichertes Passwort entfernen", + "archive_port": "Port", + "archive_profile": "Profil", + "archive_profile_archivecow": "archivecow (Besitzer, Envelope, Signatur)", + "archive_profile_archivecow_info": "Für archivecow: Signierte Header über der E-Mail nennen die archivierten Mailboxen, denen sie gehört, und den SMTP-Envelope (auch BCC-Empfänger). archivecow prüft die Signatur mit dem Signaturschlüssel.", + "archive_profile_envelope": "Envelope-Absender und -Empfänger", + "archive_profile_envelope_info": "Für andere Archive (z. B. MailPiler): Zwei Header über der E-Mail nennen Envelope-Absender und -Empfänger, damit das Archiv auch BCC-Empfänger zuordnen kann. Die Header sind nicht signiert.", + "archive_profile_none": "Keine (unveränderte Kopie)", + "archive_profile_none_info": "Die E-Mail geht unverändert ins Archiv. BCC-Empfänger sind nicht erkennbar, das Archiv ordnet die E-Mail nur nach ihren eigenen Headern (From, To, Cc) zu.", + "archive_queue_id": "Queue-ID", + "archive_save_first": "Erst speichern: Die Tests verwenden die gespeicherten Einstellungen.", + "archive_section_connection": "Verbindung", + "archive_section_headers": "Archiv-Header", + "archive_signing_secret": "Signaturschlüssel", + "archive_signing_secret_info": "Signiert die Archiv-Header (X-Archive-Signature). Muss mit dem Schlüssel in archivecow übereinstimmen.", + "archive_state_dropped": "%s E-Mails wurden in den letzten 24 Stunden nicht archiviert, zuletzt um %s: %s", + "archive_state_off": "Die Archivierung ist deaktiviert.", + "archive_state_ok": "Die Archivierung läuft, letzte Zustellung an das Archiv: %s", + "archive_state_ok_none": "Die Archivierung ist aktiv, das Protokoll enthält noch keine Zustellung an das Archiv", + "archive_state_queued": "%s Kopien warten auf das Archiv, die älteste seit %s", + "archive_state_unconfigured": "Archivadresse oder Signaturschlüssel fehlt", + "archive_test": "Verbindung testen", + "archive_test_info": "Die Tests verwenden die gespeicherten Einstellungen. Die Test-E-Mail geht über die Mailqueue an das Archiv.", + "archive_testmail": "Test-E-Mail senden", + "archive_tls": "TLS", + "archive_tls_encrypt": "Erforderlich, ohne Zertifikatsprüfung", + "archive_tls_may": "Wenn verfügbar", + "archive_tls_verify": "Erforderlich, mit Zertifikatsprüfung", + "archive_username": "SMTP-Benutzername", "arrival_time": "Ankunftszeit (Serverzeit)", "authed_user": "Auth. Benutzer", "ays": "Soll der Vorgang wirklich ausgeführt werden?", @@ -516,6 +568,19 @@ "api_key_service_locked": "API-Key %s ist in der mailcow.conf definiert und kann nur dort geändert werden", "app_name_empty": "App-Name darf nicht leer sein", "app_passwd_id_invalid": "App-Passwort ID %s ist ungültig", + "archive_address_invalid": "Ungültige Archivadresse", + "archive_address_local": "Die Archivadresse darf in keiner Domain oder Alias-Domain dieser mailcow liegen", + "archive_header_invalid": "Ungültiger Header-Name: Buchstaben, Ziffern und Bindestriche, beginnend mit einem Buchstaben", + "archive_host_invalid": "Ungültiger Archivserver", + "archive_incomplete": "Für die Archivierung sind eine Archivadresse und beim Profil archivecow ein Signaturschlüssel erforderlich", + "archive_password_invalid": "Ungültiges SMTP-Passwort: druckbare ASCII-Zeichen ohne Leerzeichen, höchstens 255 Zeichen", + "archive_port_invalid": "Ungültiger Port", + "archive_profile_invalid": "Ungültiges Profil für die Archiv-Header", + "archive_signing_secret_invalid": "Ungültiger Signaturschlüssel: druckbare ASCII-Zeichen ohne Leerzeichen, höchstens 255 Zeichen", + "archive_test_failed": "Verbindungstest zum Archiv fehlgeschlagen: %s", + "archive_testmail_failed": "Test-E-Mail konnte nicht in die Queue gestellt werden: %s", + "archive_tls_invalid": "Ungültiger TLS-Modus", + "archive_username_invalid": "Ungültiger SMTP-Benutzername", "authsource_in_use": "Der Identity Provider kann nicht geändert oder gelöscht werden, da er derzeit von einem oder mehreren Benutzern verwendet wird.", "bcc_empty": "BCC-Ziel darf nicht leer sein", "bcc_exists": "Ein BCC-Map-Eintrag %s existiert bereits als Typ %s", @@ -780,6 +845,15 @@ "app_name": "App-Name", "app_passwd": "App-Passwörter", "app_passwd_protocols": "Zugelassene Protokolle für App-Passwort", + "archive": "Archivierung", + "archive_active": "Archiviert", + "archive_domain": "Alle Mailboxen standardmäßig archivieren", + "archive_domain_info": "Gilt für Mailboxen mit der Einstellung „Domain-Standard“.", + "archive_inactive": "Nicht archiviert", + "archive_info": "„Domain-Standard“ folgt der Einstellung der Domain. Von archivierten Mailboxen geht eine Kopie jeder gesendeten und empfangenen E-Mail an das Archiv.", + "archive_inherit": "Domain-Standard", + "archive_off": "Deaktiviert", + "archive_on": "Aktiviert", "backup_mx_options": "Relay-Optionen", "bcc_dest_format": "BCC-Ziel muss eine gültige E-Mail-Adresse sein.", "client_id": "Client-ID", @@ -1061,6 +1135,7 @@ "allow_from_smtp": "Nur folgende IPs für SMTP erlauben", "allow_from_smtp_info": "Leer lassen, um keine Prüfung durchzuführen.
    IPv4- sowie IPv6-Adressen und Subnetzwerke.", "allowed_protocols": "Erlaubte Protokolle", + "archive": "Archiviert", "backup_mx": "Relay-Domain", "bcc": "BCC", "bcc_destination": "BCC-Ziel", @@ -1305,6 +1380,9 @@ "app_links": "Änderungen an App-Links wurden gespeichert", "app_passwd_added": "App-Passwort wurde gespeichert", "app_passwd_removed": "App-Passwort-ID %s wurde entfernt", + "archive_settings_saved": "Archivierungseinstellungen gespeichert", + "archive_test_ok": "Verbindungstest erfolgreich: Der Archivserver akzeptiert die Archivadresse", + "archive_testmail_queued": "Test-E-Mail an das Archiv in die Queue gestellt (Queue-ID %s)", "bcc_deleted": "BCC-Map-Einträge gelöscht: %s", "bcc_edited": "BCC-Map-Eintrag %s wurde geändert", "bcc_saved": "BCC- Map-Eintrag wurde gespeichert", @@ -1718,6 +1796,7 @@ "pushover_sound": "Ton" }, "warning": { + "archive_test_ok_missing": "Verbindungstest erfolgreich, aber der Archivserver kündigt %s nicht an: 8-Bit-E-Mails werden unverändert gesendet und eventuell abgelehnt", "cannot_delete_self": "Kann derzeit eingeloggten Benutzer nicht entfernen", "domain_added_sogo_failed": "Domain wurde hinzugefügt, aber SOGo konnte nicht neugestartet werden", "dovecot_restart_failed": "Dovecot wurde nicht erfolgreich neu gestartet, bitte prüfen Sie die Logs.", diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index d2848bcbb..8434052f8 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -3,6 +3,7 @@ "alias_domains": "Add alias domains", "alias_external_goto": "Allow aliases with external goto domains", "app_passwds": "Manage app passwords", + "archive": "Change archiving of domains and mailboxes", "bcc_maps": "BCC maps", "delimiter_action": "Delimiter action", "domain_desc": "Change domain description", @@ -142,6 +143,57 @@ "app_links": "App links", "app_name": "App name", "apps_name": "\"mailcow Apps\" name", + "archive": "Archiving", + "archive_address": "Archive address", + "archive_address_info": "mailcow sends a copy of every archived message to this address.", + "archive_delivery": "Result", + "archive_delivery_bounced": "Bounced", + "archive_delivery_deferred": "Deferred", + "archive_delivery_dropped": "Not archived", + "archive_delivery_expired": "Expired, discarded", + "archive_delivery_sent": "Delivered", + "archive_delivery_softbounce": "Refused, retrying", + "archive_details": "Answer of the archive / reason", + "archive_enabled": "Enable archiving", + "archive_enabled_info": "Archives the domains and mailboxes that have archiving enabled. Disabling stops new copies; copies already in the mail queue are still delivered.", + "archive_header_from": "Header name for the envelope sender", + "archive_header_rcpt": "Header name for the envelope recipients", + "archive_host": "Archive server", + "archive_host_info": "Leave empty to use the MX of the archive address domain.", + "archive_info": "mailcow sends a copy of every accepted incoming, outgoing and internal message of archived mailboxes to an external archive by SMTP (journaling). Copies are queued and retried while the archive is unreachable; senders never get a bounce or delay notice for them.", + "archive_log": "Archive log", + "archive_log_info": "Copies sent to the archive, from the Postfix log in memory, and mails that were not archived. Search for the Message-ID of a mail.", + "archive_message_id": "Message-ID", + "archive_password": "SMTP password", + "archive_password_remove": "Remove stored password", + "archive_port": "Port", + "archive_profile": "Profile", + "archive_profile_archivecow": "archivecow (owners, envelope, signature)", + "archive_profile_archivecow_info": "For archivecow: signed headers above the mail name the archived mailboxes it belongs to and the SMTP envelope (BCC recipients included). archivecow verifies the signature with the signing key.", + "archive_profile_envelope": "Envelope sender and recipients", + "archive_profile_envelope_info": "For other archives (e.g. MailPiler): two headers above the mail name the envelope sender and recipients, so the archive can assign BCC recipients too. The headers are not signed.", + "archive_profile_none": "None (unchanged copy)", + "archive_profile_none_info": "The mail goes to the archive unchanged. BCC recipients cannot be seen, the archive assigns the mail by its own headers (From, To, Cc) only.", + "archive_queue_id": "Queue ID", + "archive_save_first": "Save your changes first: the tests use the saved settings.", + "archive_section_connection": "Connection", + "archive_section_headers": "Archive headers", + "archive_signing_secret": "Signing key", + "archive_signing_secret_info": "Signs the archive headers (X-Archive-Signature). Must match the key in archivecow.", + "archive_state_dropped": "%s mails were not archived in the last 24 hours, the last at %s: %s", + "archive_state_off": "Archiving is disabled.", + "archive_state_ok": "Archiving is working, last delivery to the archive: %s", + "archive_state_ok_none": "Archiving is enabled, the log has no delivery to the archive yet", + "archive_state_queued": "%s copies are waiting for the archive, the oldest since %s", + "archive_state_unconfigured": "Archive address or signing key missing", + "archive_test": "Test connection", + "archive_test_info": "The tests use the saved settings. The test mail goes through the mail queue to the archive.", + "archive_testmail": "Send test mail", + "archive_tls": "TLS", + "archive_tls_encrypt": "Required, no certificate check", + "archive_tls_may": "If available", + "archive_tls_verify": "Required, with certificate check", + "archive_username": "SMTP username", "arrival_time": "Arrival time (server time)", "authed_user": "Auth. user", "ays": "Are you sure you want to proceed?", @@ -508,6 +560,19 @@ "api_key_description_invalid": "API key description must not exceed 255 characters", "api_key_network_invalid": "Invalid IP address or network: %s", "api_key_service_locked": "API key %s is defined in mailcow.conf and can only be changed there", + "archive_address_invalid": "Invalid archive address", + "archive_address_local": "The archive address must not be in a domain or alias domain of this mailcow", + "archive_header_invalid": "Invalid header name: letters, digits and hyphens, starting with a letter", + "archive_host_invalid": "Invalid archive server", + "archive_incomplete": "Archiving requires an archive address and, for the archivecow profile, a signing key", + "archive_password_invalid": "Invalid SMTP password: printable ASCII without spaces, at most 255 characters", + "archive_port_invalid": "Invalid port", + "archive_profile_invalid": "Invalid archive header profile", + "archive_signing_secret_invalid": "Invalid signing key: printable ASCII without spaces, at most 255 characters", + "archive_test_failed": "Connection test to the archive failed: %s", + "archive_testmail_failed": "Test mail could not be queued: %s", + "archive_tls_invalid": "Invalid TLS mode", + "archive_username_invalid": "Invalid SMTP username", "group_address_invalid": "Group address %s is invalid or not in the domain of the group", "group_description_invalid": "The group description must not exceed 255 characters", "group_grant_invalid": "Invalid share for mailbox %s", @@ -780,6 +845,15 @@ "app_name": "App name", "app_passwd": "App password", "app_passwd_protocols": "Allowed protocols for app password", + "archive": "Archiving", + "archive_active": "Archived", + "archive_domain": "Archive all mailboxes by default", + "archive_domain_info": "Applies to mailboxes set to \"Domain default\".", + "archive_inactive": "Not archived", + "archive_info": "\"Domain default\" follows the domain setting. Archived mailboxes send a copy of every message they send or receive to the archive.", + "archive_inherit": "Domain default", + "archive_off": "Disabled", + "archive_on": "Enabled", "backup_mx_options": "Relay options", "bcc_dest_format": "BCC destination must be a single valid email address.
    If you need to send a copy to multiple addresses, create an alias and use it here.", "client_id": "Client ID", @@ -1062,6 +1136,7 @@ "allow_from_smtp": "Only allow these IPs to use SMTP", "allow_from_smtp_info": "Leave empty to allow all senders.
    IPv4/IPv6 addresses and networks.", "allowed_protocols": "Allowed protocols", + "archive": "Archived", "backup_mx": "Relay domain", "bcc": "BCC", "bcc_destination": "BCC destination", @@ -1312,6 +1387,9 @@ "app_links": "Saved changes to app links", "app_passwd_added": "Added new app password", "app_passwd_removed": "Removed app password ID %s", + "archive_settings_saved": "Archive settings saved", + "archive_test_ok": "Connection test passed: the archive server accepts the archive address", + "archive_testmail_queued": "Test mail to the archive queued (queue ID %s)", "bcc_deleted": "BCC map entries deleted: %s", "bcc_edited": "BCC map entry %s edited", "bcc_saved": "BCC map entry saved", @@ -1725,6 +1803,7 @@ "years": "years" }, "warning": { + "archive_test_ok_missing": "Connection test passed, but the archive server does not announce %s: 8-bit messages are sent unchanged and may be rejected", "cannot_delete_self": "Cannot delete logged in user", "domain_added_sogo_failed": "Added domain but failed to restart SOGo, please check your server logs.", "dovecot_restart_failed": "Dovecot failed to restart, please check the logs", diff --git a/data/web/templates/admin.twig b/data/web/templates/admin.twig index 0c487c4ea..5b53656d3 100644 --- a/data/web/templates/admin.twig +++ b/data/web/templates/admin.twig @@ -29,6 +29,7 @@ +
  • @@ -47,6 +48,7 @@ {% include 'admin/tab-config-oauth2.twig' %} {% include 'admin/tab-config-rspamd.twig' %} {% include 'admin/tab-routing.twig' %} + {% include 'admin/tab-archive.twig' %} {% include 'admin/tab-config-dkim.twig' %} {% include 'admin/tab-config-fwdhosts.twig' %} {% include 'admin/tab-config-f2b.twig' %} diff --git a/data/web/templates/admin/tab-archive.twig b/data/web/templates/admin/tab-archive.twig new file mode 100644 index 000000000..738037653 --- /dev/null +++ b/data/web/templates/admin/tab-archive.twig @@ -0,0 +1,168 @@ +
    +
    +
    + + {{ lang.admin.archive }} +
    +
    +
    +

    {{ lang.admin.archive_info }}

    +
    +
    +
    +
    + +
    +
    + +
    + +
    +
    +
    + {{ lang.admin.archive_enabled_info }} +
    +
    + {% if archive_settings.enabled == 1 %} + {# Filled by js/site/archive.js: one line per problem; details in the archive log and the queue manager #} + + {% endif %} + + {{ lang.admin.archive_section_connection }}
    +
    +
    + +
    +
    + +
    +
    + {{ lang.admin.archive_address_info }} +
    +
    +
    +
    + +
    +
    +
    + + {{ lang.admin.archive_port }} + +
    +
    +
    + {{ lang.admin.archive_host_info }} +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + + {% if archive_settings.has_password == 1 %} +
    + + +
    + {% endif %} +
    +
    + + {{ lang.admin.archive_section_headers }}
    +
    +
    + +
    +
    + +
    +
    + {% for profile in ['archivecow', 'envelope', 'none'] %} + {{ attribute(lang.admin, 'archive_profile_' ~ profile ~ '_info') }} + {% endfor %} +
    +
    + {# Fields of the other profiles are hidden and disabled, so their stored values are kept #} + {% set archivecow = archive_settings.profile == 'archivecow' %} + {% set envelope = archive_settings.profile == 'envelope' %} +
    +
    + +
    +
    + +
    +
    + {{ lang.admin.archive_signing_secret_info }} +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    + {% set archive_ready = archive_settings.address and (archive_settings.profile != 'archivecow' or archive_settings.has_signing_secret == 1) %} + + + + {{ lang.admin.archive_test_info }} +
    +
    +
    +
    +
    +
    diff --git a/data/web/templates/dashboard.twig b/data/web/templates/dashboard.twig index 3c57afa5c..9014e2619 100644 --- a/data/web/templates/dashboard.twig +++ b/data/web/templates/dashboard.twig @@ -17,6 +17,7 @@
    • +
  • {{ lang.debug.external_logs }}
  • {{ lang.debug.static_logs }}
    • @@ -547,6 +548,21 @@ +
    +
    {{ lang.admin.archive_log_info }}
    +
    +
    + {{ lang.admin.archive }} +
    + +
    +
    +
    +
    +
    +
    +
    + diff --git a/data/web/templates/edit/domain-templates.twig b/data/web/templates/edit/domain-templates.twig index d4612a198..0fe3c8276 100644 --- a/data/web/templates/edit/domain-templates.twig +++ b/data/web/templates/edit/domain-templates.twig @@ -8,6 +8,7 @@ + @@ -70,6 +71,14 @@ +
    +
    +
    + + {{ lang.edit.archive_domain_info }} +
    +
    +
    diff --git a/data/web/templates/edit/domain.twig b/data/web/templates/edit/domain.twig index 45ec654ce..d6da53108 100644 --- a/data/web/templates/edit/domain.twig +++ b/data/web/templates/edit/domain.twig @@ -25,6 +25,7 @@ +
    @@ -133,6 +134,14 @@
    +
    +
    +
    + + {{ lang.edit.archive_domain_info }} +
    +
    +
    diff --git a/data/web/templates/edit/mailbox-templates.twig b/data/web/templates/edit/mailbox-templates.twig index 8384fa053..f8efaff92 100644 --- a/data/web/templates/edit/mailbox-templates.twig +++ b/data/web/templates/edit/mailbox-templates.twig @@ -114,6 +114,17 @@
    +
    + +
    + + {{ lang.edit.archive_info }} +
    +
    diff --git a/data/web/templates/edit/mailbox.twig b/data/web/templates/edit/mailbox.twig index 9f56ac56c..0b6325172 100644 --- a/data/web/templates/edit/mailbox.twig +++ b/data/web/templates/edit/mailbox.twig @@ -331,6 +331,19 @@ {{ lang.edit.allow_from_smtp_info }}
    +
    + +
    + + {{ lang.edit.archive_info }} +
    +
    diff --git a/data/web/templates/mailbox.twig b/data/web/templates/mailbox.twig index 6a6cbe58d..d54cacad5 100644 --- a/data/web/templates/mailbox.twig +++ b/data/web/templates/mailbox.twig @@ -81,6 +81,7 @@ var csrf_token = '{{ csrf_token }}'; var pagination_size = Math.trunc('{{ pagination_size }}'); var role = '{{ role }}'; + var archive_enabled = {{ archive_enabled ? 1 : 0 }}; var is_dual = {{ is_dual }}; var ALLOW_ADMIN_EMAIL_LOGIN = {{ allow_admin_email_login }}; diff --git a/data/web/templates/modals/mailbox.twig b/data/web/templates/modals/mailbox.twig index 0a886957c..a3e962d60 100644 --- a/data/web/templates/modals/mailbox.twig +++ b/data/web/templates/modals/mailbox.twig @@ -39,7 +39,7 @@
    -
    @@ -154,6 +154,17 @@
    +
    + +
    + + {{ lang.edit.archive_info }} +
    +
    @@ -360,6 +371,17 @@
    +
    + +
    + + {{ lang.edit.archive_info }} +
    +
    @@ -451,6 +473,7 @@
    + @@ -525,6 +548,14 @@
    {% endif %} +
    +
    +
    + + {{ lang.edit.archive_domain_info }} +
    +
    +
    @@ -664,6 +695,14 @@
    +
    +
    +
    + + {{ lang.edit.archive_domain_info }} +
    +
    +
    diff --git a/docker-compose.yml b/docker-compose.yml index 7bfa7b160..c7ee9b2fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # Version of the database schema -x-db-version: &db_version DB_VERSION=02112026_1600 +x-db-version: &db_version DB_VERSION=06112026_1800 services: @@ -375,6 +375,7 @@ services: - REDISPASS=${REDISPASS} - MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME} - SPAMHAUS_DQS_KEY=${SPAMHAUS_DQS_KEY:-} + - *db_version cap_add: - NET_BIND_SERVICE ports: diff --git a/tests/php/Archive/Controller/ArchiveApiControllerTest.php b/tests/php/Archive/Controller/ArchiveApiControllerTest.php new file mode 100644 index 000000000..dc4b44072 --- /dev/null +++ b/tests/php/Archive/Controller/ArchiveApiControllerTest.php @@ -0,0 +1,101 @@ +settings = $this->createMock(SettingsService::class); + $this->status = $this->createMock(StatusService::class); + $this->journal = $this->createMock(JournalService::class); + $this->mailboxes = $this->createMock(MailboxListService::class); + $this->api = new ArchiveApiController($this->settings, $this->status, $this->journal, $this->mailboxes, + new ApiResponse(['password', 'signing_secret'])); + } + + /** Messages written to the session as [type, msg] */ + private function messages(): array + { + return array_map(fn(array $m): array => [$m['type'], $m['msg']], $_SESSION['return'] ?? []); + } + + public function testSaveLogsWithoutSecrets(): void + { + $this->settings->expects($this->once())->method('save')->willReturn(new Settings()); + + $this->assertTrue($this->api->edit(['enabled' => '1', 'username' => 'mailcow', 'password' => 'pw', 'signing_secret' => 'key'])); + + $this->assertSame([[ + 'type' => 'success', + 'log' => ['archive', 'edit', 'settings', ['enabled' => '1', 'username' => 'mailcow', 'password' => '*', 'signing_secret' => '*']], + 'msg' => 'archive_settings_saved', + ]], $_SESSION['return']); + } + + public function testActionsAndErrors(): void + { + $this->settings->method('test')->willReturnOnConsecutiveCalls( + ['archive_test_ok_missing', '8BITMIME'], + $this->throwException(new UserError('archive_test_failed', ['archive.example:25 connect: Connection refused'])), + ); + $this->journal->method('testMail')->willReturn('4XyZ<1>'); + $this->settings->method('save')->willThrowException(new PDOException('gone away')); + + $this->assertTrue($this->api->edit(['action' => 'test'])); + $this->assertFalse($this->api->edit(['action' => 'test'])); + $this->assertTrue($this->api->edit(['action' => 'testmail'])); + $this->assertFalse($this->api->edit(['address' => 'journal@archive.example'])); + + $this->assertSame([ + ['warning', ['archive_test_ok_missing', '8BITMIME']], + ['danger', ['archive_test_failed', 'archive.example:25 connect: Connection refused']], + ['success', ['archive_testmail_queued', '4XyZ<1>']], + ['danger', ['mysql_error', 'gone away']], + ], $this->messages()); + } + + public function testActionsNeedWriteAccess(): void + { + $this->journal->method('testMail')->willThrowException(new AccessDenied()); + $this->settings->expects($this->never())->method('save'); + + $this->assertFalse($this->api->edit(['action' => 'testmail'])); + $this->assertSame([['danger', 'access_denied']], $this->messages()); + } + + public function testReadsAreEmptyWhenDenied(): void + { + $this->settings->method('get')->willThrowException(new AccessDenied()); + $this->status->method('status')->willThrowException(new AccessDenied()); + $this->status->method('log')->willThrowException(new AccessDenied()); + $this->mailboxes->method('archived')->willReturn(['enabled' => 1, 'mailboxes' => []]); + + $this->assertSame([], $this->api->getSettings()); + $this->assertSame([], $this->api->getStatus()); + $this->assertSame([], $this->api->getLog()); + $this->assertSame(['enabled' => 1, 'mailboxes' => []], $this->api->getMailboxes()); + } +} diff --git a/tests/php/Archive/Model/MailboxTest.php b/tests/php/Archive/Model/MailboxTest.php new file mode 100644 index 000000000..e20aff27d --- /dev/null +++ b/tests/php/Archive/Model/MailboxTest.php @@ -0,0 +1,60 @@ + ['inherit', 1, true], + 'inherit, domain off' => ['inherit', 0, false], + 'missing attribute' => [null, 1, true], + 'unknown value' => ['yes', 0, false], + 'on' => ['1', 0, true], + 'on as number' => [1, 0, true], + 'off' => ['0', 1, false], + 'off as number' => [0, 1, false], + 'on as boolean' => [true, 0, true], + 'off as boolean' => [false, 1, false], + ]; + } + + #[DataProvider('settings')] + public function testEffective(mixed $archive, int $domainArchive, bool $archived): void + { + $this->assertSame($archived, Mailbox::effective($archive, $domainArchive)); + } + + public function testListOfArchivedMailboxes(): void + { + $lookup = new FakeArchiveLookup(); + $lookup->mailboxes = ['b@firma.de' => ['inherit', 1], 'a@other.de' => ['1', 0], 'c@firma.de' => ['0', 1]]; + $redis = new FakeRedis(); + $readOnly = new Context('admin', 'API', [], false, 'ro'); + + $this->assertSame(['enabled' => 0, 'mailboxes' => []], (new MailboxListService($lookup, new ArchiveState($redis), $readOnly))->archived()); + + $redis->strings['ARCHIVE_ENABLED'] = '1'; + $this->assertSame(['enabled' => 1, 'mailboxes' => [ + ['username' => 'b@firma.de', 'domain' => 'firma.de', 'name' => '', 'active' => 1, 'source' => 'domain'], + ['username' => 'a@other.de', 'domain' => 'other.de', 'name' => '', 'active' => 1, 'source' => 'mailbox'], + ]], (new MailboxListService($lookup, new ArchiveState($redis), $readOnly))->archived()); + + $this->expectException(UserError::class); + (new MailboxListService($lookup, new ArchiveState($redis), FakeContext::domainAdmin()))->archived(); + } +} diff --git a/tests/php/Archive/Model/SettingsTest.php b/tests/php/Archive/Model/SettingsTest.php new file mode 100644 index 000000000..8daa755e0 --- /dev/null +++ b/tests/php/Archive/Model/SettingsTest.php @@ -0,0 +1,51 @@ + ['', 25, 'archive.example'], + 'mx, other port' => ['', 2525, 'archive.example:2525'], + 'host' => ['smtp.archive.example', 25, '[smtp.archive.example]:25'], + 'ipv4' => ['192.0.2.5', 587, '[192.0.2.5]:587'], + 'ipv6' => ['2001:db8::25', 25, '[ipv6:2001:db8::25]:25'], + ]; + } + + #[DataProvider('nexthops')] + public function testNexthop(string $host, int $port, string $nexthop): void + { + $settings = new Settings(); + $settings->address = 'journal@archive.example'; + $settings->host = $host; + $settings->port = $port; + + $this->assertSame($nexthop, $settings->nexthop()); + } + + public function testCanDeliverAndNoSecretsInTheApi(): void + { + $settings = new Settings(); + $this->assertFalse($settings->canDeliver()); + $settings->address = 'journal@archive.example'; + $this->assertFalse($settings->canDeliver()); + $settings->profile = 'envelope'; + $this->assertTrue($settings->canDeliver()); + + $settings->password = 'password-value'; + $settings->signingKey = 'signing-key-value'; + $api = json_encode($settings->toArray()); + $this->assertStringNotContainsString('password-value', $api); + $this->assertStringNotContainsString('signing-key-value', $api); + $this->assertSame([1, 1], [$settings->toArray()['has_password'], $settings->toArray()['has_signing_secret']]); + } +} diff --git a/tests/php/Archive/Redis/ArchiveStateTest.php b/tests/php/Archive/Redis/ArchiveStateTest.php new file mode 100644 index 000000000..0fdcaf682 --- /dev/null +++ b/tests/php/Archive/Redis/ArchiveStateTest.php @@ -0,0 +1,92 @@ +redis = new FakeRedis(); + $this->state = new ArchiveState($this->redis); + } + + /** A Postfix log entry as syslog-ng pushes it (newest first) */ + private function log(string $program, string $message, int $time): void + { + $this->redis->lPush('POSTFIX_MAILLOG', json_encode(['time' => $time, 'program' => $program, 'message' => $message])); + } + + public function testGlobalSwitchIsKeptHere(): void + { + // no key yet: off + $this->assertFalse($this->state->enabled()); + $this->state->setEnabled(true); + $this->assertSame('1', $this->redis->strings['ARCHIVE_ENABLED']); + $this->assertTrue($this->state->enabled()); + $this->state->setEnabled(false); + $this->assertSame('0', $this->redis->strings['ARCHIVE_ENABLED']); + $this->assertFalse($this->state->enabled()); + // no connection or a failing one: off, rspamd cannot read the switch either + $this->assertFalse((new ArchiveState(null))->enabled()); + $failing = new class { + public function get(string $key): string + { + throw new \RuntimeException('connection lost'); + } + }; + $this->assertFalse((new ArchiveState($failing))->enabled()); + } + + public function testDroppedCopiesAreCountedAndTheLast100Listed(): void + { + for ($i = 1; $i <= 105; $i++) { + $this->state->dropped('Q' . $i, 'owner lookup failed', 'm' . $i . '@firma.de'); + } + + $stats = $this->state->droppedStats(2); + $this->assertSame(105, $stats['count']); + $this->assertSame(['Q105', 'Q104'], array_column($stats['recent'], 'qid')); + $this->assertSame(['owner lookup failed', 'm105@firma.de'], [$stats['recent'][0]['reason'], $stats['recent'][0]['message_id']]); + $this->assertCount(100, $this->redis->lists['ARCHIVE_DROPPED_LOG']); + } + + public function testEventsOfTheArchiveServiceOnly(): void + { + $this->log('postfix/cleanup', '1A2B: message-id=', 1700000000); + $this->log('postfix/archive/smtp', '1A2B: to=, relay=archive.example[192.0.2.5]:25, delay=0.2, dsn=2.0.0, ' + . 'status=sent (250 2.0.0 Ok: queued as X1)', 1700000001); + $this->log('postfix/smtp', '3C4D: to=, relay=mx.extern.example[192.0.2.9]:25, dsn=2.0.0, status=sent (250 ok)', 1700000002); + // notices about copies are dropped by archive_discard, they are no deliveries + $this->log('postfix/archive/discard', '5E6F: to=, relay=none, dsn=2.0.0, status=sent (discards mail)', 1700000003); + $this->log('postfix/archive/smtp', 'connect to archive.example[192.0.2.5]:25: Connection refused', 1700000004); + $this->log('postfix/archive/smtp', '7A8B: to=, relay=archive.example[192.0.2.5]:25, dsn=5.1.1, ' + . 'status=SOFTBOUNCE (host archive.example[192.0.2.5] said: 550 5.1.1 unknown (in reply to RCPT TO command))', 1700000005); + $this->redis->lPush('POSTFIX_MAILLOG', 'no json'); + + $this->assertSame([ + ['time' => 1700000005, 'status' => 'softbounce', 'qid' => '7A8B', 'message_id' => '', + 'text' => 'host archive.example[192.0.2.5] said: 550 5.1.1 unknown (in reply to RCPT TO command)'], + ['time' => 1700000001, 'status' => 'sent', 'qid' => '1A2B', 'message_id' => 'orig@extern.example', 'text' => '250 2.0.0 Ok: queued as X1'], + ], $this->state->events()); + $this->assertCount(1, $this->state->events(1)); + } + + public function testWithoutRedisNothingHappens(): void + { + $state = new ArchiveState(null); + $state->setEnabled(true); + $state->dropped('Q1', 'reason'); + + $this->assertSame(['count' => 0, 'recent' => []], $state->droppedStats()); + $this->assertSame([], $state->events()); + } +} diff --git a/tests/php/Archive/Service/HeaderBuilderTest.php b/tests/php/Archive/Service/HeaderBuilderTest.php new file mode 100644 index 000000000..c284a8a63 --- /dev/null +++ b/tests/php/Archive/Service/HeaderBuilderTest.php @@ -0,0 +1,105 @@ +profile = $profile; + $settings->signingKey = 'key'; + return $settings; + } + + private static function journal(array $rcpts = ['anna@firma.de', 'bcc@extern.example'], string $from = 'x@extern.example'): Journal + { + return new Journal(['anna@firma.de'], $rcpts, $from, '4XyZ1', 'internal', 'smtp', 1700000000.25, str_repeat('a', 64)); + } + + public function testArchivecowHeadersAndSignature(): void + { + $block = (new HeaderBuilder())->build(self::settings('archivecow'), self::journal()); + + $this->assertSame("X-Archive-Owner: anna@firma.de\r\nX-Archive-Rcpt: anna@firma.de,bcc@extern.example\r\n" + . "X-Archive-From: x@extern.example\r\nX-Archive-Queue-ID: 4XyZ1\r\nX-Archive-Received: 2023-11-14T22:13:20.250Z\r\n" + . "X-Archive-Direction: internal\r\nX-Archive-Source: smtp\r\nX-Archive-SHA256: " . str_repeat('a', 64) . "\r\n", + substr($block, 0, strpos($block, 'X-Archive-Signature:'))); + $signed = implode("\n", ['x-archive-owner:anna@firma.de', 'x-archive-rcpt:anna@firma.de,bcc@extern.example', + 'x-archive-from:x@extern.example', 'x-archive-queue-id:4XyZ1', 'x-archive-received:2023-11-14T22:13:20.250Z', + 'x-archive-direction:internal', 'x-archive-source:smtp', 'x-archive-sha256:' . str_repeat('a', 64)]); + $this->assertStringEndsWith("X-Archive-Signature: v=1; a=hmac-sha256; s=" . hash_hmac('sha256', $signed, 'key') . "\r\n", $block); + } + + /** Fixed vector shared with archivecow: if this changes, every archive that verifies the signature refuses the copies */ + public function testWireFormatIsStable(): void + { + $message = "From: sender@example.com\r\nTo: anna.schmidt@example.org\r\nSubject: Test\r\nMessage-ID: <1@example.com>\r\n\r\nHallo.\r\n"; + $settings = self::settings('archivecow'); + $settings->signingKey = 'test-signing-key'; + $journal = new Journal(['anna.schmidt@example.org', 'buchhaltung@example.org'], + ['anna.schmidt@example.org', 'buchhaltung@example.org', 'extern.partner@example.net'], + 'sender@example.com', '4ZxYw1abcDz9', 'in', 'smtp', 1793983297.25, hash('sha256', $message)); + + $this->assertSame('0042f6cd8a846e5ce8f4a57c97121324c1213380fc204736d15ffe5693a9ef5c', hash('sha256', $message)); + $this->assertSame("X-Archive-Owner: anna.schmidt@example.org,buchhaltung@example.org\r\n" + . "X-Archive-Rcpt: anna.schmidt@example.org,buchhaltung@example.org,\r\n extern.partner@example.net\r\n" + . "X-Archive-From: sender@example.com\r\n" + . "X-Archive-Queue-ID: 4ZxYw1abcDz9\r\n" + . "X-Archive-Received: 2026-11-06T16:41:37.250Z\r\n" + . "X-Archive-Direction: in\r\n" + . "X-Archive-Source: smtp\r\n" + . "X-Archive-SHA256: 0042f6cd8a846e5ce8f4a57c97121324c1213380fc204736d15ffe5693a9ef5c\r\n" + . "X-Archive-Signature: v=1; a=hmac-sha256; s=443c927e165fd1f418fdfde2be54645d3201ef3b17d9e74ed3569d937fcf84d1\r\n", + (new HeaderBuilder())->build($settings, $journal)); + } + + public function testLongListsAreFoldedAndSignedUnfolded(): void + { + $rcpts = array_map(fn(int $i): string => sprintf('recipient%04d@extern.example', $i), range(1, 200)); + $block = (new HeaderBuilder())->build(self::settings('archivecow'), self::journal($rcpts)); + + foreach (explode("\r\n", $block) as $line) { + $this->assertLessThanOrEqual(78 + 30, strlen($line)); + } + preg_match('/^X-Archive-Rcpt: (.*?)\r\n(?=\S)/ms', $block, $m); + $this->assertSame($rcpts, explode(',', preg_replace('/\r\n /', '', $m[1]))); + // the signature covers the values without whitespace, so folding does not matter + $this->assertStringContainsString(hash_hmac('sha256', implode("\n", [ + 'x-archive-owner:anna@firma.de', 'x-archive-rcpt:' . implode(',', $rcpts), 'x-archive-from:x@extern.example', + 'x-archive-queue-id:4XyZ1', 'x-archive-received:2023-11-14T22:13:20.250Z', 'x-archive-direction:internal', + 'x-archive-source:smtp', 'x-archive-sha256:' . str_repeat('a', 64)]), 'key'), $block); + } + + public function testEmptyValuesAreLeftOutAndControlCharactersRemoved(): void + { + $journal = new Journal(["anna@firma.de\r\nX-Evil: 1"], ['a@b.example', ''], '', '', 'in', 'quarantine', 1700000000.0, ''); + $block = (new HeaderBuilder())->build(self::settings('archivecow'), $journal); + + $this->assertStringNotContainsString("\r\nX-Evil", $block); + $this->assertStringNotContainsString('X-Archive-From', $block); + $this->assertStringNotContainsString('X-Archive-Queue-ID', $block); + $this->assertStringNotContainsString('X-Archive-SHA256', $block); + $this->assertStringContainsString("X-Archive-Owner: anna@firma.de X-Evil: 1\r\n", $block); + $this->assertStringContainsString("X-Archive-Rcpt: a@b.example\r\n", $block); + } + + public function testEnvelopeAndNoneProfiles(): void + { + $settings = self::settings('envelope'); + $settings->headerFrom = 'X-Real-MailFrom'; + $settings->headerRcpt = 'X-Real-RcptTo'; + $this->assertSame("X-Real-MailFrom: x@extern.example\r\nX-Real-RcptTo: anna@firma.de,bcc@extern.example\r\n", + (new HeaderBuilder())->build($settings, self::journal())); + $this->assertSame("X-Real-MailFrom: <>\r\nX-Real-RcptTo: anna@firma.de,bcc@extern.example\r\n", + (new HeaderBuilder())->build($settings, self::journal(from: ''))); + $this->assertSame('', (new HeaderBuilder())->build(self::settings('none'), self::journal())); + } +} diff --git a/tests/php/Archive/Service/JournalServiceTest.php b/tests/php/Archive/Service/JournalServiceTest.php new file mode 100644 index 000000000..31db81fb5 --- /dev/null +++ b/tests/php/Archive/Service/JournalServiceTest.php @@ -0,0 +1,165 @@ +stored = new Settings(); + $this->stored->address = 'journal@archive.example'; + $this->stored->signingKey = 'key'; + $this->lookup = new FakeArchiveLookup(); + $this->lookup->mailboxes = ['anna@firma.de' => ['inherit', 1], 'olga@other.de' => ['inherit', 0]]; + foreach (array_keys($this->lookup->mailboxes) as $username) { + $this->lookup->aliases[$username] = $username; + } + $this->redis = new FakeRedis(); + $this->redis->strings['ARCHIVE_ENABLED'] = '1'; + $this->smtp = new FakeSmtpClient(); + } + + private function service(?Context $context = null): JournalService + { + $repo = $this->createStub(SettingsRepository::class); + $repo->method('load')->willReturnCallback(fn(): Settings => clone $this->stored); + return new JournalService($repo, new OwnerResolver($this->lookup), new HeaderBuilder(), $this->smtp, + new ArchiveState($this->redis), $context ?? Context::system()); + } + + private function expectUserError(string|array $msg, callable $call): void + { + try { + $call(); + $this->fail('UserError ' . json_encode($msg) . ' expected'); + } catch (UserError $e) { + $this->assertSame($msg, $e->msg()); + } + } + + /** Header block and mail of a submitted copy, split after X-Archive-Signature */ + private static function split(string $data): array + { + $end = strpos($data, "\r\n", (int)strpos($data, 'X-Archive-Signature: ')) + 2; + return [substr($data, 0, $end), substr($data, $end)]; + } + + public function testPrepareForAnArchivedRecipient(): void + { + $sha256 = hash('sha256', 'mail'); + $result = $this->service()->prepare(['from' => 'x@extern.example', 'rcpt' => ['anna@firma.de', 'olga@other.de'], 'user' => '', + 'bcc' => true, 'qid' => '4XyZ1', 'received' => 1700000000.25, 'sha256' => $sha256]); + + $this->assertSame(['archive' => true, 'rcpt' => 'journal@archive.example', 'sender' => 'archive-bounce@localhost'], + array_slice($result, 0, 3)); + $this->assertStringStartsWith("X-Archive-Owner: anna@firma.de\r\nX-Archive-Rcpt: anna@firma.de,olga@other.de\r\n" + . "X-Archive-From: x@extern.example\r\nX-Archive-Queue-ID: 4XyZ1\r\nX-Archive-Received: 2023-11-14T22:13:20.250Z\r\n" + . "X-Archive-Direction: in\r\nX-Archive-Source: smtp\r\nX-Archive-SHA256: $sha256\r\nX-Archive-Signature: ", $result['headers']); + $this->assertSame('1', $this->redis->strings['ARCHIVE_ENABLED']); + } + + public function testPrepareWithoutCopy(): void + { + $this->assertSame(['archive' => false], $this->service()->prepare(['from' => 'x@extern.example', 'rcpt' => ['olga@other.de']])); + + $this->stored->signingKey = ''; + $this->assertSame(['archive' => false], $this->service()->prepare(['from' => 'x@extern.example', 'rcpt' => ['anna@firma.de']])); + + $this->redis->strings['ARCHIVE_ENABLED'] = '0'; + $this->assertSame(['archive' => false], $this->service()->prepare(['from' => 'x@extern.example', 'rcpt' => ['anna@firma.de']])); + $this->assertSame([], $this->smtp->submitted); + } + + public function testPrepareCleansTheRequest(): void + { + $result = $this->service()->prepare(['from' => "x@extern.example\r\nBcc: y@extern.example", 'rcpt' => ['anna@firma.de', ['x'], 5], + 'qid' => str_repeat('Q', 100), 'received' => 'yesterday', 'sha256' => 'ABC']); + + $this->assertTrue($result['archive']); + $this->assertStringContainsString("\r\nX-Archive-From: x@extern.example Bcc: y@extern.example\r\n", $result['headers']); + $this->assertStringContainsString("\r\nX-Archive-Rcpt: anna@firma.de\r\n", $result['headers']); + $this->assertStringContainsString("\r\nX-Archive-Queue-ID: " . str_repeat('Q', 64) . "\r\n", $result['headers']); + $this->assertMatchesRegularExpression('/\r\nX-Archive-Received: \d{4}-\d\d-\d\dT[\d:.]+Z\r\n/', $result['headers']); + $this->assertStringNotContainsString('X-Archive-SHA256', $result['headers']); + } + + public function testCopyOfADeliveredMail(): void + { + $this->assertTrue($this->service()->submitDelivered("Subject: hi\n\n.body\n", 'x@extern.example', ['anna@firma.de'], 'quarantine')); + + $sent = $this->smtp->submitted[0]; + $this->assertSame(['postfix', 591, 'archive-bounce@localhost', 'journal@archive.example'], + [$sent['host'], $sent['port'], $sent['from'], $sent['rcpt']]); + [$headers, $message] = self::split($sent['data']); + $this->assertSame("Subject: hi\r\n\r\n.body\r\n", $message); + $this->assertStringStartsWith("X-Archive-Owner: anna@firma.de\r\nX-Archive-Rcpt: anna@firma.de\r\n", $headers); + $this->assertStringContainsString("\r\nX-Archive-Source: quarantine\r\nX-Archive-SHA256: " . hash('sha256', $message) . "\r\n", $headers); + } + + public function testCopyOfADeliveredMailNeverThrows(): void + { + $this->assertFalse($this->service()->submitDelivered("x\r\n", 'x@extern.example', ['olga@other.de'], 'massmail')); + $this->assertSame([], $this->smtp->submitted); + + $this->smtp->error = new SmtpException('connect', 'Connection refused'); + $errorLog = ini_set('error_log', '/dev/null'); + try { + $this->assertFalse($this->service()->submitDelivered("Message-ID: \r\nSubject: x\r\n\r\nx\r\n", + 'x@extern.example', ['anna@firma.de'], 'massmail')); + } finally { + ini_set('error_log', (string)$errorLog); + } + $this->assertSame('1', $this->redis->strings['ARCHIVE_DROPPED']); + // the Message-ID finds the mail in the archive log + $this->assertSame(['qid' => '', 'reason' => 'massmail: connect: Connection refused', 'message_id' => 'mm1@firma.de'], + array_slice(json_decode($this->redis->lists['ARCHIVE_DROPPED_LOG'][0], true), 1)); + } + + public function testTestMail(): void + { + $this->redis->strings['ARCHIVE_ENABLED'] = '0'; + + $this->assertSame('Q1', $this->service(FakeContext::admin())->testMail()); + + [$headers, $message] = self::split($this->smtp->submitted[0]['data']); + $this->assertStringStartsWith("X-Archive-Rcpt: journal@archive.example\r\nX-Archive-From: archive-bounce@localhost\r\n", $headers); + $this->assertStringContainsString("\r\nX-Archive-Source: test\r\n", $headers); + $this->assertStringContainsString("\r\nSubject: mailcow archive test\r\n", $message); + } + + public function testTestMailErrors(): void + { + $this->expectUserError('access_denied', fn() => $this->service(new Context('admin', 'API', [], false, 'ro'))->testMail()); + $this->expectUserError('access_denied', fn() => $this->service(FakeContext::domainAdmin(acl: ['archive' => '1']))->testMail()); + + $this->smtp->error = new SmtpException('recipient', '550 5.7.1 : relay denied'); + $this->expectUserError(['archive_testmail_failed', 'recipient: 550 5.7.1 journal@archive.example: relay denied'], + fn() => $this->service(FakeContext::admin())->testMail()); + + $this->stored->address = ''; + $this->expectUserError('archive_incomplete', fn() => $this->service(FakeContext::admin())->testMail()); + $this->assertSame([], $this->smtp->submitted); + } +} diff --git a/tests/php/Archive/Service/MailboxListServiceTest.php b/tests/php/Archive/Service/MailboxListServiceTest.php new file mode 100644 index 000000000..cef77cbf6 --- /dev/null +++ b/tests/php/Archive/Service/MailboxListServiceTest.php @@ -0,0 +1,68 @@ +redis = new FakeRedis(); + $this->lookup = new FakeArchiveLookup(); + // username => [mailbox setting, domain default] + $this->lookup->mailboxes = [ + 'own@firma.de' => ['1', 0], + 'inherit@firma.de' => ['inherit', 1], + 'excluded@firma.de' => ['0', 1], + 'plain@other.example' => ['inherit', 0], + ]; + } + + private function service(bool $enabled, ?Context $context = null): MailboxListService + { + $this->redis->strings['ARCHIVE_ENABLED'] = $enabled ? '1' : '0'; + return new MailboxListService($this->lookup, new ArchiveState($this->redis), $context ?? FakeContext::admin()); + } + + public function testArchivedMailboxesWithTheirSource(): void + { + $expected = ['enabled' => 1, 'mailboxes' => [ + ['username' => 'own@firma.de', 'domain' => 'firma.de', 'name' => '', 'active' => 1, 'source' => 'mailbox'], + ['username' => 'inherit@firma.de', 'domain' => 'firma.de', 'name' => '', 'active' => 1, 'source' => 'domain'], + ]]; + + $this->assertSame($expected, $this->service(true)->archived()); + // read-only API keys too: archivecow keeps its mailboxes in line with this list + $this->assertSame($expected, $this->service(true, new Context('admin', 'API', [], false, 'ro'))->archived()); + } + + public function testEmptyWhileArchivingIsOff(): void + { + $this->assertSame(['enabled' => 0, 'mailboxes' => []], $this->service(false)->archived()); + } + + public function testOnlyAdmins(): void + { + foreach ([FakeContext::domainAdmin(['own@firma.de'], ['firma.de'], ['archive' => '1']), FakeContext::user('own@firma.de')] as $context) { + try { + $this->service(true, $context)->archived(); + $this->fail('AccessDenied expected'); + } catch (AccessDenied) { + $this->addToAssertionCount(1); + } + } + } +} diff --git a/tests/php/Archive/Service/OwnerResolverTest.php b/tests/php/Archive/Service/OwnerResolverTest.php new file mode 100644 index 000000000..baffcd541 --- /dev/null +++ b/tests/php/Archive/Service/OwnerResolverTest.php @@ -0,0 +1,158 @@ +lookup = new FakeArchiveLookup(); + // archived domain firma.de, domain without archiving other.de + $this->lookup->mailboxes = [ + 'anna@firma.de' => ['inherit', 1], + 'bernd@firma.de' => ['inherit', 1], + 'buchhaltung@firma.de' => ['inherit', 1], + 'br@firma.de' => ['0', 1], + 'olga@other.de' => ['inherit', 0], + 'otto@other.de' => ['1', 0], + ]; + // mailboxes have alias self rows + foreach (array_keys($this->lookup->mailboxes) as $username) { + $this->lookup->aliases[$username] = $username; + } + } + + private function owners(array $rcpts, string $from = 'sender@extern.example', string $user = '', bool $bcc = true): array + { + return (new OwnerResolver($this->lookup))->resolve($from, $rcpts, $user, $bcc)['owners']; + } + + public static function incoming(): array + { + return [ + 'mailbox' => [['anna@firma.de'], ['anna@firma.de']], + 'upper case and brackets' => [[''], ['anna@firma.de']], + 'tag' => [['anna+news@firma.de'], ['anna@firma.de']], + 'external' => [['someone@extern.example'], []], + 'mailbox setting off' => [['br@firma.de'], []], + 'domain without archiving' => [['olga@other.de'], []], + 'mailbox setting on in a domain without archiving' => [['otto@other.de'], ['otto@other.de']], + 'several recipients, sorted and unique' => [['bernd@firma.de', 'anna@firma.de', 'anna+x@firma.de'], ['anna@firma.de', 'bernd@firma.de']], + ]; + } + + #[DataProvider('incoming')] + public function testIncomingRecipients(array $rcpts, array $owners): void + { + $this->assertSame($owners, $this->owners($rcpts)); + } + + public function testAliasExpandsAllBranches(): void + { + $this->lookup->aliases['team@firma.de'] = 'sales@firma.de,support@firma.de'; + $this->lookup->aliases['sales@firma.de'] = 'anna@firma.de'; + $this->lookup->aliases['support@firma.de'] = 'bernd@firma.de,someone@extern.example'; + + $this->assertSame(['anna@firma.de', 'bernd@firma.de'], $this->owners(['team@firma.de'])); + } + + public function testAliasKeepingACopyAndLoops(): void + { + $this->lookup->aliases['anna@firma.de'] = 'anna@firma.de,bernd@firma.de'; + $this->lookup->aliases['loop1@firma.de'] = 'loop2@firma.de'; + $this->lookup->aliases['loop2@firma.de'] = 'loop1@firma.de,olga@other.de,otto@other.de'; + + $this->assertSame(['anna@firma.de', 'bernd@firma.de'], $this->owners(['anna@firma.de'])); + $this->assertSame(['otto@other.de'], $this->owners(['loop1@firma.de'])); + } + + public function testCatchAllOnlyForAddressesWithoutOwnEntry(): void + { + $this->lookup->aliases['@firma.de'] = 'buchhaltung@firma.de'; + $this->lookup->aliases['old@firma.de'] = 'gone@firma.de'; + + $this->assertSame(['buchhaltung@firma.de'], $this->owners(['unknown@firma.de'])); + $this->assertSame(['anna@firma.de'], $this->owners(['anna@firma.de'])); + // a target without entry of its own gets the catch-all too, like Postfix + $this->assertSame(['buchhaltung@firma.de'], $this->owners(['old@firma.de'])); + } + + public function testAliasDomain(): void + { + $this->lookup->aliasDomains['firma.com'] = 'firma.de'; + $this->lookup->aliases['@firma.com'] = 'bernd@firma.de'; + + $this->assertSame(['anna@firma.de'], $this->owners(['anna@firma.com'])); + $this->assertSame(['anna@firma.de'], $this->owners(['anna+tag@firma.com'])); + $this->assertSame(['bernd@firma.de'], $this->owners(['nobody@firma.com'])); + } + + public function testRecipientMapsRewriteFirst(): void + { + $this->lookup->recipientMaps['anna@extern.example'] = 'anna@firma.de'; + $this->lookup->recipientMaps['@alt.example'] = '@firma.de'; + + $this->assertSame(['anna@firma.de'], $this->owners(['anna@extern.example'])); + $this->assertSame(['bernd@firma.de'], $this->owners(['bernd@alt.example'])); + } + + public function testSpamAliasResourceAndLocalTargets(): void + { + $this->lookup->spamAliases['x7k2@firma.de'] = 'anna@firma.de'; + $this->lookup->resources = ['room@firma.de']; + $this->lookup->aliases['@firma.de'] = 'bernd@firma.de'; + $this->lookup->aliases['learn@firma.de'] = 'spam@localhost'; + + $this->assertSame(['anna@firma.de'], $this->owners(['x7k2@firma.de'])); + $this->assertSame([], $this->owners(['room@firma.de'])); + $this->assertSame([], $this->owners(['learn@firma.de'])); + } + + public function testBccDestinationsOnlyWhenRspamdSentTheCopy(): void + { + $this->lookup->bcc['rcpt']['olga@other.de'] = 'buchhaltung@firma.de'; + $this->lookup->bcc['sender']['@extern.example'] = 'bernd@firma.de'; + + $this->assertSame(['bernd@firma.de', 'buchhaltung@firma.de'], $this->owners(['olga+tag@other.de'])); + $this->assertSame([], $this->owners(['olga@other.de'], bcc: false)); + } + + public function testOutgoingSaslUserAndSentAsMailbox(): void + { + $this->lookup->aliases['info@firma.de'] = 'anna@firma.de,bernd@firma.de'; + $this->lookup->aliasDomains['firma.com'] = 'firma.de'; + $ext = ['someone@extern.example']; + + $this->assertSame(['anna@firma.de'], $this->owners($ext, 'anna@firma.de', 'anna@firma.de')); + $this->assertSame(['anna@firma.de', 'buchhaltung@firma.de'], $this->owners($ext, 'buchhaltung@firma.de', 'anna@firma.de')); + $this->assertSame(['anna@firma.de', 'buchhaltung@firma.de'], $this->owners($ext, 'buchhaltung+x@firma.com', 'Anna@firma.de')); + // an alias as sender is no mailbox + $this->assertSame(['anna@firma.de'], $this->owners($ext, 'info@firma.de', 'anna@firma.de')); + // the sent-as mailbox has archiving off + $this->assertSame(['anna@firma.de'], $this->owners($ext, 'br@firma.de', 'anna@firma.de')); + } + + public function testSenderWithoutLoginNeverOwns(): void + { + $this->assertSame([], $this->owners(['someone@extern.example'], 'anna@firma.de')); + $this->assertSame([], $this->owners(['someone@extern.example'], 'anna@firma.de', 'x1@mailcow.local')); + } + + public function testDirection(): void + { + $resolver = new OwnerResolver($this->lookup); + + $this->assertSame('in', $resolver->resolve('x@extern.example', ['anna@firma.de'], '', false)['direction']); + $this->assertSame('out', $resolver->resolve('anna@firma.de', ['x@extern.example'], 'anna@firma.de', false)['direction']); + $this->assertSame('internal', $resolver->resolve('anna@firma.de', ['olga@other.de'], 'anna@firma.de', false)['direction']); + } +} diff --git a/tests/php/Archive/Service/SettingsServiceTest.php b/tests/php/Archive/Service/SettingsServiceTest.php new file mode 100644 index 000000000..3af90c2f8 --- /dev/null +++ b/tests/php/Archive/Service/SettingsServiceTest.php @@ -0,0 +1,195 @@ +stored = new Settings(); + $this->stored->address = 'journal@archive.example'; + $this->stored->password = 'old-password'; + $this->stored->signingKey = 'old-key'; + $this->repo = $this->createMock(SettingsRepository::class); + $this->repo->method('load')->willReturnCallback(fn(): Settings => clone $this->stored); + $this->redis = new FakeRedis(); + $this->smtp = new FakeSmtpClient(); + $this->lookup = new FakeArchiveLookup(); + $this->lookup->domains = ['firma.de', 'alias-firma.de']; + } + + private function service(?Context $context = null): SettingsService + { + return new SettingsService($this->repo, $this->lookup, new ArchiveState($this->redis), $this->smtp, $context ?? FakeContext::admin()); + } + + private function expectUserError(string|array $msg, callable $call): void + { + try { + $call(); + $this->fail('UserError ' . json_encode($msg) . ' expected'); + } catch (UserError $e) { + $this->assertSame($msg, $e->msg()); + } + } + + public function testSaveKeepsMissingValuesAndEmptySecrets(): void + { + $this->repo->expects($this->once())->method('save')->with($this->callback(fn(Settings $s): bool => $s->enabled + && $s->address === 'journal@archive.example' && $s->password === 'old-password' && $s->signingKey === 'old-key')); + + $settings = $this->service()->save(['enabled' => ['0', '1'], 'password' => '', 'signing_secret' => '']); + + $this->assertSame(1, $settings->toArray()['enabled']); + $this->assertSame('1', $this->redis->strings['ARCHIVE_ENABLED']); + } + + public function testSaveRemovesThePassword(): void + { + $this->assertSame('old-password', $this->service()->save(['password' => '', 'password_remove' => '0'])->password); + $this->assertSame('', $this->service()->save(['password' => '', 'password_remove' => '1'])->password); + } + + public function testSwitchIsKeptInRedisOnly(): void + { + $this->assertFalse($this->service()->enabled()); + $this->assertSame(0, $this->service()->get()->toArray()['enabled']); + $this->service()->save(['enabled' => '1']); + $this->assertSame('1', $this->redis->strings['ARCHIVE_ENABLED']); + $this->assertSame(1, $this->service()->get()->toArray()['enabled']); + // saving other settings keeps the switch + $this->service()->save(['port' => '2525']); + $this->assertTrue($this->service()->enabled()); + // a lost Redis key shows as off, as rspamd sees it + unset($this->redis->strings['ARCHIVE_ENABLED']); + $this->assertFalse($this->service()->enabled()); + } + + public function testAddressOutsideTheMailcowDomains(): void + { + $this->expectUserError('archive_address_local', fn() => $this->service()->save(['address' => 'journal@firma.de'])); + $this->expectUserError('archive_address_local', fn() => $this->service()->save(['address' => 'Journal@Alias-Firma.de'])); + $this->assertSame('journal@archiv.firma.de', $this->service()->save(['address' => 'journal@archiv.firma.de'])->address); + } + + public function testSaveAllFields(): void + { + $settings = $this->service()->save(['enabled' => '0', 'address' => ' Journal@Archive.Example ', 'host' => '[2001:DB8::25]', + 'port' => '2525', 'tls' => 'verify', 'username' => 'mailcow', 'password' => 'new:password', 'profile' => 'envelope', + 'signing_secret' => 'new-key', 'header_from' => 'X-Real-MailFrom', 'header_rcpt' => 'X-Real-RcptTo']); + + $this->assertSame(['enabled' => 0, 'address' => 'journal@archive.example', 'host' => '2001:db8::25', 'port' => 2525, + 'tls' => 'verify', 'username' => 'mailcow', 'has_password' => 1, 'profile' => 'envelope', 'has_signing_secret' => 1, + 'header_from' => 'X-Real-MailFrom', 'header_rcpt' => 'X-Real-RcptTo'], $settings->toArray()); + $this->assertSame(['new:password', 'new-key'], [$settings->password, $settings->signingKey]); + $this->assertSame('0', $this->redis->strings['ARCHIVE_ENABLED']); + } + + public static function invalid(): array + { + return [ + 'address' => [['address' => 'journal'], 'archive_address_invalid'], + 'address array' => [['address' => ['x']], 'archive_address_invalid'], + 'host' => [['host' => 'archive example'], 'archive_host_invalid'], + 'port' => [['port' => '0'], 'archive_port_invalid'], + 'port too high' => [['port' => '65536'], 'archive_port_invalid'], + 'tls' => [['tls' => 'none'], 'archive_tls_invalid'], + 'username with colon' => [['username' => 'a:b'], 'archive_username_invalid'], + 'password with space' => [['password' => 'a b'], 'archive_password_invalid'], + 'password too long' => [['password' => str_repeat('a', 256)], 'archive_password_invalid'], + 'profile' => [['profile' => 'journal'], 'archive_profile_invalid'], + 'signing secret not ascii' => [['signing_secret' => 'schlüssel'], 'archive_signing_secret_invalid'], + 'header name with space' => [['header_from' => 'X Envelope'], 'archive_header_invalid'], + 'header name with line break' => [['header_rcpt' => "X-To\r\nBcc"], 'archive_header_invalid'], + 'switch on without address' => [['enabled' => '1', 'address' => ''], 'archive_incomplete'], + ]; + } + + #[DataProvider('invalid')] + public function testSaveValidates(array $input, string $error): void + { + $this->repo->expects($this->never())->method('save'); + $this->expectUserError($error, fn() => $this->service()->save($input)); + $this->assertArrayNotHasKey('ARCHIVE_ENABLED', $this->redis->strings); + } + + public function testArchivecowNeedsASigningKey(): void + { + $this->stored->signingKey = ''; + $this->expectUserError('archive_incomplete', fn() => $this->service()->save(['enabled' => '1'])); + + $this->assertTrue($this->service()->save(['enabled' => '1', 'profile' => 'none'])->enabled); + } + + public function testPermissions(): void + { + $readOnly = new Context('admin', 'API', [], false, 'ro'); + $this->repo->expects($this->never())->method('save'); + + $this->assertSame(1, $this->service($readOnly)->get()->toArray()['has_password']); + $this->expectUserError('access_denied', fn() => $this->service($readOnly)->save(['enabled' => '0'])); + $this->expectUserError('access_denied', fn() => $this->service($readOnly)->test()); + $this->expectUserError('access_denied', fn() => $this->service(FakeContext::domainAdmin(acl: ['archive' => '1']))->get()); + $this->expectUserError('access_denied', fn() => $this->service(FakeContext::user())->save([])); + $this->assertSame([], $this->smtp->probed); + } + + public function testConnectionTestWithTheStoredTarget(): void + { + $this->stored->host = 'smtp.archive.example'; + $this->stored->port = 587; + $this->stored->tls = 'verify'; + $this->stored->username = 'mailcow'; + + $this->assertSame('archive_test_ok', $this->service()->test()); + $this->assertSame([['host' => 'smtp.archive.example', 'port' => 587, 'tls' => 'verify', 'username' => 'mailcow', + 'password' => 'old-password', 'from' => 'archive-bounce@localhost', 'rcpt' => 'journal@archive.example']], $this->smtp->probed); + } + + public function testConnectionTestReportsMissingExtensions(): void + { + $this->smtp->mx['archive.example'] = 'mx1.archive.example'; + $this->smtp->extensions = ['STARTTLS', 'SIZE']; + + $this->assertSame(['archive_test_ok_missing', '8BITMIME'], $this->service()->test()); + $this->assertSame('mx1.archive.example', $this->smtp->probed[0]['host']); + } + + public function testConnectionTestFailure(): void + { + $this->smtp->error = new SmtpException('recipient', "550 5.1.1 :\n unknown"); + + $this->expectUserError(['archive_test_failed', 'archive.example:25 recipient: 550 5.1.1 journal@archive.example: unknown'], + fn() => $this->service()->test()); + } + + public function testConnectionTestNeedsAnAddress(): void + { + $this->stored->address = ''; + $this->expectUserError('archive_incomplete', fn() => $this->service()->test()); + $this->assertSame([], $this->smtp->probed); + } +} diff --git a/tests/php/Archive/Service/StatusServiceTest.php b/tests/php/Archive/Service/StatusServiceTest.php new file mode 100644 index 000000000..535761488 --- /dev/null +++ b/tests/php/Archive/Service/StatusServiceTest.php @@ -0,0 +1,122 @@ +redis = new FakeRedis(); + $this->redis->strings['ARCHIVE_ENABLED'] = '1'; + } + + private function service(?Context $context = null): StatusService + { + $settings = new Settings(); + $settings->address = 'journal@archive.example'; + $settings->profile = 'none'; + $repo = $this->createStub(SettingsRepository::class); + $repo->method('load')->willReturn($settings); + $queue = $this->createMock(LegacyQueue::class); + $queue->method('entries')->with('archive-bounce@localhost')->willReturn([ + ['queue_id' => 'A1', 'arrival_time' => 1700000100, 'message_size' => 2048, 'recipients' => ['journal@archive.example (' . self::REFUSED . ')']], + ['queue_id' => 'A2', 'arrival_time' => 1700000000, 'message_size' => 1024, 'recipients' => ['journal@archive.example (' . self::REFUSED . ')']], + ['queue_id' => 'A3', 'arrival_time' => 1700000200, 'message_size' => 10, 'recipients' => ['journal@archive.example']], + ]); + return new StatusService($repo, new ArchiveState($this->redis), $queue, $context ?? new Context('admin', 'API', [], false, 'ro')); + } + + /** POSTFIX_MAILLOG line as syslog-ng writes it */ + private static function log(int $time, string $program, string $message): string + { + return json_encode(['time' => (string)$time, 'program' => $program, 'priority' => 'info', 'message' => $message]); + } + + public function testStatus(): void + { + $state = new ArchiveState($this->redis); + // a day old: no longer a problem, still counted + $this->redis->lPush('ARCHIVE_DROPPED_LOG', json_encode(['time' => time() - 90000, 'qid' => 'OLD1', 'reason' => 'old', 'message_id' => ''])); + $state->dropped('4XyZ1', 'owner lookup failed', 'm1@firma.de'); + $this->redis->lists['POSTFIX_MAILLOG'] = [ + self::log(1700000300, 'postfix/archive/smtp', 'A4: to=, relay=archive.example[192.0.2.5]:25, delay=0.2, ' + . 'delays=0.1/0/0.05/0.05, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 77)'), + self::log(1700000280, 'postfix/archive/smtp', 'A1: to=, relay=none, delay=180, dsn=4.4.1, status=deferred (' . self::REFUSED . ')'), + self::log(1700000200, 'postfix/archive/smtp', 'A0: to=, relay=archive.example[192.0.2.5]:25, dsn=2.0.0, status=sent (250 ok)'), + ]; + + $this->assertSame([ + 'enabled' => 1, 'configured' => 1, 'queued' => 3, 'queued_bytes' => 3082, 'oldest_queued' => 1700000000, + 'delay_reasons' => [self::REFUSED], 'last_delivery' => 1700000300, 'dropped' => 1, 'dropped_24h' => 1, + 'last_dropped' => ['time' => $this->redis->lists['ARCHIVE_DROPPED_LOG'] ? json_decode($this->redis->lists['ARCHIVE_DROPPED_LOG'][0], true)['time'] : 0, + 'reason' => 'owner lookup failed'], + ], $this->service()->status()); + } + + public function testSwitchFromRedis(): void + { + $this->redis->strings['ARCHIVE_ENABLED'] = '0'; + $this->assertSame(0, $this->service()->status()['enabled']); + } + + public function testLogWithMessageIds(): void + { + (new ArchiveState($this->redis))->dropped('4XyZ1', 'owner lookup failed', 'm1@firma.de'); + $this->redis->lists['POSTFIX_MAILLOG'] = [ + self::log(1700000310, 'postfix/qmgr', 'E1: from=, status=expired, returned to sender'), + self::log(1700000300, 'postfix/archive/smtp', 'A4: to=, relay=archive.example[192.0.2.5]:25, dsn=2.0.0, ' + . 'status=sent (250 2.0.0 Ok: queued as 77)'), + self::log(1700000295, 'postfix/qmgr', 'Q9: from=, status=expired, returned to sender'), + self::log(1700000290, 'postfix/smtp', 'B1: to=, relay=mx.extern.example[198.51.100.7]:25, dsn=2.0.0, status=sent (250 Ok)'), + self::log(1700000285, 'postfix/archive/discard', 'D1: to=, relay=none, dsn=2.0.0, status=sent (localhost)'), + self::log(1700000280, 'postfix/archive/smtp', 'A2: to=, relay=archive.example[192.0.2.5]:25, dsn=4.1.1, ' + . 'status=SOFTBOUNCE (host archive.example[192.0.2.5] said: 550 5.1.1 unknown (in reply to RCPT TO command))'), + self::log(1700000270, 'postfix/archive/smtp', 'Untrusted TLS connection established to archive.example[192.0.2.5]:25'), + self::log(1700000262, 'postfix/cleanup', 'A4: message-id='), + self::log(1700000261, 'postfix/cleanup', 'A2: message-id='), + 'not json', + ]; + + $log = $this->service()->log(); + + // the mail not archived just now is the newest entry + $this->assertSame(['status' => 'dropped', 'qid' => '4XyZ1', 'message_id' => 'm1@firma.de', 'text' => 'owner lookup failed'], + array_diff_key($log[0], ['time' => 0])); + $this->assertSame([ + ['time' => 1700000310, 'status' => 'expired', 'qid' => 'E1', 'message_id' => '', 'text' => ''], + ['time' => 1700000300, 'status' => 'sent', 'qid' => 'A4', 'message_id' => 'm4@extern.example', 'text' => '250 2.0.0 Ok: queued as 77'], + ['time' => 1700000280, 'status' => 'softbounce', 'qid' => 'A2', 'message_id' => 'm2@firma.de', + 'text' => 'host archive.example[192.0.2.5] said: 550 5.1.1 unknown (in reply to RCPT TO command)'], + ], array_slice($log, 1)); + $this->assertCount(2, $this->service()->log(2)); + } + + public function testOnlyAdmins(): void + { + foreach (['status', 'log'] as $method) { + try { + $this->service(FakeContext::domainAdmin(acl: ['archive' => '1']))->$method(); + $this->fail('UserError expected'); + } catch (UserError) { + $this->addToAssertionCount(1); + } + } + } +} diff --git a/tests/php/Archive/Smtp/SmtpClientTest.php b/tests/php/Archive/Smtp/SmtpClientTest.php new file mode 100644 index 000000000..8ff710eaa --- /dev/null +++ b/tests/php/Archive/Smtp/SmtpClientTest.php @@ -0,0 +1,123 @@ +written[] = "OPEN $host:$port"; + } + + protected function close(): void + { + $this->closed = true; + } + + protected function write(string $data): void + { + $this->written[] = $data; + } + + protected function readLine(): ?string + { + return array_shift($this->replies); + } + + protected function startTls(string $host, bool $verify): void + { + $this->tls[] = [$host, $verify]; + } + }; + } + + public function testSubmitWithCrlfAndDotStuffing(): void + { + $client = self::client(['220 postfix ESMTP', '250-postfix', '250 8BITMIME', '250 2.1.0 Ok', '250 2.1.5 Ok', + '354 End data with .', '250 2.0.0 Ok: queued as 4XyZ1']); + + $qid = $client->submit('postfix', 591, 'archive-bounce@localhost', 'journal@archive.example', "X-A: 1\r\n\r\n.dot\nline\r\n"); + + $this->assertSame('4XyZ1', $qid); + $this->assertSame(['OPEN postfix:591', 'EHLO ' . (gethostname() ?: 'mailcow') . "\r\n", "MAIL FROM:\r\n", + "RCPT TO:\r\n", "DATA\r\n", "X-A: 1\r\n\r\n..dot\r\nline\r\n.\r\n", "QUIT\r\n"], $client->written); + $this->assertTrue($client->closed); + } + + public function testProbeWithStartTlsAndAuth(): void + { + $client = self::client(['220 archive ESMTP', '250-archive', '250-STARTTLS', '250 AUTH PLAIN', '220 2.0.0 Ready', + '250-archive', '250-8BITMIME', '250-SMTPUTF8', '250 AUTH PLAIN', '235 2.7.0 Authenticated', '250 Ok', '250 Ok', '250 Ok']); + + $extensions = $client->probe('archive.example', 25, 'verify', 'user', 'pw', 'archive-bounce@localhost', 'journal@archive.example'); + + $this->assertSame(['8BITMIME', 'SMTPUTF8', 'AUTH'], $extensions); + $this->assertSame([['archive.example', true]], $client->tls); + $this->assertContains('AUTH PLAIN ' . base64_encode("\0user\0pw") . "\r\n", $client->written); + $this->assertContains("RSET\r\n", $client->written); + } + + public function testProbeNeedsStartTlsUnlessOptional(): void + { + try { + self::client(['220 archive ESMTP', '250 archive'])->probe('archive.example', 25, 'encrypt', '', '', 'a@localhost', 'j@a.example'); + $this->fail('SmtpException expected'); + } catch (SmtpException $e) { + $this->assertSame('tls: STARTTLS not offered', $e->describe()); + } + $client = self::client(['220 archive ESMTP', '250 archive', '250 Ok', '250 Ok', '250 Ok']); + $this->assertSame([], $client->probe('archive.example', 25, 'may', '', '', 'a@localhost', 'j@a.example')); + $this->assertSame([], $client->tls); + } + + public function testRefusalsNameTheStep(): void + { + $cases = [ + 'connect' => ['421 4.3.2 Service not available'], + 'auth' => ['220 archive', '250-archive', '250 STARTTLS', '220 Go', '250 archive', '535 5.7.8 failed'], + 'recipient' => ['220 archive', '250-archive', '250 STARTTLS', '220 Go', '250 archive', '235 Ok', '250 Ok', '550 5.1.1 : unknown'], + ]; + foreach ($cases as $step => $replies) { + try { + self::client($replies)->probe('archive.example', 25, 'encrypt', 'user', 'pw', 'a@localhost', 'j@a.example'); + $this->fail("SmtpException at $step expected"); + } catch (SmtpException $e) { + $this->assertSame($step, $e->step); + $this->assertSame(end($replies), $e->reply); + } + } + } + + public function testConnectionClosedDuringData(): void + { + $this->expectException(SmtpException::class); + self::client(['220 postfix', '250 postfix', '250 Ok', '250 Ok', '354 go'])->submit('postfix', 591, 'a@localhost', 'j@a.example', "x\r\n"); + } + + public function testCrlfAndDotStuffing(): void + { + $this->assertSame("a\r\nb\r\nc\r\n", SmtpClient::crlf("a\nb\rc")); + $this->assertSame("a\r\n", SmtpClient::crlf("a\r\n")); + $this->assertSame("..a\r\n..\r\n", SmtpClient::dotStuff(".a\r\n.\r\n")); + } +} diff --git a/tests/php/Support/FakeArchiveLookup.php b/tests/php/Support/FakeArchiveLookup.php new file mode 100644 index 000000000..8985adb18 --- /dev/null +++ b/tests/php/Support/FakeArchiveLookup.php @@ -0,0 +1,87 @@ + [archive setting, domain archive]. + */ +class FakeArchiveLookup extends LookupRepository +{ + public array $recipientMaps = []; + public array $aliases = []; + public array $resources = []; + public array $spamAliases = []; + /** alias domain => target domain */ + public array $aliasDomains = []; + public array $mailboxes = []; + /** ['rcpt' => [key => dest], 'sender' => [key => dest]] */ + public array $bcc = ['rcpt' => [], 'sender' => []]; + /** @var string[] domains and alias domains of the mailcow */ + public array $domains = []; + /** @var string[] every lookup as ":" */ + public array $queries = []; + + public function __construct() + { + } + + public function recipientMap(string $key): ?string + { + $this->queries[] = 'recipientMap:' . $key; + return $this->recipientMaps[$key] ?? null; + } + + public function aliasGoto(string $key): ?string + { + $this->queries[] = 'aliasGoto:' . $key; + return $this->aliases[$key] ?? null; + } + + public function isResource(string $username): bool + { + return in_array($username, $this->resources, true); + } + + public function spamAliasGoto(string $key): ?string + { + return $this->spamAliases[$key] ?? null; + } + + public function aliasDomainMailbox(string $local, string $domain): ?string + { + $target = $this->aliasDomains[$domain] ?? null; + return ($target !== null && isset($this->mailboxes[$local . '@' . $target])) ? $local . '@' . $target : null; + } + + public function bccDestination(string $key, string $type): ?string + { + return $this->bcc[$type][$key] ?? null; + } + + public function isMailcowDomain(string $domain): bool + { + return in_array($domain, $this->domains, true); + } + + public function mailbox(string $username): ?Mailbox + { + if (!isset($this->mailboxes[$username])) { + return null; + } + [$archive, $domainArchive] = $this->mailboxes[$username]; + return Mailbox::fromRow(['username' => $username, 'domain' => substr($username, strrpos($username, '@') + 1), + 'archive' => $archive, 'domain_archive' => $domainArchive, 'name' => '', 'active' => 1]); + } + + public function archivedMailboxes(): array + { + return array_values(array_filter(array_map(fn(string $username): Mailbox => $this->mailbox($username), array_keys($this->mailboxes)), + fn(Mailbox $mailbox): bool => $mailbox->archived())); + } +} diff --git a/tests/php/Support/FakeRedis.php b/tests/php/Support/FakeRedis.php new file mode 100644 index 000000000..ad5a93500 --- /dev/null +++ b/tests/php/Support/FakeRedis.php @@ -0,0 +1,54 @@ +strings[$key] = $value; + return true; + } + + public function get(string $key): string|false + { + return $this->strings[$key] ?? false; + } + + public function incr(string $key): int + { + $this->strings[$key] = (string)((int)($this->strings[$key] ?? 0) + 1); + return (int)$this->strings[$key]; + } + + public function lPush(string $key, string $value): int + { + $this->lists[$key] ??= []; + array_unshift($this->lists[$key], $value); + return count($this->lists[$key]); + } + + public function lTrim(string $key, int $start, int $stop): bool + { + $this->lists[$key] = array_slice($this->lists[$key] ?? [], $start, $stop - $start + 1); + return true; + } + + /** Like Redis, a negative stop counts from the end (-1 is the last element) */ + public function lRange(string $key, int $start, int $stop): array + { + $list = $this->lists[$key] ?? []; + if ($stop < 0) { + $stop += count($list); + } + return array_slice($list, $start, max(0, $stop - $start + 1)); + } +} diff --git a/tests/php/Support/FakeSmtpClient.php b/tests/php/Support/FakeSmtpClient.php new file mode 100644 index 000000000..95bc225d1 --- /dev/null +++ b/tests/php/Support/FakeSmtpClient.php @@ -0,0 +1,46 @@ +error !== null) { + throw $this->error; + } + $this->submitted[] = compact('host', 'port', 'from', 'rcpt', 'data'); + return 'Q' . count($this->submitted); + } + + public function probe(string $host, int $port, string $tls, string $username, string $password, string $from, string $rcpt): array + { + $this->probed[] = compact('host', 'port', 'tls', 'username', 'password', 'from', 'rcpt'); + if ($this->error !== null) { + throw $this->error; + } + return $this->extensions; + } + + public function mailHost(string $domain): string + { + return $this->mx[$domain] ?? $domain; + } +}