Merge branch 'staging' into nightly

data/conf/dovecot/auth/mailcowauth.php

@@ -116,8 +116,8 @@ if ($result === false){
     exit;
   }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip);
-    set_sasl_log($post['username'], $client_ip, $post['service']);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+    set_sasl_log($post['username'], $client_ip, $service);
   }
 }
 

data/conf/nginx/templates/sites-default.conf.j2

@@ -94,11 +94,11 @@ location / {
 }
 
 location /qhandler {
-    rewrite ^/qhandler/(.*)/(.*) /qhandler.php?action=$1&hash=$2;
+    rewrite ^/qhandler/([^/]+)/(.+?)/?$ /qhandler.php?action=$1&hash=$2;
 }
 
 location /edit {
-    rewrite ^/edit/(.*)/(.*) /edit.php?$1=$2;
+    rewrite ^/edit/([^/]+)/(.+?)/?$ /edit.php?$1=$2;
 }
 
 location ~ ^/api/v1/(.*)$ {
@@ -187,6 +187,12 @@ location /sogo-auth-verify {
 }
 
 location ^~ /Microsoft-Server-ActiveSync {
+    # Outlook for iOS/Android (Microsoft's sync service) sends EAS commands without credentials
+    # and only authenticates after a 401 Basic challenge. SOGo answers those with 403, so challenge here.
+    if ($http_authorization = "") {
+        add_header WWW-Authenticate 'Basic realm="SOGo"' always;
+        return 401;
+    }
     auth_request /sogo-auth-verify;
     auth_request_set $user $upstream_http_x_user;
     auth_request_set $auth $upstream_http_x_auth;

data/conf/rspamd/local.d/composites.conf

@@ -1,7 +1,3 @@
-MX_IMPLICIT {
-  expression = "MX_GOOD & MX_MISSING";
-  score = -0.01;
-}
 VIRUS_FOUND {
   expression = "CLAM_VIRUS & !MAILCOW_WHITE";
   score = 2000.0;

data/conf/rspamd/local.d/groups.conf

@@ -23,24 +23,6 @@ symbols {
   }
 }
 
-group "MX" {
-  "MX_INVALID" {
-    score = 0.5;
-    description = "No connectable MX";
-    one_shot = true;
-  }
-  "MX_MISSING" {
-    score = 2.0;
-    description = "No MX record";
-    one_shot = true;
-  }
-  "MX_GOOD" {
-    score = -0.01;
-    description = "MX was ok";
-    one_shot = true;
-  }
-}
-
 group "reputation" {
   symbols = {
     "IP_REPUTATION_HAM" {

data/conf/rspamd/local.d/multimap.conf

@@ -5,12 +5,6 @@ RCPT_MAILCOW_DOMAIN {
   symbols_set = ["RCPT_MAILCOW_DOMAIN"];
 }
 
-WHITELISTED_FWD_HOST {
-  type = "ip";
-  map = "redis://WHITELISTED_FWD_HOST";
-  symbols_set = ["WHITELISTED_FWD_HOST"];
-}
-
 BULK_HEADER {
   type = "content";
   map = "${LOCAL_CONFDIR}/custom/bulk_header.map";
@@ -151,20 +145,6 @@ BAD_LANG {
   score = 5.0;
 }
 
-BAZAAR_ABUSE_CH {
-  type = "selector";
-  selector = "attachments(hex,md5)";
-  map = "https://bazaar.abuse.ch/export/txt/md5/recent/";
-  score = 10.0;
-}
-
-URLHAUS_ABUSE_CH {
-  type = "selector";
-  selector = "urls";
-  map = "https://urlhaus.abuse.ch/downloads/text_online/";
-  score = 10.0;
-}
-
 SMTP_LIMITED_ACCESS {
   type = "user";
   map = "redis://SMTP_LIMITED_ACCESS";

data/conf/rspamd/local.d/mx_check.conf

@@ -1,6 +1,5 @@
 timeout = 8.0;
 symbol_bad_mx = "MX_INVALID";
-symbol_no_mx = "MX_MISSING";
 symbol_good_mx = "MX_GOOD";
 expire = 86400;
 key_prefix = "rmx";

data/conf/rspamd/local.d/options.inc

@@ -9,3 +9,5 @@ disable_monitoring = true;
 # instead of silently accepting the message without further processing.
 soft_reject_on_timeout = true;
 local_addrs = /etc/rspamd/custom/mailcow_networks.map;
+# Must not be lower than message_size_limit in postfix main.cf
+max_message = 134217728;

data/conf/rspamd/lua/rspamd.local.lua

@@ -362,6 +362,61 @@ rspamd_config:register_symbol({
   priority = 19
 })
 
+rspamd_config:register_symbol({
+  name = 'WHITELISTED_FWD_HOST',
+  type = 'prefilter',
+  score = 0.0,
+  callback = function(task)
+    local rspamd_logger = require "rspamd_logger"
+    local redis_params = rspamd_parse_redis_server('fwd_host')
+    local ip = task:get_from_ip()
+
+    if ip == nil or not ip:is_valid() then
+      return false
+    end
+
+    local from_ip_string = tostring(ip)
+    local ip_check_table = {'WHITELISTED_FWD_HOST', from_ip_string}
+
+    local maxbits = 128
+    local minbits = 7
+    if ip:get_version() == 4 then
+        maxbits = 32
+        minbits = 8
+    end
+    for i=maxbits,minbits,-1 do
+      table.insert(ip_check_table, ip:apply_mask(i):to_string() .. "/" .. i)
+    end
+
+    local function fwd_host_cb(err, data)
+      if err or type(data) ~= 'table' then
+        rspamd_logger.infox(rspamd_config, "fwd_host query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
+        return
+      end
+      for _,v in pairs(data) do
+        if type(v) == 'string' then
+          task:insert_result('WHITELISTED_FWD_HOST', 1.0, from_ip_string)
+          task:set_flag('pass_all')
+          return
+        end
+      end
+    end
+
+    local redis_ret = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'WHITELISTED_FWD_HOST', -- hash key
+      false, -- is write
+      fwd_host_cb, --callback
+      'HMGET', -- command
+      ip_check_table -- arguments
+    )
+    if not redis_ret then
+      rspamd_logger.infox(rspamd_config, "cannot check fwd_host redis map")
+    end
+  end,
+  priority = 18
+})
+
 rspamd_config:register_symbol({
   name = 'TLS_HEADER',
   type = 'postfilter',
@@ -390,9 +445,8 @@ rspamd_config:register_symbol({
     local rspamd_logger = require "rspamd_logger"
     local redis_params = rspamd_parse_redis_server('taghandler')
     local rspamd_http = require "rspamd_http"
-    local rcpts = task:get_recipients('smtp')
+    local rcpts = task:get_recipients({'smtp', 'orig'})
     local lua_util = require "lua_util"
-    local tagged_rcpt = task:get_symbol("TAGGED_RCPT")
 
     local function remove_moo_tag()
       local moo_tag_header = task:get_header('X-Moo-Tag', false)
@@ -417,8 +471,9 @@ rspamd_config:register_symbol({
 
     -- Check if recipient has a tag (contains '+')
     local tag = nil
-    if tagged_rcpt ~= nil then
-      tag = tagged_rcpt
+    local base_user, tag_part = rcpt_user:match('^([^%+]+)%+(.+)$')
+    if base_user and tag_part then
+      tag = tag_part
       rspamd_logger.infox("TAG_MOO: found tag in recipient: %s (base: %s, tag: %s)", rcpt_addr, base_user, tag)
     end
 
@@ -498,8 +553,7 @@ rspamd_config:register_symbol({
           else
             rspamd_logger.infox("TAG_MOO: user wants subject modified for tagged mail")
             local sbj = task:get_header('Subject') or ''
-            local tag_value = tag[1] and tag[1].options and tag[1].options[1] or ''
-            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag_value .. '] ' .. sbj)) .. '?='
+            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag .. '] ' .. sbj)) .. '?='
             task:set_milter_reply({
               remove_headers = {
                 ['Subject'] = 1,
@@ -907,6 +961,9 @@ rspamd_config:register_symbol({
 
             if not seen_cte and rewrite.need_rewrite_ct then
               out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', new_cte)
+              task:set_milter_reply({
+                add_headers = {['Content-Transfer-Encoding'] = new_cte}
+              })
             end
 
             -- End of headers

data/conf/rspamd/meta_exporter/pipe.php

@@ -109,7 +109,7 @@ foreach ($rcpts as $rcpt) {
   }
 
   // Skip if domain is excluded
-  if (in_array($parsed_rcpt['domain'], $exclude_domains)) {
+  if (is_array($exclude_domains) && in_array($parsed_rcpt['domain'], $exclude_domains)) {
     error_log(sprintf("QUARANTINE: Skipped domain %s", $parsed_rcpt['domain']) . PHP_EOL);
     continue;
   }