diff --git a/data/Dockerfiles/netfilter/modules/BanManager.py b/data/Dockerfiles/netfilter/modules/BanManager.py index 77818dd4a..9a7cee9c5 100644 --- a/data/Dockerfiles/netfilter/modules/BanManager.py +++ b/data/Dockerfiles/netfilter/modules/BanManager.py @@ -7,9 +7,6 @@ class BanManager: and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter (chains, SNAT), because the backends must not run in parallel.""" - # Seconds failed logins of an address are not counted after the auth backend was unavailable for it - EXCUSE_TIME = 60 - def __init__(self, r, tables, options, logger, lock, clock=time.time): self.r = r self.tables = tables @@ -21,16 +18,6 @@ class BanManager: self.banned = set() self.denied = {} self.allowlist = [] - self.excused = {} - - def excuse(self, ip): - """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME""" - with self.lock: - now = self.clock() - if self.excused.get(ip, 0) <= now: - self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % ( - ip, self.EXCUSE_TIME)) - self.excused[ip] = now + self.EXCUSE_TIME def attempt(self, ip): """Counts a failed attempt of ip and bans its network after max_attempts within retry_window""" @@ -43,9 +30,6 @@ class BanManager: if allowed is not None: self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed)) return - if self.excused.get(ip, 0) > now: - self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip) - return if net in self.banned: # log lines of connections from before the ban return @@ -208,8 +192,6 @@ class BanManager: for net in [net for net, counter in self.counters.items() if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]: del self.counters[net] - for ip in [ip for ip, until in self.excused.items() if until <= now]: - del self.excused[ip] def apply(self, nets): for net in self.tables.ban(nets): diff --git a/data/Dockerfiles/netfilter/modules/Netfilter.py b/data/Dockerfiles/netfilter/modules/Netfilter.py index bd75938c8..ccc6c65ee 100644 --- a/data/Dockerfiles/netfilter/modules/Netfilter.py +++ b/data/Dockerfiles/netfilter/modules/Netfilter.py @@ -109,12 +109,6 @@ class Netfilter: def handle_line(self, line): # a broken line must not end the watcher try: - excused = Regexes.temporary_failure(line) - if excused is not None: - ip = Regexes.parse_ip(excused) - if self.external(ip): - self.bans.excuse(ip) - return result = self.regexes.match(line) if result is None: return diff --git a/data/Dockerfiles/netfilter/modules/Regexes.py b/data/Dockerfiles/netfilter/modules/Regexes.py index 8818d4ee3..9cfd00bfd 100644 --- a/data/Dockerfiles/netfilter/modules/Regexes.py +++ b/data/Dockerfiles/netfilter/modules/Regexes.py @@ -6,20 +6,41 @@ import re class Regexes: """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address""" + # Anchored to the line start: text clients control in other lines (login names, SMTP commands) must not match. + # Failed IMAP, POP3, Sieve and SMTP logins are reported by mailcowauth.php (rule 3), not by Dovecot or Postfix. DEFAULTS = { - 1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)', - 2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)', - 3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+', - 4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+', - 5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+', - 6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)', - 7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)', - 8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked', - 9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+', + 1: r'^mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)$', + 2: r'^Rspamd UI: Invalid password by ([0-9a-f\.:]+)$', + 3: r'^MAILCOWAUTH: Login failed for user .+ with service \w* from IP ([0-9a-f\.:]+)$', + 4: r'^warning: non-SMTP command from [^\[\]\s]*\[([0-9a-f\.:]+)\]:.+', + 5: r'^NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+', + 6: r'^(?:\[\d+\]: )?SOGo\w* Login from \'(?:[0-9a-f\.:]+, )*([0-9a-f\.:]+)\' for user .+ might not have worked', + 7: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+', } - # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs - # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while. - TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)') + # Former default rules (with any rule id) => rule id of their replacement, None: the rule is removed. + # Rules the admin changed are kept. + FORMER_DEFAULTS = { + r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)': 1, + r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)': 2, + r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed': 3, + r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+': 3, + r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+': 4, + r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+': 5, + r"SOGo.+ Login from '([0-9a-f\.:]+)' for user .+ might not have worked": 6, + r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked': 6, + r'-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None, + r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None, + r'-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None, + r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None, + r'-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None, + r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None, + r'auth: static\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None, + r'auth: \w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None, + r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None, + r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)': None, + } + # Lines of the web UI are not sanitized like syslog lines: a newline in a login name must not hide the address + CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f]') def __init__(self, logger): self.logger = logger @@ -42,6 +63,10 @@ class Regexes: if not isinstance(stored, dict): self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules') return + migrated = self.migrate(stored) + if migrated != stored: + self.store_migrated(r) + stored = migrated rules = [] for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])): @@ -54,6 +79,7 @@ class Regexes: def match(self, line): """(rule id, captured address) of the first rule that matches the line, None if no rule matches""" + line = self.CONTROL_CHARS.sub('_', line) for rule_id, regex in self.rules: result = regex.search(line) if result and result.lastindex and result.group(1): @@ -61,10 +87,35 @@ class Regexes: return None @classmethod - def temporary_failure(cls, line): - """Address of a login that failed because the auth backend was unavailable, None for other lines""" - result = cls.TEMPORARY_FAILURE.search(line) - return result.group(1) if result else None + def migrate(cls, stored): + """The stored rules with former default rules replaced by the current ones (or removed)""" + migrated = {} + for rule_id, regex in stored.items(): + if isinstance(regex, str) and regex in cls.FORMER_DEFAULTS: + replacement = cls.FORMER_DEFAULTS[regex] + if replacement is None: + continue + regex = cls.DEFAULTS[replacement] + migrated[rule_id] = regex + return migrated + + def store_migrated(self, r): + # WATCH keeps a save of the UI between reading and writing + def replace(pipe): + current = json.loads(pipe.get('F2B_REGEX') or '{}') + if not isinstance(current, dict): + return + migrated = self.migrate(current) + if migrated == current: + return + pipe.multi() + pipe.set('F2B_REGEX', json.dumps(migrated, ensure_ascii=False)) + + try: + r.transaction(replace, 'F2B_REGEX') + self.logger.logInfo('Replaced former default rules in F2B_REGEX') + except Exception as ex: + self.logger.logWarn('Could not replace former default rules in F2B_REGEX: %s' % ex) @staticmethod def parse_ip(text): diff --git a/data/conf/dovecot/auth/mailcowauth.php b/data/conf/dovecot/auth/mailcowauth.php index c8a73e39e..e18e8fcec 100644 --- a/data/conf/dovecot/auth/mailcowauth.php +++ b/data/conf/dovecot/auth/mailcowauth.php @@ -68,6 +68,8 @@ require_once 'functions.acl.inc.php'; $isSOGoRequest = $post['real_rip'] == getenv('IPV4_NETWORK') . '.248'; +// real_rip is the address that connected to Dovecot (for proxied connections the proxy), client_ip the client +$client_ip = filter_var($post['client_ip'] ?? '', FILTER_VALIDATE_IP) ? $post['client_ip'] : $post['real_rip']; $result = false; if ($isSOGoRequest) { // This is a SOGo Auth request. First check for SSO password. @@ -121,7 +123,14 @@ if ($result) { http_response_code(200); // OK $return['success'] = true; } else { - error_log("MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . $post['service'] . " from IP " . $post['real_rip']); + // Same line for the log and netfilter, which counts it for IMAP, POP3, Sieve and SMTP (Postfix asks Dovecot); + // SOGo logins have their own rule + $failed = "MAILCOWAUTH: Login failed for user " . $post['username'] . " with service " . + preg_replace('/[^a-z0-9]/', '', strtolower((string)$service)) . " from IP " . $client_ip; + error_log($failed); + if (!$isSOGoRequest) { + $redis->publish("F2B_CHANNEL", $failed); + } http_response_code(401); // Unauthorized } diff --git a/data/conf/dovecot/auth/passwd-verify.lua b/data/conf/dovecot/auth/passwd-verify.lua index ddd5a316a..cb4a5ebdc 100644 --- a/data/conf/dovecot/auth/passwd-verify.lua +++ b/data/conf/dovecot/auth/passwd-verify.lua @@ -1,8 +1,4 @@ function auth_password_verify(request, password) - if request.domain == nil then - return dovecot.auth.PASSDB_RESULT_USER_UNKNOWN, "No such user" - end - local json = require "cjson" local ltn12 = require "ltn12" local https = require "ssl.https" @@ -11,7 +7,10 @@ function auth_password_verify(request, password) local req = { username = request.user, password = password, + -- address that connected to Dovecot, for proxied connections the proxy real_rip = request.real_rip, + -- address of the client, also for proxied connections + client_ip = request.rip, service = request.service } local req_json = json.encode(req) diff --git a/data/conf/dovecot/dovecot.conf b/data/conf/dovecot/dovecot.conf index c3789fb24..c1da4fb7c 100644 --- a/data/conf/dovecot/dovecot.conf +++ b/data/conf/dovecot/dovecot.conf @@ -51,13 +51,6 @@ recipient_delimiter = + auth_master_user_separator = * mail_shared_explicit_inbox = yes mail_prefetch_count = 30 -passdb { - driver = lua - args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w - result_success = return-ok - result_failure = continue - result_internalfail = continue -} # try a master passwd passdb { driver = passwd-file @@ -67,8 +60,9 @@ passdb { } # check for regular password - if empty (e.g. force-passwd-reset), previous pass=yes passdbs also fail # a return of the following passdb is mandatory +# each check asks mailcowauth.php, which reports a failed login to netfilter once # cache_key with password: the lua default (%u) would let a temporary failure fall back to the negative entry -# of a passdb lookup (master logins) and log "unknown user", which netfilter counts as failed login +# of a passdb lookup (master logins) and reject the login as unknown user passdb { driver = lua args = file=/etc/dovecot/auth/passwd-verify.lua blocking=yes cache_key=%s:%u:%w diff --git a/docker-compose.yml b/docker-compose.yml index 9e3d5c424..473b33466 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -515,7 +515,7 @@ services: - acme netfilter-mailcow: - image: ghcr.io/mailcow/netfilter:1.66 + image: ghcr.io/mailcow/netfilter:1.67 stop_grace_period: 30s restart: always privileged: true diff --git a/tests/netfilter/test_ban_manager.py b/tests/netfilter/test_ban_manager.py index ebacea568..f40dcbd75 100644 --- a/tests/netfilter/test_ban_manager.py +++ b/tests/netfilter/test_ban_manager.py @@ -188,25 +188,6 @@ class BanManagerTest(unittest.TestCase): self.bans.purge() self.assertEqual(self.bans.counters, {}) - def test_excused_address_is_not_counted(self): - ip = ipaddress.ip_address('203.0.113.1') - self.bans.excuse(ip) - self.bans.excuse(ip) - self.attempts('203.0.113.1', 5) - self.assertEqual(self.active(), {}) - self.assertEqual(len([message for message in self.logger.messages('warn') if 'Auth backend unavailable' in message]), 1) - # other addresses are counted - self.attempts('203.0.113.2', 3) - self.assertIn('203.0.113.2/32', self.active()) - - def test_excuse_ends(self): - self.bans.excuse(ipaddress.ip_address('203.0.113.1')) - self.clock.now += BanManager.EXCUSE_TIME + 1 - self.bans.purge() - self.assertEqual(self.bans.excused, {}) - self.attempts('203.0.113.1', 3) - self.assertIn('203.0.113.1/32', self.active()) - if __name__ == '__main__': unittest.main() diff --git a/tests/netfilter/test_netfilter.py b/tests/netfilter/test_netfilter.py index dc702335b..8ba5eabd4 100644 --- a/tests/netfilter/test_netfilter.py +++ b/tests/netfilter/test_netfilter.py @@ -25,8 +25,8 @@ class NetfilterTest(unittest.TestCase): netfilter.regexes.refresh(self.r) return netfilter - def sasl(self, address): - return 'warning: unknown[%s]: SASL LOGIN authentication failed: UGFzc3dvcmQ6' % address + def failed(self, address): + return 'MAILCOWAUTH: Login failed for user a@example.org with service imap from IP %s' % address def test_setup_restores_bans_and_resets_the_firewall(self): self.r.hset('F2B_ACTIVE_BANS', '1.2.3.4/32', '9999999999') @@ -49,8 +49,8 @@ class NetfilterTest(unittest.TestCase): def test_shutdown_keeps_the_active_bans(self): netfilter = self.create() - netfilter.handle_line(self.sasl('1.2.3.4')) - netfilter.handle_line(self.sasl('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) self.r.hset('F2B_PERM_BANS', '198.51.100.1', '1') netfilter.shutdown() self.assertEqual(self.tables.rules, set()) @@ -59,52 +59,51 @@ class NetfilterTest(unittest.TestCase): def test_handle_line_bans(self): netfilter = self.create() - netfilter.handle_line(self.sasl('1.2.3.4')) - netfilter.handle_line(self.sasl('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) self.assertEqual(self.tables.rules, {'1.2.3.4/32'}) - self.assertIn('1.2.3.4 matched rule id 3 (%s)' % self.sasl('1.2.3.4'), self.logger.messages('warn')) + self.assertIn('1.2.3.4 matched rule id 3 (%s)' % self.failed('1.2.3.4'), self.logger.messages('warn')) def test_handle_line_with_trailing_colon(self): netfilter = self.create() - netfilter.handle_line(self.sasl('1.2.3.4:')) - netfilter.handle_line(self.sasl('1.2.3.4:')) + netfilter.handle_line(self.failed('1.2.3.4:')) + netfilter.handle_line(self.failed('1.2.3.4:')) self.assertEqual(self.tables.rules, {'1.2.3.4/32'}) def test_handle_line_ignores_private_and_invalid_addresses(self): netfilter = self.create() for _ in range(3): - netfilter.handle_line(self.sasl('192.168.1.1')) - netfilter.handle_line(self.sasl('::1')) - netfilter.handle_line(self.sasl('1.2.3')) + netfilter.handle_line(self.failed('192.168.1.1')) + netfilter.handle_line(self.failed('::1')) + netfilter.handle_line(self.failed('1.2.3')) self.assertEqual(self.tables.rules, set()) self.assertTrue(any('invalid address 1.2.3' in message for message in self.logger.messages('warn'))) def test_handle_line_with_replaced_characters(self): netfilter = self.create() - line = b'warning: unknown[1.2.3.4]: SASL LOGIN authentication failed: \xff\xfe'.decode('utf-8', 'replace') + line = b'MAILCOWAUTH: Login failed for user \xff\xfe with service imap from IP 1.2.3.4'.decode('utf-8', 'replace') netfilter.handle_line(line) netfilter.handle_line(line) self.assertEqual(self.tables.rules, {'1.2.3.4/32'}) - def test_unavailable_auth_backend_is_no_failed_login(self): - # Dovecot logs the temporary failure, Postfix then the failed SASL login like for a wrong password + def test_postfix_and_dovecot_lines_are_no_failed_logins(self): + # mailcowauth.php reports failed logins, not when the auth backend was unavailable netfilter = self.create() - netfilter.handle_line('auth-worker(1): Error: conn unix:auth-worker (pid=2,uid=401): auth-worker<4>: ' - 'lua(a@example.org,1.2.3.4): passdb-lua: Upstream unavailable') for _ in range(3): netfilter.handle_line('warning: unknown[1.2.3.4]: SASL LOGIN authentication failed: (reason unavailable), sasl_username=a@example.org') + netfilter.handle_line('imap-login: Disconnected: Connection closed (auth failed, 1 attempts in 2 secs): user=, ' + 'method=PLAIN, rip=1.2.3.4, lip=172.22.1.250') self.assertEqual(self.tables.rules, set()) - self.assertNotIn('1.2.3.4/32', self.r.hgetall('F2B_ACTIVE_BANS')) def test_handle_line_survives_errors(self): netfilter = self.create() with mock.patch.object(netfilter.bans, 'attempt', side_effect=RuntimeError('boom')): - netfilter.handle_line(self.sasl('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) self.assertTrue(any('boom' in message for message in self.logger.messages('warn'))) def test_watch_reconnects_after_errors(self): netfilter = self.create() - self.r.messages.extend([self.sasl('1.2.3.4'), ConnectionError('gone'), self.sasl('1.2.3.4')]) + self.r.messages.extend([self.failed('1.2.3.4'), ConnectionError('gone'), self.failed('1.2.3.4')]) netfilter.loop('watch', netfilter.watch, 0) self.assertEqual(self.tables.rules, {'1.2.3.4/32'}) self.assertEqual(len(self.r.pubsubs), 2) @@ -126,8 +125,8 @@ class NetfilterTest(unittest.TestCase): def test_recreated_chain_gets_isolation_and_bans_again(self): netfilter = self.create() - netfilter.handle_line(self.sasl('1.2.3.4')) - netfilter.handle_line(self.sasl('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) self.tables.rules = set() self.tables.calls = [] self.tables.recreate = True @@ -151,8 +150,8 @@ class NetfilterTest(unittest.TestCase): self.r.hset('F2B_BLACKLIST', '198.51.100.1', 1) netfilter.update_lists() self.assertEqual(self.tables.rules, {'198.51.100.1/32'}) - netfilter.handle_line(self.sasl('1.2.3.4')) - netfilter.handle_line(self.sasl('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) + netfilter.handle_line(self.failed('1.2.3.4')) self.assertNotIn('1.2.3.4/32', self.tables.rules) def test_resolve_keeps_addresses_on_dns_errors(self): diff --git a/tests/netfilter/test_regexes.py b/tests/netfilter/test_regexes.py index e55cac4f5..3a4bf6b77 100644 --- a/tests/netfilter/test_regexes.py +++ b/tests/netfilter/test_regexes.py @@ -10,14 +10,33 @@ class RegexesTest(unittest.TestCase): LINES = { '1': ('mailcow UI: Invalid password for admin by 203.0.113.5', '203.0.113.5'), '2': ('Rspamd UI: Invalid password by 2001:db8::5', '2001:db8::5'), - '3': ('warning: unknown[203.0.113.6]: SASL LOGIN authentication failed: UGFzc3dvcmQ6', '203.0.113.6'), - '4': ('warning: non-SMTP command from unknown[203.0.113.7]: GET / HTTP/1.1', '203.0.113.7'), - '5': ('NOQUEUE: reject: RCPT from [203.0.113.8]: 503 5.5.0 : Protocol error; proto=ESMTP', '203.0.113.8'), - '6': ('auth: lua(user@example.org,203.0.113.9,): Password mismatch (SHA1 of given password: 0123ab)', '203.0.113.9'), - '7': ('auth: lua(nobody@example.org,203.0.113.10,): unknown user (SHA1 of given password: 0123ab)', '203.0.113.10'), - '8': ("SOGo[123]: Login from '203.0.113.11' for user nobody might not have worked", '203.0.113.11'), - '9': ('203.0.113.12 "GET /SOGo/so/ HTTP/1.1" 403 123 "-"', '203.0.113.12'), + '3': ('MAILCOWAUTH: Login failed for user user@example.org with service smtp from IP 203.0.113.6', '203.0.113.6'), + '4': ('warning: non-SMTP command from unknown[203.0.113.7]: GET _ HTTP_1.1', '203.0.113.7'), + '5': ('NOQUEUE: reject: RCPT from [203.0.113.8]:52516: 550 5.5.1 Protocol error; from= to=', + '203.0.113.8'), + '6': ("[6714]: SOGoRootPage Login from '203.0.113.11' for user 'nobody@example.org' might not have worked - " + "password policy: 65535 grace: -1 expire: -1 bound: 0", '203.0.113.11'), + '7': ('203.0.113.12 "GET /SOGo/so/ HTTP/1.1" 403 123 "-"', '203.0.113.12'), } + # Defaults of 2025-02 to 2026-10 and of 2022-11 to 2025-02 + FORMER = { + '1': r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)', + '2': r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)', + '3': r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+', + '4': r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+', + '5': r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+', + '6': r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)', + '7': r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)', + '8': r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked', + '9': r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+', + } + OLDER = dict(FORMER, **{ + '6': r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),', + '7': r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+', + '8': r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+', + '9': r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked', + '10': r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+', + }) def setUp(self): self.r = FakeRedis() @@ -30,10 +49,65 @@ class RegexesTest(unittest.TestCase): for rule_id, (line, address) in self.LINES.items(): self.assertEqual(self.regexes.match(line), (rule_id, address), line) - def test_lost_connection_to_auth_server_is_no_attempt(self): + def test_postfix_and_dovecot_lines_are_no_attempts(self): + # mailcowauth.php reports the failed login once, also when Postfix asked Dovecot self.regexes.refresh(self.r) - line = 'warning: unknown[203.0.113.6]: SASL LOGIN authentication failed: Connection lost to authentication server' - self.assertIsNone(self.regexes.match(line)) + for line in ('warning: unknown[203.0.113.6]: SASL LOGIN authentication failed: (reason unavailable), sasl_username=a@example.org', + 'auth-worker(37358): conn unix:auth-worker (pid=577,uid=401): auth-worker<25>: ' + 'lua(a@example.org,203.0.113.6,): Password mismatch (SHA1 of given password: 6426a0)', + 'imap-login: Disconnected: Connection closed (auth failed, 1 attempts in 2 secs): user=, ' + 'method=PLAIN, rip=203.0.113.6, lip=172.22.1.250'): + self.assertIsNone(self.regexes.match(line), line) + + def test_text_of_clients_does_not_match(self): + # Login names, SMTP commands and addresses of other lines must not ban a foreign address + self.regexes.refresh(self.r) + self.assertEqual(self.regexes.match('warning: non-SMTP command from unknown[198.51.100.7]: ' + 'mailcow UI: Invalid password for x by 203.0.113.1'), ('4', '198.51.100.7')) + self.assertEqual(self.regexes.match('warning: non-SMTP command from unknown[198.51.100.7]: x [203.0.113.1]: y'), + ('4', '198.51.100.7')) + self.assertIsNone(self.regexes.match('NOQUEUE: reject: RCPT from unknown[198.51.100.7]: 554 5.7.1 Relay access denied; ' + 'from=')) + self.assertIsNone(self.regexes.match('improper command pipelining after EHLO from unknown[198.51.100.7]: ' + 'MAILCOWAUTH: Login failed for user x with service imap from IP 203.0.113.1')) + self.assertEqual(self.regexes.match('MAILCOWAUTH: Login failed for user x with service imap from IP 203.0.113.1 ' + 'with service smtp from IP 198.51.100.7'), + ('3', '198.51.100.7')) + self.assertEqual(self.regexes.match("[1]: SOGoRootPage Login from '198.51.100.7' for user 'x' might not have worked " + "SOGoRootPage Login from '203.0.113.1' for user y might not have worked"), ('6', '198.51.100.7')) + # nginx appends the client to the X-Forwarded-For header a client sent + self.assertEqual(self.regexes.match("[1]: SOGoRootPage Login from '203.0.113.1, 198.51.100.7' for user 'x' might not have worked"), + ('6', '198.51.100.7')) + # lines of the web UI are not sanitized like syslog lines + self.assertEqual(self.regexes.match('mailcow UI: Invalid password for admin\n by 203.0.113.5'), ('1', '203.0.113.5')) + + def test_former_defaults_are_replaced(self): + self.r.set('F2B_REGEX', json.dumps(self.FORMER)) + self.regexes.refresh(self.r) + defaults = {str(rule_id): regex for rule_id, regex in Regexes.DEFAULTS.items()} + expected = dict(defaults, **{'8': defaults['6'], '9': defaults['7']}) + del expected['6'], expected['7'] + self.assertEqual(json.loads(self.r.get('F2B_REGEX')), expected) + self.assertEqual(self.regexes.match(self.LINES['3'][0]), ('3', '203.0.113.6')) + self.assertEqual(self.regexes.match(self.LINES['6'][0]), ('8', '203.0.113.11')) + + def test_older_defaults_are_replaced(self): + self.r.set('F2B_REGEX', json.dumps(self.OLDER)) + self.regexes.refresh(self.r) + stored = json.loads(self.r.get('F2B_REGEX')) + self.assertEqual(sorted(stored, key=int), ['1', '2', '3', '4', '5', '9', '10']) + self.assertEqual(stored['9'], Regexes.DEFAULTS[6]) + self.assertEqual(stored['10'], self.OLDER['10']) + + def test_changed_rules_are_kept(self): + custom = {'1': r'mailcow UI: Invalid password for (?:x|y) by ([0-9a-f\.:]+)', '2': self.FORMER['3'], '3': r'custom ([0-9\.]+)'} + self.r.set('F2B_REGEX', json.dumps(custom)) + self.regexes.refresh(self.r) + self.assertEqual(json.loads(self.r.get('F2B_REGEX')), dict(custom, **{'2': Regexes.DEFAULTS[3]})) + # current rules are not written again + raw = self.r.get('F2B_REGEX') + self.regexes.refresh(self.r) + self.assertEqual(self.r.get('F2B_REGEX'), raw) def test_only_the_first_matching_rule_counts(self): self.r.set('F2B_REGEX', json.dumps({'2': r'by ([0-9a-f\.:]+)', '10': r'from ([0-9a-f\.:]+)', '1': r'login from ([0-9a-f\.:]+)'})) @@ -58,13 +132,6 @@ class RegexesTest(unittest.TestCase): self.regexes.refresh(self.r) self.assertIsNone(self.regexes.match('failed by nobody')) - def test_temporary_failure(self): - worker = 'auth-worker(3933): Error: conn unix:auth-worker (pid=3932,uid=401): auth-worker<4>: ' - self.assertEqual(Regexes.temporary_failure(worker + 'lua(a@example.org,203.0.113.1,): passdb-lua: Upstream unavailable'), '203.0.113.1') - self.assertEqual(Regexes.temporary_failure(worker + 'lua(a@example.org,2001:db8::1): passdb-lua: Upstream unreachable'), '2001:db8::1') - self.assertIsNone(Regexes.temporary_failure('auth: lua(a@example.org,203.0.113.1,): unknown user (SHA1 of given password: 0123ab)')) - self.assertIsNone(Regexes.temporary_failure('warning: unknown[203.0.113.1]: SASL LOGIN authentication failed: (reason unavailable)')) - def test_parse_ip(self): self.assertEqual(Regexes.parse_ip('203.0.113.1'), ipaddress.ip_address('203.0.113.1')) self.assertEqual(Regexes.parse_ip('203.0.113.1:'), ipaddress.ip_address('203.0.113.1'))