From 7c62a57910bf37253409b907d749e390c12dd35f Mon Sep 17 00:00:00 2001 From: FreddleSpl0it <75116288+FreddleSpl0it@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:45:05 +0200 Subject: [PATCH] [Web][Dovecot] Rework imapsync: sync sources, OAuth2 (XOAUTH2), worker pool, Python runner --- data/Dockerfiles/dovecot/Dockerfile | 4 +- data/Dockerfiles/dovecot/docker-entrypoint.sh | 2 +- data/Dockerfiles/dovecot/imapsync_runner.pl | 196 --- data/Dockerfiles/dovecot/imapsync_runner.py | 274 ++++ .../phpfpm/crons/imapsync-oauth-refresh.php | 73 ++ data/web/admin/mailbox.php | 2 + data/web/admin/system.php | 1 + data/web/api/openapi.yaml | 1162 ++++++++++++++--- data/web/domainadmin/mailbox.php | 2 + data/web/edit.php | 10 +- data/web/inc/ajax/syncjob_logs.php | 2 +- data/web/inc/footer.inc.php | 5 +- data/web/inc/functions.admin.inc.php | 6 +- data/web/inc/functions.domain_admin.inc.php | 7 +- data/web/inc/functions.mailbox.inc.php | 493 +------ data/web/inc/init_db.inc.php | 312 ++++- data/web/inc/vars.inc.php | 221 ++-- data/web/js/build/013-mailcow.js | 10 +- data/web/js/site/imapsync.js | 707 ++++++++++ data/web/js/site/mailbox.js | 59 +- data/web/js/site/user.js | 62 +- data/web/json_api.php | 104 +- data/web/lang/lang.en-gb.json | 209 ++- data/web/syncjob-oauth.php | 103 ++ data/web/templates/admin.twig | 2 + .../templates/admin/tab-config-syncjob.twig | 35 + data/web/templates/edit.twig | 1 + data/web/templates/edit/imapsync_source.twig | 233 ++++ data/web/templates/edit/syncjob.twig | 133 +- data/web/templates/mailbox.twig | 3 +- data/web/templates/mailbox/tab-syncjobs.twig | 43 +- .../web/templates/modals/imapsync_source.twig | 177 +++ data/web/templates/modals/mailbox.twig | 126 +- data/web/templates/modals/user.twig | 120 +- data/web/templates/user.twig | 2 +- data/web/templates/user/Syncjobs.twig | 29 +- .../templates/user_domainadmin_common.twig | 1 + data/web/user.php | 2 + docker-compose.yml | 7 +- 39 files changed, 3582 insertions(+), 1358 deletions(-) delete mode 100644 data/Dockerfiles/dovecot/imapsync_runner.pl create mode 100755 data/Dockerfiles/dovecot/imapsync_runner.py create mode 100644 data/conf/phpfpm/crons/imapsync-oauth-refresh.php create mode 100644 data/web/js/site/imapsync.js create mode 100644 data/web/syncjob-oauth.php create mode 100644 data/web/templates/admin/tab-config-syncjob.twig create mode 100644 data/web/templates/edit/imapsync_source.twig create mode 100644 data/web/templates/modals/imapsync_source.twig diff --git a/data/Dockerfiles/dovecot/Dockerfile b/data/Dockerfiles/dovecot/Dockerfile index f1152c8a1..bd57f1adf 100644 --- a/data/Dockerfiles/dovecot/Dockerfile +++ b/data/Dockerfiles/dovecot/Dockerfile @@ -68,7 +68,6 @@ RUN addgroup -g 5000 vmail \ perl-package-stash-xs \ perl-par-packer \ perl-parse-recdescent \ - perl-lockfile-simple \ libproc2 \ perl-readonly \ perl-regexp-common \ @@ -84,7 +83,6 @@ RUN addgroup -g 5000 vmail \ perl-test-warn \ perl-try-tiny \ perl-unicode-string \ - perl-proc-processtable \ perl-app-cpanminus \ procps \ python3 \ @@ -120,7 +118,7 @@ COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf COPY imapsync /usr/local/bin/imapsync -COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl +COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham diff --git a/data/Dockerfiles/dovecot/docker-entrypoint.sh b/data/Dockerfiles/dovecot/docker-entrypoint.sh index e58abbb6a..d5175e96d 100755 --- a/data/Dockerfiles/dovecot/docker-entrypoint.sh +++ b/data/Dockerfiles/dovecot/docker-entrypoint.sh @@ -279,7 +279,7 @@ chmod g+rw /dev/console chown root:tty /dev/console chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \ /usr/lib/dovecot/sieve/rspamd-pipe-spam \ - /usr/local/bin/imapsync_runner.pl \ + /usr/local/bin/imapsync_runner.py \ /usr/local/bin/imapsync \ /usr/local/bin/trim_logs.sh \ /usr/local/bin/sa-rules.sh \ diff --git a/data/Dockerfiles/dovecot/imapsync_runner.pl b/data/Dockerfiles/dovecot/imapsync_runner.pl deleted file mode 100644 index 1030603ce..000000000 --- a/data/Dockerfiles/dovecot/imapsync_runner.pl +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/perl - -use DBI; -use LockFile::Simple qw(lock trylock unlock); -use Proc::ProcessTable; -use Data::Dumper qw(Dumper); -use IPC::Run 'run'; -use File::Temp; -use Try::Tiny; -use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT); - -sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s }; -my $t = Proc::ProcessTable->new; -my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table}; -if ($imapsync_running ge 1) -{ - print "imapsync is active, exiting..."; - exit; -} - -sub qqw($) { - my @params = (); - my @values = split(/(?=--)/, $_[0]); - foreach my $val (@values) { - my @tmpparam = split(/ /, $val, 2); - foreach my $tmpval (@tmpparam) { - if ($tmpval ne '') { - push @params, $tmpval; - } - } - } - foreach my $val (@params) { - $val=trim($val); - } - return @params; -} - -$run_dir="/tmp"; -$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock'; -$lock_file = $run_dir . "/imapsync_busy"; -$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1); -$lockmgr->lock($lock_file) || die "can't lock ${lock_file}"; -$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, { - mysql_auto_reconnect => 1, - mysql_enable_utf8mb4 => 1 -}); -$dbh->do("UPDATE imapsync SET is_running = 0"); - -sub sig_handler { - # Send die to force exception in "run" - die "sig_handler received signal, preparing to exit...\n"; -}; - -open my $file, '<', "/etc/sogo/sieve.creds"; -my $creds = <$file>; -close $file; -my ($master_user, $master_pass) = split /:/, $creds; -my $sth = $dbh->prepare("SELECT id, - user1, - user2, - host1, - authmech1, - password1, - exclude, - port1, - enc1, - delete2duplicates, - maxage, - subfolder2, - delete1, - delete2, - automap, - skipcrossduplicates, - maxbytespersecond, - custom_params, - subscribeall, - timeout1, - timeout2, - dry - FROM imapsync - WHERE active = 1 - AND is_running = 0 - AND ( - UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60 - OR - last_run IS NULL) - ORDER BY last_run"); - -$sth->execute(); -my $row; - -while ($row = $sth->fetchrow_arrayref()) { - - $id = @$row[0]; - $user1 = @$row[1]; - $user2 = @$row[2]; - $host1 = @$row[3]; - $authmech1 = @$row[4]; - $password1 = @$row[5]; - $exclude = @$row[6]; - $port1 = @$row[7]; - $enc1 = @$row[8]; - $delete2duplicates = @$row[9]; - $maxage = @$row[10]; - $subfolder2 = @$row[11]; - $delete1 = @$row[12]; - $delete2 = @$row[13]; - $automap = @$row[14]; - $skipcrossduplicates = @$row[15]; - $maxbytespersecond = @$row[16]; - $custom_params = @$row[17]; - $subscribeall = @$row[18]; - $timeout1 = @$row[19]; - $timeout2 = @$row[20]; - $dry = @$row[21]; - - if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; } - - my $template = $run_dir . '/imapsync.XXXXXXX'; - my $passfile1 = File::Temp->new(TEMPLATE => $template); - my $passfile2 = File::Temp->new(TEMPLATE => $template); - - binmode( $passfile1, ":utf8" ); - - print $passfile1 "$password1\n"; - print $passfile2 trim($master_pass) . "\n"; - - my @custom_params_a = qqw($custom_params); - my $custom_params_ref = \@custom_params_a; - - my $generated_cmds = [ "/usr/local/bin/imapsync", - "--tmpdir", "/tmp", - "--nofoldersizes", - "--addheader", - ($timeout1 le "0" ? () : ('--timeout1', $timeout1)), - ($timeout2 le "0" ? () : ('--timeout2', $timeout2)), - ($exclude eq "" ? () : ("--exclude", $exclude)), - ($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)), - ($maxage eq "0" ? () : ('--maxage', $maxage)), - ($maxbytespersecond eq "0" ? () : ('--maxbytespersecond', $maxbytespersecond)), - ($delete2duplicates ne "1" ? () : ('--delete2duplicates')), - ($subscribeall ne "1" ? () : ('--subscribeall')), - ($delete1 ne "1" ? () : ('--delete')), - ($delete2 ne "1" ? () : ('--delete2')), - ($automap ne "1" ? () : ('--automap')), - ($skipcrossduplicates ne "1" ? () : ('--skipcrossduplicates')), - (!defined($enc1) ? () : ($enc1)), - "--host1", $host1, - "--user1", $user1, - "--passfile1", $passfile1->filename, - "--port1", $port1, - "--host2", "localhost", - "--user2", $user2 . '*' . trim($master_user), - "--passfile2", $passfile2->filename, - ($dry eq "1" ? ('--dry') : ()), - '--no-modulesversion', - '--noreleasecheck']; - - try { - $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?"); - $is_running->bind_param( 1, ${id} ); - $is_running->execute(); - - run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout; - - # check exit code and status - ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/); - - $success = 0; - if (defined $exit_code && $exit_code == 0) { - $success = 1; - } - - $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?"); - $update->bind_param( 1, ${stdout} ); - $update->bind_param( 2, ${success} ); - $update->bind_param( 3, ${exit_status} ); - $update->bind_param( 4, ${id} ); - $update->execute(); - } catch { - $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?"); - $update->bind_param( 1, ${id} ); - $update->execute(); - } finally { - $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?"); - $update->bind_param( 1, ${id} ); - $update->execute(); - }; - - -} - -$sth->finish(); -$dbh->disconnect(); - -$lockmgr->unlock($lock_file); diff --git a/data/Dockerfiles/dovecot/imapsync_runner.py b/data/Dockerfiles/dovecot/imapsync_runner.py new file mode 100755 index 000000000..b74e1d845 --- /dev/null +++ b/data/Dockerfiles/dovecot/imapsync_runner.py @@ -0,0 +1,274 @@ +#!/usr/bin/python3 + +import fcntl +import json +import os +import re +import signal +import subprocess +import sys +import tempfile +import time +import MySQLdb +import MySQLdb.cursors + +run_dir = '/tmp' +# Max size of imapsync output stored in returned_text (the tail holds the summary) +max_output = 16 * 1024 * 1024 +lock_file = run_dir + '/imapsync_busy.lock' + +# Next due job: higher prio first, never-run jobs first, then oldest run +next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage, + i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params, + i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id, + s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow, s.oauth_access_token, s.oauth_token_expires + FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id + WHERE i.active = 1 + AND s.active = 1 + AND i.is_running = 0 + AND (UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(i.last_run) > i.mins_interval * 60 OR i.last_run IS NULL) + ORDER BY i.prio DESC, i.last_run IS NOT NULL, i.last_run ASC, i.id ASC + LIMIT 1""" +# Atomic claim, a job is only started when this flips is_running 0 -> 1 +claim_sql = """UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL + WHERE id = %s AND is_running = 0 AND active = 1""" +# Token of authorization_code sources: source + local mailbox (user2) + remote user (user1) +token_sql = """SELECT access_token, token_expires FROM imapsync_source_oauth_token + WHERE source_id = %s AND owner = %s AND username = %s""" + +conn = None +# Popen => job of running workers +running = {} +stopping = False + +def num(value): + try: + return int(value) + except (TypeError, ValueError): + return 0 + +def unlink(files): + for path in files: + try: + os.unlink(path) + except OSError: + pass + +def imapsync_running(): + for pid in filter(str.isdigit, os.listdir('/proc')): + try: + with open('/proc/' + pid + '/cmdline', 'rb') as f: + cmdline = f.read().replace(b'\0', b' ') + except OSError: + continue + if re.search(rb'imapsync\s', cmdline, re.I): + return True + return False + +def connect(): + return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'], + passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True) + +# Reconnect when the server dropped the connection (syncs can outlast wait_timeout) +def query(sql, args=None): + global conn + try: + conn.ping() + except MySQLdb.Error: + conn = connect() + cur = conn.cursor(MySQLdb.cursors.DictCursor) + cur.execute(sql, args) + return cur + +# Store a result and release the job +def finish_job(job_id, text, success, exit_status): + try: + query("UPDATE imapsync SET returned_text = %s, success = %s, exit_status = %s, last_run = NOW(), is_running = 0 WHERE id = %s", + (text, success, exit_status, job_id)) + except MySQLdb.Error: + query("UPDATE imapsync SET success = 0, last_run = NOW(), is_running = 0 WHERE id = %s", (job_id,)) + +# Max concurrent imapsync processes and global per-process bandwidth cap in bytes/s (0 = off), +# re-read before every claim round so changes apply to long runs +def read_settings(): + settings = {r['name']: r['value'] for r in query("SELECT name, value FROM imapsync_settings")} + return max(num(settings.get('max_parallel')), 1), max(num(settings.get('max_bytes_per_second')), 0) + +# Build temp files and argv for a claimed job. Returns None when the job was finished without running imapsync. +def prepare_job(j, global_bw, files): + def tmpfile(content): + fd, path = tempfile.mkstemp(prefix='imapsync.', dir=run_dir) + files.append(path) + with os.fdopen(fd, 'w', encoding='utf-8') as f: + f.write(content) + return path + + auth_type = j['auth_type'] or 'PLAIN' + if auth_type == 'XOAUTH2': + # authorization_code: token of this local mailbox + remote user, client_credentials: shared source token + token, expires = j['oauth_access_token'], j['oauth_token_expires'] + if j['oauth_flow'] == 'authorization_code': + row = query(token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {} + token, expires = row.get('access_token'), row.get('token_expires') + # Skip sync if the cached token is missing or about to expire (<2 min) + if not token or expires is None or expires - time.time() < 120: + finish_job(j['id'], "OAuth access token for the configured source is missing or expires too soon; refresh cron will retry.", + 0, 'OAUTH_TOKEN_NOT_READY') + return None + auth_args = ['--authmech1', 'XOAUTH2', '--oauthaccesstoken1', tmpfile(token + '\n')] + else: + auth_args = ['--passfile1', tmpfile(j['password1'] + '\n')] + # imapsync auto-picks PLAIN, only pass --authmech1 for LOGIN/CRAM-MD5 + if auth_type != 'PLAIN': + auth_args += ['--authmech1', auth_type] + + # Custom params: JSON array of {o,v}, validated by the backend on write. One argv token per pair, + # "--opt" or "--opt=value" (value glued to its option, no shell) + custom_params = [] + try: + pairs = json.loads(j['custom_params'] or '[]') + except ValueError: + pairs = [] + for p in pairs if isinstance(pairs, list) else []: + if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']): + continue + value = '' if p.get('v') is None else str(p['v']) + custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value) + + # Effective per-process bandwidth cap (bytes/s): per-job value, capped by the global limit + eff_bw = num(j['maxbytespersecond']) + if global_bw > 0 and (eff_bw <= 0 or eff_bw > global_bw): + eff_bw = global_bw + + cmd = ['/usr/local/bin/imapsync', '--tmpdir', '/tmp', '--nofoldersizes', '--addheader'] + if num(j['timeout1']) > 0: + cmd += ['--timeout1', str(j['timeout1'])] + if num(j['timeout2']) > 0: + cmd += ['--timeout2', str(j['timeout2'])] + if j['exclude']: + cmd += ['--exclude', j['exclude']] + if j['subfolder2']: + cmd += ['--subfolder2', j['subfolder2']] + if num(j['maxage']) != 0: + cmd += ['--maxage', str(j['maxage'])] + if eff_bw > 0: + cmd += ['--maxbytespersecond', str(eff_bw)] + for col, opt in (('delete2duplicates', '--delete2duplicates'), ('subscribeall', '--subscribeall'), + ('delete1', '--delete'), ('delete2', '--delete2'), ('automap', '--automap'), + ('skipcrossduplicates', '--skipcrossduplicates')): + if j[col] == 1: + cmd.append(opt) + enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1']) + if enc1: + cmd.append(enc1) + cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])] + cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')] + if j['dry'] == 1: + cmd.append('--dry') + cmd += ['--no-modulesversion', '--noreleasecheck'] + custom_params + + return {'id': j['id'], 'files': files, 'outfile': tmpfile(''), 'cmd': cmd} + +# Worker: imapsync with output to a file, no inherited fds (DB socket, lock) +def start_job(job): + with open(job['outfile'], 'wb') as out: + return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT) + +# Store the result of a finished worker and remove its temp files +def reap_job(job): + stdout = b'' + try: + with open(job['outfile'], 'rb') as f: + size = os.fstat(f.fileno()).st_size + if size > max_output: + f.seek(size - max_output) + stdout = b'[output truncated, last %d bytes]\n' % max_output + stdout += f.read() + except OSError: + pass + unlink(job['files']) + text = stdout.decode('utf-8', 'replace') + # Last match: the echoed command line at the top may contain user controlled values + m = re.findall(r'Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)', text) + exit_code, exit_status = m[-1] if m else (None, None) + if exit_code is None: + text = 'Could not start or finish imapsync\n' + text + finish_job(job['id'], text, 1 if exit_code is not None and int(exit_code) == 0 else 0, exit_status) + +# Stop claiming new jobs, pass the signal on to the running imapsync processes +def stop(signum, frame): + global stopping + stopping = True + for proc in list(running): + proc.send_signal(signal.SIGTERM) + +# Released by the kernel when the process ends, a crashed run leaves no stale lock +lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600) +try: + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) +except OSError: + sys.exit("can't lock " + lock_file) + +if imapsync_running(): + print('imapsync is active, exiting...') + sys.exit(0) + +for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT): + signal.signal(sig, stop) + +conn = connect() +# Stale flags from crashed runs, safe only here: under the lock and with no imapsync running +query("UPDATE imapsync SET is_running = 0") + +with open('/etc/sogo/sieve.creds') as f: + master_user, master_pass = [s.strip() for s in f.readline().split(':', 1)] + +# Worker pool: keep up to max_parallel workers busy, claim the next due job whenever a slot is free +# (after every worker exit and at least every 10 seconds while workers run) +try: + while True: + for proc, job in list(running.items()): + if proc.poll() is not None: + del running[proc] + reap_job(job) + + max_parallel, global_bw = read_settings() + attempts = 0 + while not stopping and len(running) < max_parallel and attempts < 100: + attempts += 1 + j = query(next_sql).fetchone() + if not j: + break + if query(claim_sql, (j['id'],)).rowcount != 1: + continue + + files = [] + try: + job = prepare_job(j, global_bw, files) + except Exception as e: + print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr) + finish_job(j['id'], 'Could not start or finish imapsync', 0, None) + job = None + if not job: + unlink(files) + continue + try: + running[start_job(job)] = job + except OSError as e: + # Release the claim, the job is picked up again later + print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr) + unlink(files) + query("UPDATE imapsync SET is_running = 0 WHERE id = %s", (j['id'],)) + break + + if not running: + break + # A worker exit ends the wait early + for _ in range(50): + if any(proc.poll() is not None for proc in running): + break + time.sleep(0.2) +finally: + # Never leave password/token files behind + for job in running.values(): + unlink(job['files']) diff --git a/data/conf/phpfpm/crons/imapsync-oauth-refresh.php b/data/conf/phpfpm/crons/imapsync-oauth-refresh.php new file mode 100644 index 000000000..a90088963 --- /dev/null +++ b/data/conf/phpfpm/crons/imapsync-oauth-refresh.php @@ -0,0 +1,73 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false, +]; +try { + $pdo = new PDO($dsn, $database_user, $database_pass, $opt); +} catch (PDOException $e) { + fwrite(STDERR, "DB connect failed: " . $e->getMessage() . "\n"); + exit(1); +} + +$redis = new Redis(); +try { + if (!empty(getenv('REDIS_SLAVEOF_IP'))) { + $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT')); + } else { + $redis->connect('redis-mailcow', 6379); + } + $redis->auth(getenv("REDISPASS")); +} catch (Exception $e) { + fwrite(STDERR, "Redis connect failed: " . $e->getMessage() . "\n"); + exit(1); +} + +function logMsg($priority, $message, $task = "imapsync OAuth refresh") { + global $redis; + $redis->lPush('CRON_LOG', json_encode(array( + "time" => time(), + "priority" => $priority, + "task" => $task, + "message" => $message, + ))); +} + +require_once __DIR__ . '/../web/inc/functions.inc.php'; +require_once __DIR__ . '/../web/inc/functions.auth.inc.php'; +require_once __DIR__ . '/../web/inc/sessions.inc.php'; +require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php'; + +// File-lock to prevent concurrent runs +$lock_file = '/tmp/imapsync-oauth-refresh.lock'; +if (file_exists($lock_file)) { + $pid = (int)trim(@file_get_contents($lock_file)); + if ($pid > 0 && posix_kill($pid, 0)) { + logMsg("info", "Previous refresh still running (pid=$pid), exiting"); + exit(0); + } + @unlink($lock_file); +} +file_put_contents($lock_file, getmypid()); + +try { + // client_credentials source tokens, unused per-user tokens (1 day grace), authorization_code per-user tokens; + // failing tokens are retried 15 minutes after their last attempt (C2), errors are sanitized + $imapsync = \Mailcow\Imapsync\ImapsyncModule::create($pdo, \Mailcow\Core\Context::system(), [ + 'hostname' => (string)getenv('MAILCOW_HOSTNAME'), + ]); + foreach ($imapsync->oauth->refreshDue() as $e) { + logMsg($e['priority'], $e['message']); + } +} finally { + @unlink($lock_file); +} diff --git a/data/web/admin/mailbox.php b/data/web/admin/mailbox.php index 7b07d6fa9..a79c68c1a 100644 --- a/data/web/admin/mailbox.php +++ b/data/web/admin/mailbox.php @@ -9,6 +9,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI']; +$js_minifier->add('/web/js/site/imapsync.js'); $js_minifier->add('/web/js/site/mailbox.js'); $js_minifier->add('/web/js/presets/sieveMailbox.js'); $js_minifier->add('/web/js/site/pwgen.js'); @@ -42,6 +43,7 @@ $template_data = [ 'lang_rl' => json_encode($lang['ratelimit']), 'lang_edit' => json_encode($lang['edit']), 'lang_datatables' => json_encode($lang['datatables']), + 'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(), ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php'; diff --git a/data/web/admin/system.php b/data/web/admin/system.php index b5e00808f..c74f8c6a6 100644 --- a/data/web/admin/system.php +++ b/data/web/admin/system.php @@ -118,6 +118,7 @@ $template_data = [ 'ui_themes' => customize('get', 'ui_themes'), 'ui_theme' => customize('get', 'ui_theme'), 'password_complexity' => password_complexity('get'), + 'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(), 'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'], 'cors_settings' => $cors_settings, 'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on', diff --git a/data/web/api/openapi.yaml b/data/web/api/openapi.yaml index c86561903..0044ac1e8 100644 --- a/data/web/api/openapi.yaml +++ b/data/web/api/openapi.yaml @@ -1465,31 +1465,15 @@ paths: response: value: - log: - - mailbox - - add - syncjob - - active: "1" - automap: "1" - custom_params: "" - delete1: "0" - delete2: "0" - delete2duplicates: "1" - enc1: SSL - exclude: (?i)spam|(?i)junk - host1: imap.server.tld - maxage: "0" - maxbytespersecond: "0" - mins_interval: "20" - password1: supersecret - port1: 993 - skipcrossduplicates: "0" - subfolder2: External - subscribeall: "1" - timeout1: "600" - timeout2: "600" + - add + - job + - username: mailbox@domain.tld + source_id: "3" user1: username - username: mailbox@domain.tld - - null + password1: "*" + mins_interval: "20" + active: "1" msg: - mailbox_modified - mailbox@domain.tld @@ -1497,7 +1481,7 @@ paths: schema: properties: log: - description: contains request object + description: contains request object (secrets are masked as "*") items: {} type: array msg: @@ -1515,8 +1499,10 @@ paths: tags: - Sync jobs description: >- - You can create new sync job using this action. It takes a JSON object - containing a domain informations. + Create a sync job for a local mailbox. Connection data (host, port, + encryption, authentication) comes from the sync source referenced by + `source_id`. Requires the `syncjobs` ACL (user ACL for mailbox users, + domain admin ACL for domain admins). operationId: Create sync job summary: Create sync job requestBody: @@ -1525,11 +1511,9 @@ paths: schema: example: username: lisa@mailcow.tld - host1: mail.mailcow.tld - port1: "143" - user1: demo@mailcow.tld + source_id: "3" + user1: demo@remote.tld password1: supersecretpw - enc1: TLS mins_interval: "20" subfolder2: "/SyncIntoSubfolder" maxage: "0" @@ -1537,44 +1521,47 @@ paths: timeout1: "600" timeout2: "600" exclude: "(?i)spam|(?i)junk" - custom_params: "--dry" + custom_params: + - {o: folder, v: INBOX} + - {o: syncinternaldates, v: ""} delete2duplicates: "1" - delete1: "1" + delete1: "0" delete2: "0" automap: "1" skipcrossduplicates: "0" - subscribeall: "0" + subscribeall: "1" + dry: "0" active: "1" properties: - parameters: - description: your local mailcow mailbox - type: string - host1: - description: the smtp server where mails should be synced from - type: string - port1: - description: the smtp port of the target mail server + username: + description: local mailcow mailbox (destination, user2); users always create jobs for their own mailbox type: string + source_id: + description: >- + id of a sync source visible to the caller (`GET /api/v1/get/syncjob_source/all`). + type: integer user1: - description: the username of the mailbox + description: >- + remote username at the source. For XOAUTH2 authorization_code sources + a token for (source_id, local mailbox, user1) must exist, i.e. the + mailbox owner must have signed in via `/syncjob-oauth` first + (`imapsync_source_oauth_connect_required` otherwise). type: string - password: - description: the password of the mailbox + password1: + description: >- + remote password, required unless the source uses XOAUTH2 (ignored then) type: string - enc1: - description: the encryption method used to connect to the mailserver - type: string - mins_internal: - description: the interval in which messages should be syned + mins_interval: + description: sync interval in minutes (1-43800) type: number subfolder2: description: sync into subfolder on destination (empty = do not use subfolder) type: string maxage: - description: only sync messages up to this age in days + description: only sync messages up to this age in days (0 = ignore age) type: number maxbytespersecond: - description: max speed transfer limit for the sync + description: per-job bandwidth limit in bytes/s (0 = unlimited, capped by the global limit) type: number timeout1: description: timeout for connection to remote host @@ -1586,8 +1573,22 @@ paths: description: exclude objects (regex) type: string custom_params: - description: custom parameters - type: string + description: >- + Allowlisted imapsync options as a list of option/value pairs (must be + an array). + `o` is the option name without leading dashes (see `GET + /api/v1/get/syncjob_options`), `v` the value. The allowlist decides + per option: `true` = a non-empty value is required, `false` = flag + without value (`v` must be empty), string = regular expression the + whole value must match. Values are passed to imapsync as a single argument. + type: array + items: + type: object + properties: + o: + type: string + v: + type: string delete2duplicates: description: delete duplicates on destination (--delete2duplicates) type: boolean @@ -1606,9 +1607,215 @@ paths: subscribeall: description: subscribe all folders (--subscribeall) type: boolean + dry: + description: dry run, nothing is changed (--dry) + type: boolean active: description: enables or disables the sync job type: boolean + host1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + port1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + enc1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + authmech1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + type: object + /api/v1/add/syncjob_source: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - add + - source + - name: Example import + host: imap.example.org + port: "993" + enc: SSL + auth_type: PLAIN + scope: all + msg: + - imapsync_source_added + - Example import + type: success + schema: + properties: + log: + description: contains request object (secrets are masked as "*") + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Create a reusable IMAP sync source. A sync job binds to a source by id; + the source carries host/port/encryption and either static credentials + (PLAIN/LOGIN/CRAM-MD5 — the password is supplied per job) or an OAuth2 + configuration (XOAUTH2). Requires the `syncjobs` ACL. Validation errors + are returned as translated `danger` messages, nothing is written then. + operationId: Create sync job source + summary: Create sync job source + requestBody: + content: + application/json: + schema: + example: + name: Example import + description: "" + host: imap.example.org + port: "993" + enc: SSL + auth_type: PLAIN + active: "1" + scope: domain + domains: + - example.org + properties: + name: + description: >- + source name, max. 100 characters, unique per creator + type: string + description: + description: >- + free text shown in the UI, max. 255 characters + type: string + host: + description: >- + remote IMAP host (hostname or IP, max. 253 characters). + `host1` is accepted as an alias. + type: string + port: + description: >- + remote IMAP port (1-65535). `port1` is accepted as an alias. + type: integer + enc: + description: >- + TLS (STARTTLS) / SSL / PLAIN. `enc1` is accepted as an alias. + type: string + enum: + - TLS + - SSL + - PLAIN + auth_type: + description: >- + authentication mechanism; PLAIN/LOGIN/CRAM-MD5 use the per-job + password, XOAUTH2 uses OAuth2 tokens. XOAUTH2 sources can only be created + and edited by admins and domain admins (`access_denied` for users); users + can use XOAUTH2 sources visible to them in their sync jobs. + type: string + enum: + - PLAIN + - LOGIN + - CRAM-MD5 + - XOAUTH2 + oauth_flow: + description: >- + XOAUTH2 grant type (default client_credentials). client_credentials + sources are always private to their creator (scope is forced to `user` + without targets). + type: string + enum: + - client_credentials + - authorization_code + oauth_authorize_endpoint: + description: >- + authorize URL, required for authorization_code. Must be an https:// URL + without credentials and with a valid hostname + type: string + oauth_token_endpoint: + description: >- + token URL, required for XOAUTH2. Must be an https:// URL without credentials and with a valid hostname + type: string + oauth_userinfo_endpoint: + description: >- + optional identity endpoint for authorization_code (used when the token + response has no usable id_token). Must be an https:// URL without credentials and with a valid hostname + type: string + oauth_client_id: + description: >- + OAuth2 client id, required for XOAUTH2 + type: string + oauth_client_secret: + description: >- + OAuth2 client secret, required for XOAUTH2. Never returned by GET. + type: string + oauth_scope: + description: >- + OAuth2 scope, required for XOAUTH2 + type: string + oauth_extra_params: + description: >- + optional extra token request parameters: a JSON object with scalar values, + sent either as object or as JSON string (empty = none). They never + override the grant fields. + oneOf: + - type: object + additionalProperties: true + - type: string + scope: + description: >- + visibility: `all` (admins only), `domain` (needs `domains`) or `user` + (listed `users`; no targets = private to the creator). Users always + create private sources. + type: string + enum: + - all + - domain + - user + domains: + description: target domains when scope is domain + type: array + items: + type: string + users: + description: target mailboxes when scope is user + type: array + items: + type: string + active: + description: >- + enables or disables the source + type: boolean type: object /api/v1/add/tls-policy-map: post: @@ -2782,6 +2989,62 @@ paths: type: object type: object summary: Delete sync job + /api/v1/delete/syncjob_source: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - delete + - source + - id: + - "3" + msg: + - imapsync_source_deleted + - Example import + type: success + schema: + properties: + log: + description: contains request object + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Delete one or more sync job sources by id. Sources still referenced by + a sync job cannot be deleted. Delete or + repoint the dependent jobs first. + operationId: Delete sync job source + summary: Delete sync job source + requestBody: + content: + application/json: + schema: + example: + - "3" + items: + type: string + type: array /api/v1/delete/tls-policy-map: post: responses: @@ -3813,18 +4076,21 @@ paths: examples: response: value: - log: - - entity - - action - - object - msg: - - message - - entity name - type: success + - log: + - syncjob + - edit + - job + - id: + - "1" + active: "0" + msg: + - mailbox_modified + - mailbox@domain.tld + type: success schema: properties: log: - description: contains request object + description: contains request object (secrets are masked as "*") items: {} type: array msg: @@ -3842,8 +4108,8 @@ paths: tags: - Sync jobs description: >- - You can update one or more sync jobs per request. You can also send just - attributes you want to change. + Update one or more sync jobs per request. Only send the attributes you + want to change. Requires the `syncjobs` ACL. operationId: Update sync job requestBody: content: @@ -3852,107 +4118,526 @@ paths: example: attr: active: "1" - automap: "1" - custom_params: "" - delete1: "0" - delete2: "0" - delete2duplicates: "1" - enc1: SSL - exclude: (?i)spam|(?i)junk - host1: imap.server.tld - maxage: "0" - maxbytespersecond: "0" - mins_interval: "20" - password1: supersecret - port1: "993" - skipcrossduplicates: "0" - subfolder2: External - subscribeall: "1" - timeout1: "600" - timeout2: "600" + source_id: "3" user1: username - items: "1" + mins_interval: "20" + custom_params: + - {o: folder, v: INBOX} + items: + - "1" properties: attr: properties: - active: - description: Is sync job active - type: boolean - automap: + source_id: description: >- - Try to automap folders ("Sent items", "Sent" => "Sent" - etc.) - type: boolean - custom_params: - description: Custom parameters passed to imapsync command + id of a sync source visible to the caller (`GET /api/v1/get/syncjob_source/all`). + Visibility is only checked when the source changes. + type: integer + user1: + description: >- + remote username at the source. For XOAUTH2 authorization_code sources + a token for (source_id, local mailbox, user1) must exist, i.e. the + mailbox owner must have signed in via `/syncjob-oauth` first + (`imapsync_source_oauth_connect_required` otherwise). For XOAUTH2 + client_credentials sources user1 may only be changed by callers who can + see the source (`access_denied` otherwise); the other fields stay editable. type: string + password1: + description: >- + remote password; empty keeps the stored one. Required when the job + is moved to a non-XOAUTH2 source and no password is stored. + type: string + mins_interval: + description: sync interval in minutes (1-43800) + type: number + subfolder2: + description: sync into subfolder on destination (empty = do not use subfolder) + type: string + maxage: + description: only sync messages up to this age in days (0 = ignore age) + type: number + maxbytespersecond: + description: per-job bandwidth limit in bytes/s (0 = unlimited, capped by the global limit) + type: number + timeout1: + description: timeout for connection to remote host + type: number + timeout2: + description: timeout for connection to local host + type: number + exclude: + description: exclude objects (regex) + type: string + custom_params: + description: >- + Allowlisted imapsync options as a list of option/value pairs (must be + an array). + `o` is the option name without leading dashes (see `GET + /api/v1/get/syncjob_options`), `v` the value. The allowlist decides + per option: `true` = a non-empty value is required, `false` = flag + without value (`v` must be empty), string = regular expression the + whole value must match. Options taking file paths are never allowed. + Values are passed to imapsync as a single argument. + type: array + items: + type: object + properties: + o: + type: string + v: + type: string + prio: + description: >- + priority (admin only, ignored otherwise). Due jobs run ordered by + prio DESC, never-run first, last_run ASC, id ASC. + type: number + last_run: + description: any value resets last_run (job becomes due) + type: string + success: + description: any value resets the last result + type: string + delete2duplicates: + description: delete duplicates on destination (--delete2duplicates) + type: boolean delete1: - description: Delete from source when completed + description: delete from source when completed (--delete1) type: boolean delete2: - description: Delete messages on destination that are not on source + description: delete messages on destination that are not on source (--delete2) type: boolean - delete2duplicates: - description: Delete duplicates on destination + automap: + description: try to automap folders ("Sent items", "Sent" => "Sent" etc.) (--automap) type: boolean + skipcrossduplicates: + description: skip duplicate messages across folders (first come, first serve) (--skipcrossduplicates) + type: boolean + subscribeall: + description: subscribe all folders (--subscribeall) + type: boolean + dry: + description: dry run, nothing is changed (--dry) + type: boolean + active: + description: enables or disables the sync job + type: boolean + host1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + port1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string enc1: - description: Encryption + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + authmech1: + description: >- + Removed. Host, port, encryption and authentication now come from + the sync source (`source_id`, see `/api/v1/add/syncjob_source`). + The attribute is ignored. + deprecated: true + type: string + order_id: + description: Removed, use `prio` (admin) or `/api/v1/edit/syncjob/run_next`. Ignored. + deprecated: true + type: number + type: object + items: + description: list of sync job ids to update + type: array + items: + type: string + type: object + summary: Update sync job + /api/v1/edit/syncjob/run_next: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - edit + - run_next + - id: + - "1" + msg: + - imapsync_run_next_done + - 1 + type: success + schema: + properties: + log: + description: contains request object (secrets are masked as "*") + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Admin only. Moves the given jobs to the front of the queue: `prio` is set + above the current maximum and `last_run` is cleared, so the runner picks + them next. Jobs that are currently running are rejected with + `imapsync_job_running` (one message per job), the others are moved. + The jobs are locked while they are moved, so a job the runner claims at + the same time is reported as running instead of being counted. + operationId: Run sync job next + summary: Run sync job next + requestBody: + content: + application/json: + schema: + example: + items: + - "1" + attr: {} + properties: + items: + description: list of sync job ids + type: array + items: + type: string + attr: + description: unused, send an empty object + type: object + type: object + /api/v1/edit/syncjob_source: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - edit + - source + - id: + - "3" + name: Example import + oauth_client_secret: "*" + msg: + - imapsync_source_modified + - Example import + type: success + schema: + properties: + log: + description: contains request object (secrets are masked as "*") + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Update one or more sync sources (creator, admins, and domain admins for + sources created by mailboxes of their domains). Fields not provided are + left unchanged. All checks run before anything is written; a rejected + edit changes nothing. Requires the `syncjobs` ACL. + operationId: Update sync job source + summary: Update sync job source + requestBody: + content: + application/json: + schema: + example: + items: + - "3" + attr: + name: Example import + host: imap.example.org + port: "993" + enc: SSL + auth_type: PLAIN + active: "1" + properties: + attr: + properties: + name: + description: >- + source name, max. 100 characters, unique per creator + type: string + description: + description: >- + free text shown in the UI, max. 255 characters + type: string + host: + description: >- + remote IMAP host (hostname or IP, max. 253 characters). + `host1` is accepted as an alias. Host, port and enc are only validated + when they change (migrated legacy values stay editable). + type: string + port: + description: >- + remote IMAP port (1-65535). `port1` is accepted as an alias. + type: integer + enc: + description: >- + TLS (STARTTLS) / SSL / PLAIN. `enc1` is accepted as an alias. + type: string enum: - TLS - SSL - PLAIN - type: string - exclude: - description: Exclude objects (regex) - type: string - host1: - description: Hostname - type: string - maxage: + auth_type: description: >- - Maximum age of messages in days that will be polled from - remote (0 = ignore age) - type: number - maxbytespersecond: - description: Max. bytes per second (0 = unlimited) - type: number - mins_interval: - description: Interval (min) - type: number - password1: - description: Password + authentication mechanism; PLAIN/LOGIN/CRAM-MD5 use the per-job + password, XOAUTH2 uses OAuth2 tokens. XOAUTH2 sources can only be created + and edited by admins and domain admins (`access_denied` for users); users + can use XOAUTH2 sources visible to them in their sync jobs. type: string - port1: - description: Port - type: string - skipcrossduplicates: + enum: + - PLAIN + - LOGIN + - CRAM-MD5 + - XOAUTH2 + oauth_flow: description: >- - Skip duplicate messages across folders (first come, - first serve) + XOAUTH2 grant type (default client_credentials). client_credentials + sources are always private to their creator (scope is forced to `user` + without targets). + type: string + enum: + - client_credentials + - authorization_code + oauth_authorize_endpoint: + description: >- + authorize URL, required for authorization_code. Must be an https:// URL + without credentials and with a valid hostname + type: string + oauth_token_endpoint: + description: >- + token URL, required for XOAUTH2. Must be an https:// URL without credentials and with a valid hostname + type: string + oauth_userinfo_endpoint: + description: >- + optional identity endpoint for authorization_code (used when the token + response has no usable id_token). Must be an https:// URL without credentials and with a valid hostname + type: string + oauth_client_id: + description: >- + OAuth2 client id, required for XOAUTH2 + type: string + oauth_client_secret: + description: >- + OAuth2 client secret; empty keeps the stored one. Changing auth_type, + oauth_flow, oauth_token_endpoint, oauth_client_id, host, port or enc drops + the cached source token and all per-user tokens of the source (users have + to sign in again). + type: string + oauth_scope: + description: >- + OAuth2 scope, required for XOAUTH2 + type: string + oauth_extra_params: + description: >- + optional extra token request parameters: a JSON object with scalar values, + sent either as object or as JSON string (empty = none). They never + override the grant fields. + oneOf: + - type: object + additionalProperties: true + - type: string + scope: + description: >- + visibility: `all` (admins only), `domain` (needs `domains`) or `user` + (listed `users`; no targets = private to the creator). Users always + create private sources. Targets are only rewritten when `scope` is sent. + type: string + enum: + - all + - domain + - user + domains: + description: target domains when scope is domain + type: array + items: + type: string + users: + description: target mailboxes when scope is user + type: array + items: + type: string + active: + description: >- + enables or disables the source type: boolean - subfolder2: - description: >- - Sync into subfolder on destination (empty = do not use - subfolder) - type: string - subscribeall: - description: Subscribe all folders - type: boolean - timeout1: - description: Timeout for connection to remote host - type: number - timeout2: - description: Timeout for connection to local host - type: number - user1: - description: Username - type: string type: object items: - description: contains list of aliases you want update + description: list of source ids to update + type: array + items: + type: string + type: object + /api/v1/edit/syncjob_source/refresh_token: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - edit + - refresh_token + - id: + - "3" + msg: + - imapsync_source_token_refreshed + - Example import + type: success + schema: + properties: + log: + description: contains request object (secrets are masked as "*") + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Force an OAuth2 access-token refresh for a XOAUTH2 client_credentials + source (other sources: `imapsync_source_refresh_unsupported`). Requires + edit rights on the source and the `syncjobs` ACL. + operationId: Refresh sync job source token + summary: Refresh sync job source token + requestBody: + content: + application/json: + schema: + example: + items: + - "3" + properties: + items: + description: source id (only the first one is used) + type: array + items: + type: string + type: object + /api/v1/edit/imapsync_settings: + post: + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - log: + - syncjob + - edit + - settings + - max_parallel: "4" + max_kb_per_second: "2000" + msg: max_parallel_saved + type: success + schema: + properties: + log: + description: contains request object (secrets are masked as "*") + items: {} + type: array + msg: + items: {} + type: array + type: + enum: + - success + - danger + - error + type: string + type: object + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Update global sync job settings (admin only). `max_parallel` is the + number of imapsync processes that may run concurrently (1-32); + `max_kb_per_second` is a per-process bandwidth cap in KB/s (0 = + unlimited, max. 10485760 = 10 GiB/s), stored internally in bytes/s. + operationId: Update sync job settings + summary: Update sync job settings + requestBody: + content: + application/json: + schema: + example: + attr: + max_parallel: "4" + max_kb_per_second: "2000" + properties: + attr: + properties: + max_parallel: + description: max. concurrent sync processes (1-32) + type: number + max_kb_per_second: + description: bandwidth limit per process in KB/s (0 = unlimited, max. 10485760) + type: number type: object type: object - summary: Update sync job /api/v1/edit/user-acl: post: responses: @@ -5607,6 +6292,14 @@ paths: summary: Get version status /api/v1/get/syncjobs/all/no_log: get: + parameters: + - description: e.g. api-key-string + example: api-key-string + in: header + name: X-API-Key + required: false + schema: + type: string responses: "401": $ref: "#/components/responses/Unauthorized" @@ -5616,43 +6309,196 @@ paths: examples: response: value: - - active: "1" - authmd51: 0 - authmech1: PLAIN + - id: 1 + user2: mailbox@domain.tld + source_id: 3 + user1: username + active: 1 automap: 1 - created: "2019-05-22 11:37:25" - custom_params: "" + custom_params: '[{"o":"folder","v":"INBOX"}]' + custom_params_legacy: null delete1: 0 delete2: 0 delete2duplicates: 1 - domain2: "" - enc1: TLS + dry: 0 exclude: (?i)spam|(?i)junk - host1: imap.server.tld - id: 1 + exit_status: EX_OK is_running: 0 - last_run: "2019-05-22 11:40:02" + last_run: "2026-10-15 11:40:02" + success: 1 log: "" maxage: 0 maxbytespersecond: "0" - mins_interval: "20" - modified: "2019-05-22 11:40:02" - port1: 993 - regextrans2: "" + mins_interval: 20 + prio: 0 skipcrossduplicates: 0 subfolder2: External subscribeall: 1 timeout1: 600 timeout2: 600 - user1: username - user2: mailbox@domain.tld + created: "2026-10-01 11:37:25" + modified: "2026-10-15 11:40:02" + source_name: Example import + source_description: "" + source_host: imap.example.org + source_port: 993 + source_enc: SSL + source_auth_type: PLAIN + source_oauth_flow: client_credentials + source_created_by: "" + source_scope: all + source_active: 1 + source_oauth_token_expires: null + source_oauth_last_refresh_error: null + user_token_exists: 0 + user_token_expires: null + user_token_error: null description: OK headers: {} tags: - Sync jobs - description: You can list all syn jobs existing in system. + description: >- + Lists all sync jobs the caller may access (`/api/v1/get/syncjobs/all` + includes the log, `/api/v1/get/syncjobs/[/no_log]` filters by + mailbox). Jobs are ordered by prio DESC, never-run first, last_run ASC, + id ASC. + `custom_params_legacy` holds options of migrated jobs that could not be + converted (informational, not used). The removed fields `host1`, + `port1`, `enc1` and `authmech1` are no longer part of a job. operationId: Get sync jobs summary: Get sync jobs + "/api/v1/get/syncjob_source/{id}": + get: + parameters: + - description: source id, or `all` to list every source visible to the session (sorted by name) + example: all + in: path + name: id + required: true + schema: + type: string + - description: e.g. api-key-string + example: api-key-string + in: header + name: X-API-Key + required: false + schema: + type: string + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + - id: 3 + name: Example import + description: "" + created_by: "" + scope: domain + domains: + - example.org + users: [] + host: imap.example.org + port: 993 + enc: SSL + host1: imap.example.org + port1: 993 + enc1: SSL + auth_type: XOAUTH2 + oauth_flow: authorization_code + oauth_token_endpoint: https://login.example.org/token + oauth_authorize_endpoint: https://login.example.org/authorize + oauth_userinfo_endpoint: null + oauth_client_id: 0000-1111 + oauth_scope: https://outlook.office.com/IMAP.AccessAsUser.All offline_access + oauth_extra_params: null + oauth_redirect_uri: https://mail.example.org/syncjob-oauth + oauth_token_expires: null + oauth_last_refresh_error: null + oauth_last_refresh_attempt: null + has_client_secret: true + has_access_token: false + can_edit: true + active: 1 + created: "2026-10-01 09:00:00" + modified: "2026-10-01 09:00:00" + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Returns a single sync source by id, or all sources visible to the caller + when `id` is `all`. `can_edit` tells whether the caller may edit/delete + the source. + operationId: Get sync job sources + summary: Get sync job sources + /api/v1/get/syncjob_options: + get: + parameters: + - description: e.g. api-key-string + example: api-key-string + in: header + name: X-API-Key + required: false + schema: + type: string + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + dry: false + folder: true + maxsize: true + sslargs1: ^SSL_verify_mode=[01]$ + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Returns the allowlist of imapsync options accepted in a sync job's + `custom_params` as option => rule: `true` = requires a non-empty value, + `false` = flag without value, string = regular expression the whole + value must match. + operationId: Get sync job options + summary: Get sync job options + /api/v1/get/imapsync_settings: + get: + parameters: + - description: e.g. api-key-string + example: api-key-string + in: header + name: X-API-Key + required: false + schema: + type: string + responses: + "401": + $ref: "#/components/responses/Unauthorized" + "200": + content: + application/json: + examples: + response: + value: + max_parallel: 4 + max_bytes_per_second: 2048000 + max_kb_per_second: 2000 + description: OK + headers: {} + tags: + - Sync jobs + description: >- + Admin only. Returns the global sync job settings (`{}` for other roles). + operationId: Get sync job settings + summary: Get sync job settings "/api/v1/get/tls-policy-map/{id}": get: parameters: diff --git a/data/web/domainadmin/mailbox.php b/data/web/domainadmin/mailbox.php index 8beb5abb9..cc1f5bc1b 100644 --- a/data/web/domainadmin/mailbox.php +++ b/data/web/domainadmin/mailbox.php @@ -9,6 +9,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI']; +$js_minifier->add('/web/js/site/imapsync.js'); $js_minifier->add('/web/js/site/mailbox.js'); $js_minifier->add('/web/js/presets/sieveMailbox.js'); $js_minifier->add('/web/js/site/pwgen.js'); @@ -42,6 +43,7 @@ $template_data = [ 'lang_rl' => json_encode($lang['ratelimit']), 'lang_edit' => json_encode($lang['edit']), 'lang_datatables' => json_encode($lang['datatables']), + 'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(), ]; require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php'; \ No newline at end of file diff --git a/data/web/edit.php b/data/web/edit.php index 48f2309c1..b003d5bf6 100644 --- a/data/web/edit.php +++ b/data/web/edit.php @@ -204,9 +204,15 @@ if (isset($_SESSION['mailcow_cc_role'])) { if (isset($_GET['syncjob']) && is_numeric($_GET['syncjob'])) { $id = $_GET["syncjob"]; - $result = mailbox('get', 'syncjob_details', $id); + $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob($id); $template = 'edit/syncjob.twig'; } + elseif (isset($_GET['syncjob_source']) && + is_numeric($_GET['syncjob_source'])) { + $id = $_GET["syncjob_source"]; + $result = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSource($id); + $template = 'edit/imapsync_source.twig'; + } elseif (isset($_GET['filter']) && is_numeric($_GET['filter'])) { $id = $_GET["filter"]; @@ -225,6 +231,7 @@ else { $template_data['access_denied'] = true; } +$js_minifier->add('/web/js/site/imapsync.js'); $js_minifier->add('/web/js/site/edit.js'); $js_minifier->add('/web/js/site/pwgen.js'); @@ -233,5 +240,6 @@ $template_data['return_to'] = $_SESSION['return_to']; $template_data['lang_user'] = json_encode($lang['user']); $template_data['lang_admin'] = json_encode($lang['admin']); $template_data['lang_datatables'] = json_encode($lang['datatables']); +$template_data['imapsync_redirect_uri'] = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(); require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php'; diff --git a/data/web/inc/ajax/syncjob_logs.php b/data/web/inc/ajax/syncjob_logs.php index 9c521aec7..5caf2e9f8 100644 --- a/data/web/inc/ajax/syncjob_logs.php +++ b/data/web/inc/ajax/syncjob_logs.php @@ -6,7 +6,7 @@ if (!isset($_SESSION['mailcow_cc_role'])) { } if (isset($_GET['id']) && is_numeric($_GET['id'])) { - if ($details = mailbox('get', 'syncjob_details', intval($_GET['id']))) { + if ($details = \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJob(intval($_GET['id']))) { echo (empty($details['log'])) ? '-' : $details['log']; } } diff --git a/data/web/inc/footer.inc.php b/data/web/inc/footer.inc.php index 4ced152ae..01dc2c532 100644 --- a/data/web/inc/footer.inc.php +++ b/data/web/inc/footer.inc.php @@ -33,11 +33,11 @@ if (array_key_exists('pending_tfa_methods', $_SESSION)) { if (isset($pending_tfa_authmechs['webauthn'])) { $pending_tfa_authmechs['webauthn'] = true; } - if (!isset($pending_tfa_authmechs['webauthn']) + if (!isset($pending_tfa_authmechs['webauthn']) && isset($pending_tfa_authmechs['yubi_otp'])) { $pending_tfa_authmechs['yubi_otp'] = true; } - if (!isset($pending_tfa_authmechs['webauthn']) + if (!isset($pending_tfa_authmechs['webauthn']) && !isset($pending_tfa_authmechs['yubi_otp']) && isset($pending_tfa_authmechs['totp'])) { $pending_tfa_authmechs['totp'] = true; @@ -72,6 +72,7 @@ $globalVariables = [ 'lang_fido2' => json_encode($lang['fido2']), 'lang_success' => json_encode($lang['success']), 'lang_danger' => json_encode($lang['danger']), + 'lang_syncjobs' => json_encode($lang['syncjobs']), 'docker_timeout' => $DOCKER_TIMEOUT, 'session_lifetime' => (int)$SESSION_LIFETIME, 'csrf_token' => $_SESSION['CSRF']['TOKEN'], diff --git a/data/web/inc/functions.admin.inc.php b/data/web/inc/functions.admin.inc.php index 7bd0af42f..69f1b7e56 100644 --- a/data/web/inc/functions.admin.inc.php +++ b/data/web/inc/functions.admin.inc.php @@ -103,7 +103,7 @@ function admin($_action, $_data = null) { continue; } } - if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new))) { + if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new)) || $username_new == 'API') { $_SESSION['return'][] = array( 'type' => 'danger', 'log' => array(__FUNCTION__, $_action, $_data_log), @@ -179,6 +179,9 @@ function admin($_action, $_data = null) { $stmt = $pdo->prepare("UPDATE `tfa` SET `username` = :username_new WHERE `username` = :username"); $stmt->execute(array(':username_new' => $username_new, ':username' => $username)); } + if ($username_new != $username) { + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username, $username_new); + } } $_SESSION['return'][] = array( 'type' => 'success', @@ -211,6 +214,7 @@ function admin($_action, $_data = null) { $stmt->execute(array( ':username' => $username, )); + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username); $stmt = $pdo->prepare("DELETE FROM `domain_admins` WHERE `username` = :username"); $stmt->execute(array( ':username' => $username, diff --git a/data/web/inc/functions.domain_admin.inc.php b/data/web/inc/functions.domain_admin.inc.php index 17aa4ad6a..e2261ea70 100644 --- a/data/web/inc/functions.domain_admin.inc.php +++ b/data/web/inc/functions.domain_admin.inc.php @@ -156,7 +156,7 @@ function domain_admin($_action, $_data = null) { } } } - if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new))) { + if (!ctype_alnum(str_replace(array('_', '.', '-'), '', $username_new)) || $username_new == 'API') { $_SESSION['return'][] = array( 'type' => 'danger', 'log' => array(__FUNCTION__, $_action, $_data_log), @@ -183,6 +183,9 @@ function domain_admin($_action, $_data = null) { ':username_new' => $username_new, ':username' => $username )); + if ($username_new != $username) { + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username, $username_new); + } if (!empty($domains)) { foreach ($domains as $domain) { $stmt = $pdo->prepare("INSERT INTO `domain_admins` (`username`, `domain`, `created`, `active`) @@ -331,6 +334,8 @@ function domain_admin($_action, $_data = null) { $stmt->execute(array( ':username' => $username, )); + // sync sources created by this account must not pass to a re-created account of the same name + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username); $_SESSION['return'][] = array( 'type' => 'success', 'log' => array(__FUNCTION__, $_action, $_data_log), diff --git a/data/web/inc/functions.mailbox.inc.php b/data/web/inc/functions.mailbox.inc.php index 37ccbd85c..645e96a4d 100644 --- a/data/web/inc/functions.mailbox.inc.php +++ b/data/web/inc/functions.mailbox.inc.php @@ -302,192 +302,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { 'msg' => array('mailbox_modified', $username) ); break; - case 'syncjob': - if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - if (isset($_data['username']) && filter_var($_data['username'], FILTER_VALIDATE_EMAIL)) { - if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data['username'])) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - else { - $username = $_data['username']; - } - } - elseif ($_SESSION['mailcow_cc_role'] == "user") { - $username = $_SESSION['mailcow_cc_username']; - } - else { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'no_user_defined' - ); - return false; - } - $active = intval($_data['active']); - $subscribeall = intval($_data['subscribeall']); - $delete2duplicates = intval($_data['delete2duplicates']); - $delete1 = intval($_data['delete1']); - $delete2 = intval($_data['delete2']); - $timeout1 = intval($_data['timeout1']); - $timeout2 = intval($_data['timeout2']); - $skipcrossduplicates = intval($_data['skipcrossduplicates']); - $automap = intval($_data['automap']); - $dry = intval($_data['dry']); - $port1 = $_data['port1']; - $host1 = strtolower($_data['host1']); - $password1 = $_data['password1']; - $exclude = $_data['exclude']; - $maxage = $_data['maxage']; - $maxbytespersecond = $_data['maxbytespersecond']; - $subfolder2 = $_data['subfolder2']; - $user1 = $_data['user1']; - $mins_interval = $_data['mins_interval']; - $enc1 = $_data['enc1']; - $custom_params = (empty(trim($_data['custom_params']))) ? '' : trim($_data['custom_params']); - - // validate custom params - foreach (explode('-', $custom_params) as $param){ - if(empty($param)) continue; - - // extract option - if (str_contains($param, '=')) $param = explode('=', $param)[0]; - else $param = rtrim($param, ' '); - // remove first char if first char is - - if ($param[0] == '-') $param = ltrim($param, $param[0]); - - if (str_contains($param, ' ')) { - // bad char - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'bad character SPACE' - ); - return false; - } - - // check if param is whitelisted - if (!in_array(strtolower($param), $GLOBALS["IMAPSYNC_OPTIONS"]["whitelist"])){ - // bad option - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'bad option '. $param - ); - return false; - } - } - if (empty($subfolder2)) { - $subfolder2 = ""; - } - if (!isset($maxage) || !filter_var($maxage, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $maxage = "0"; - } - if (!isset($timeout1) || !filter_var($timeout1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $timeout1 = "600"; - } - if (!isset($timeout2) || !filter_var($timeout2, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $timeout2 = "600"; - } - if (!isset($maxbytespersecond) || !filter_var($maxbytespersecond, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 125000000)))) { - $maxbytespersecond = "0"; - } - if (!filter_var($port1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 65535)))) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - if (!filter_var($mins_interval, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 43800)))) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - // if (!is_valid_domain_name($host1)) { - // $_SESSION['return'][] = array( - // 'type' => 'danger', - // 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - // 'msg' => 'access_denied' - // ); - // return false; - // } - if ($enc1 != "TLS" && $enc1 != "SSL" && $enc1 != "PLAIN") { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - if (@preg_match("/" . $exclude . "/", null) === false) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - $stmt = $pdo->prepare("SELECT '1' FROM `imapsync` - WHERE `user2` = :user2 AND `user1` = :user1 AND `host1` = :host1"); - $stmt->execute(array(':user1' => $user1, ':user2' => $username, ':host1' => $host1)); - $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); - if ($num_results != 0) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => array('object_exists', htmlspecialchars($host1 . ' / ' . $user1)) - ); - return false; - } - $stmt = $pdo->prepare("INSERT INTO `imapsync` (`user2`, `exclude`, `delete1`, `delete2`, `timeout1`, `timeout2`, `automap`, `skipcrossduplicates`, `maxbytespersecond`, `subscribeall`, `dry`, `maxage`, `subfolder2`, `host1`, `authmech1`, `user1`, `password1`, `mins_interval`, `port1`, `enc1`, `delete2duplicates`, `custom_params`, `active`) - VALUES (:user2, :exclude, :delete1, :delete2, :timeout1, :timeout2, :automap, :skipcrossduplicates, :maxbytespersecond, :subscribeall, :dry, :maxage, :subfolder2, :host1, :authmech1, :user1, :password1, :mins_interval, :port1, :enc1, :delete2duplicates, :custom_params, :active)"); - $stmt->execute(array( - ':user2' => $username, - ':custom_params' => $custom_params, - ':exclude' => $exclude, - ':maxage' => $maxage, - ':delete1' => $delete1, - ':delete2' => $delete2, - ':timeout1' => $timeout1, - ':timeout2' => $timeout2, - ':automap' => $automap, - ':skipcrossduplicates' => $skipcrossduplicates, - ':maxbytespersecond' => $maxbytespersecond, - ':subscribeall' => $subscribeall, - ':dry' => $dry, - ':subfolder2' => $subfolder2, - ':host1' => $host1, - ':authmech1' => 'PLAIN', - ':user1' => $user1, - ':password1' => $password1, - ':mins_interval' => $mins_interval, - ':port1' => $port1, - ':enc1' => $enc1, - ':delete2duplicates' => $delete2duplicates, - ':active' => $active, - )); - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => array('mailbox_modified', $username) - ); - break; case 'domain': if ($_SESSION['mailcow_cc_role'] != "admin") { $_SESSION['return'][] = array( @@ -2280,202 +2094,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { ); } break; - case 'syncjob': - if (!is_array($_data['id'])) { - $ids = array(); - $ids[] = $_data['id']; - } - else { - $ids = $_data['id']; - } - if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - foreach ($ids as $id) { - $is_now = mailbox('get', 'syncjob_details', $id, array('with_password')); - if (!empty($is_now)) { - $username = $is_now['user2']; - $user1 = (!empty($_data['user1'])) ? $_data['user1'] : $is_now['user1']; - $active = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active']; - $last_run = (isset($_data['last_run'])) ? NULL : $is_now['last_run']; - $success = (isset($_data['success'])) ? NULL : $is_now['success']; - $delete2duplicates = (isset($_data['delete2duplicates'])) ? intval($_data['delete2duplicates']) : $is_now['delete2duplicates']; - $subscribeall = (isset($_data['subscribeall'])) ? intval($_data['subscribeall']) : $is_now['subscribeall']; - $dry = (isset($_data['dry'])) ? intval($_data['dry']) : $is_now['dry']; - $delete1 = (isset($_data['delete1'])) ? intval($_data['delete1']) : $is_now['delete1']; - $delete2 = (isset($_data['delete2'])) ? intval($_data['delete2']) : $is_now['delete2']; - $automap = (isset($_data['automap'])) ? intval($_data['automap']) : $is_now['automap']; - $skipcrossduplicates = (isset($_data['skipcrossduplicates'])) ? intval($_data['skipcrossduplicates']) : $is_now['skipcrossduplicates']; - $port1 = (!empty($_data['port1'])) ? $_data['port1'] : $is_now['port1']; - $password1 = (!empty($_data['password1'])) ? $_data['password1'] : $is_now['password1']; - $host1 = (!empty($_data['host1'])) ? $_data['host1'] : $is_now['host1']; - $subfolder2 = (isset($_data['subfolder2'])) ? $_data['subfolder2'] : $is_now['subfolder2']; - $enc1 = (!empty($_data['enc1'])) ? $_data['enc1'] : $is_now['enc1']; - $mins_interval = (!empty($_data['mins_interval'])) ? $_data['mins_interval'] : $is_now['mins_interval']; - $exclude = (isset($_data['exclude'])) ? $_data['exclude'] : $is_now['exclude']; - $custom_params = (isset($_data['custom_params'])) ? $_data['custom_params'] : $is_now['custom_params']; - $maxage = (isset($_data['maxage']) && $_data['maxage'] != "") ? intval($_data['maxage']) : $is_now['maxage']; - $maxbytespersecond = (isset($_data['maxbytespersecond']) && $_data['maxbytespersecond'] != "") ? intval($_data['maxbytespersecond']) : $is_now['maxbytespersecond']; - $timeout1 = (isset($_data['timeout1']) && $_data['timeout1'] != "") ? intval($_data['timeout1']) : $is_now['timeout1']; - $timeout2 = (isset($_data['timeout2']) && $_data['timeout2'] != "") ? intval($_data['timeout2']) : $is_now['timeout2']; - } - else { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - - // validate custom params - foreach (explode('-', $custom_params) as $param){ - if(empty($param)) continue; - - // extract option - if (str_contains($param, '=')) $param = explode('=', $param)[0]; - else $param = rtrim($param, ' '); - // remove first char if first char is - - if ($param[0] == '-') $param = ltrim($param, $param[0]); - - if (str_contains($param, ' ')) { - // bad char - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'bad character SPACE' - ); - return false; - } - - // check if param is whitelisted - if (!in_array(strtolower($param), $GLOBALS["IMAPSYNC_OPTIONS"]["whitelist"])){ - // bad option - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'bad option '. $param - ); - return false; - } - } - if (empty($subfolder2)) { - $subfolder2 = ""; - } - if (!isset($maxage) || !filter_var($maxage, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $maxage = "0"; - } - if (!isset($timeout1) || !filter_var($timeout1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $timeout1 = "600"; - } - if (!isset($timeout2) || !filter_var($timeout2, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 32000)))) { - $timeout2 = "600"; - } - if (!isset($maxbytespersecond) || !filter_var($maxbytespersecond, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 125000000)))) { - $maxbytespersecond = "0"; - } - if (!filter_var($port1, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 65535)))) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - if (!filter_var($mins_interval, FILTER_VALIDATE_INT, array('options' => array('min_range' => 1, 'max_range' => 43800)))) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - if (!is_valid_domain_name($host1)) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - if ($enc1 != "TLS" && $enc1 != "SSL" && $enc1 != "PLAIN") { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - if (@preg_match("/" . $exclude . "/", null) === false) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - $stmt = $pdo->prepare("UPDATE `imapsync` SET `delete1` = :delete1, - `delete2` = :delete2, - `automap` = :automap, - `skipcrossduplicates` = :skipcrossduplicates, - `maxage` = :maxage, - `maxbytespersecond` = :maxbytespersecond, - `subfolder2` = :subfolder2, - `exclude` = :exclude, - `host1` = :host1, - `last_run` = :last_run, - `success` = :success, - `user1` = :user1, - `password1` = :password1, - `mins_interval` = :mins_interval, - `port1` = :port1, - `enc1` = :enc1, - `delete2duplicates` = :delete2duplicates, - `custom_params` = :custom_params, - `timeout1` = :timeout1, - `timeout2` = :timeout2, - `subscribeall` = :subscribeall, - `dry` = :dry, - `active` = :active - WHERE `id` = :id"); - $stmt->execute(array( - ':delete1' => $delete1, - ':delete2' => $delete2, - ':automap' => $automap, - ':skipcrossduplicates' => $skipcrossduplicates, - ':id' => $id, - ':exclude' => $exclude, - ':maxage' => $maxage, - ':maxbytespersecond' => $maxbytespersecond, - ':subfolder2' => $subfolder2, - ':host1' => $host1, - ':user1' => $user1, - ':password1' => $password1, - ':last_run' => $last_run, - ':success' => $success, - ':mins_interval' => $mins_interval, - ':port1' => $port1, - ':enc1' => $enc1, - ':delete2duplicates' => $delete2duplicates, - ':custom_params' => $custom_params, - ':timeout1' => $timeout1, - ':timeout2' => $timeout2, - ':subscribeall' => $subscribeall, - ':dry' => $dry, - ':active' => $active, - )); - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => array('mailbox_modified', $username) - ); - } - break; case 'filter': if (!isset($_SESSION['acl']['filters']) || $_SESSION['acl']['filters'] != "1" ) { $_SESSION['return'][] = array( @@ -3652,6 +3270,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { } } + // Sync sources created by / targeting the mailbox and its OAuth tokens (owner); a PDOException rolls back below + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username); + // Update c_uid, c_name and mail in _sogo_static_view table $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username") ->execute([ @@ -4603,68 +4224,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { } return false; break; - case 'syncjob_details': - $syncjobdetails = array(); - if (!is_numeric($_data)) { - return false; - } - if (isset($_extra) && in_array('no_log', $_extra)) { - $field_query = $pdo->query('SHOW FIELDS FROM `imapsync` WHERE FIELD NOT IN ("returned_text", "password1")'); - $fields = $field_query->fetchAll(PDO::FETCH_ASSOC); - while($field = array_shift($fields)) { - $shown_fields[] = $field['Field']; - } - $stmt = $pdo->prepare("SELECT " . implode(',', (array)$shown_fields) . ", - `active` - FROM `imapsync` WHERE id = :id"); - } - elseif (isset($_extra) && in_array('with_password', $_extra)) { - $stmt = $pdo->prepare("SELECT *, - `active` - FROM `imapsync` WHERE id = :id"); - } - else { - $field_query = $pdo->query('SHOW FIELDS FROM `imapsync` WHERE FIELD NOT IN ("password1")'); - $fields = $field_query->fetchAll(PDO::FETCH_ASSOC); - while($field = array_shift($fields)) { - $shown_fields[] = $field['Field']; - } - $stmt = $pdo->prepare("SELECT " . implode(',', (array)$shown_fields) . ", - `active` - FROM `imapsync` WHERE id = :id"); - } - $stmt->execute(array(':id' => $_data)); - $syncjobdetails = $stmt->fetch(PDO::FETCH_ASSOC); - if (!empty($syncjobdetails['returned_text'])) { - $syncjobdetails['log'] = $syncjobdetails['returned_text']; - } - else { - $syncjobdetails['log'] = ''; - } - unset($syncjobdetails['returned_text']); - if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $syncjobdetails['user2'])) { - return false; - } - return $syncjobdetails; - break; - case 'syncjobs': - $syncjobdata = array(); - if (isset($_data) && filter_var($_data, FILTER_VALIDATE_EMAIL)) { - if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $_data)) { - return false; - } - } - else { - $_data = $_SESSION['mailcow_cc_username']; - } - $stmt = $pdo->prepare("SELECT `id` FROM `imapsync` WHERE `user2` = :username"); - $stmt->execute(array(':username' => $_data)); - $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); - while($row = array_shift($rows)) { - $syncjobdata[] = $row['id']; - } - return $syncjobdata; - break; case 'spam_score': $curl = curl_init(); curl_setopt($curl, CURLOPT_UNIX_SOCKET_PATH, '/var/lib/rspamd/rspamd.sock'); @@ -5494,46 +5053,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { break; case 'delete': switch ($_type) { - case 'syncjob': - if (!is_array($_data['id'])) { - $ids = array(); - $ids[] = $_data['id']; - } - else { - $ids = $_data['id']; - } - if (!isset($_SESSION['acl']['syncjobs']) || $_SESSION['acl']['syncjobs'] != "1" ) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - return false; - } - foreach ($ids as $id) { - if (!is_numeric($id)) { - return false; - } - $stmt = $pdo->prepare("SELECT `user2` FROM `imapsync` WHERE id = :id"); - $stmt->execute(array(':id' => $id)); - $user2 = $stmt->fetch(PDO::FETCH_ASSOC)['user2']; - if (!hasMailboxObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $user2)) { - $_SESSION['return'][] = array( - 'type' => 'danger', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => 'access_denied' - ); - continue; - } - $stmt = $pdo->prepare("DELETE FROM `imapsync` WHERE `id`= :id"); - $stmt->execute(array(':id' => $id)); - $_SESSION['return'][] = array( - 'type' => 'success', - 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr), - 'msg' => array('deleted_syncjob', $id) - ); - } - break; case 'filter': if (!is_array($_data['id'])) { $ids = array(); @@ -5764,6 +5283,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $stmt->execute(array( ':domain' => $domain, )); + $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain"); + $stmt->execute(array( + ':domain' => $domain, + )); $stmt = $pdo->prepare("DELETE FROM `domain_admins` WHERE `domain` = :domain"); $stmt->execute(array( ':domain' => $domain, @@ -6071,6 +5594,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) { $stmt->execute(array( ':username' => $username )); + // Tokens, scope targets and own sync sources of the mailbox + \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->accountChanged($username); $stmt = $pdo->prepare("DELETE FROM `filterconf` WHERE `object` = :username"); $stmt->execute(array( ':username' => $username diff --git a/data/web/inc/init_db.inc.php b/data/web/inc/init_db.inc.php index f6b0cd792..a9344e6c0 100644 --- a/data/web/inc/init_db.inc.php +++ b/data/web/inc/init_db.inc.php @@ -4,7 +4,7 @@ function init_db_schema() try { global $pdo; - $db_version = "18082026_1200"; + $db_version = "16102026_1200"; $stmt = $pdo->query("SHOW TABLES LIKE 'versions'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); @@ -755,35 +755,153 @@ function init_db_schema() ), "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" ), + "imapsync_source" => array( + "cols" => array( + "id" => "INT NOT NULL AUTO_INCREMENT", + "name" => "VARCHAR(100) NOT NULL", + "description" => "VARCHAR(255) NOT NULL DEFAULT ''", + "created_by" => "VARCHAR(255) NOT NULL DEFAULT ''", + "scope" => "ENUM('all','domain','user') NOT NULL DEFAULT 'user'", + "host1" => "VARCHAR(255) NOT NULL", + "port1" => "SMALLINT UNSIGNED NOT NULL", + "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'", + "auth_type" => "ENUM('PLAIN','LOGIN','CRAM-MD5','XOAUTH2') DEFAULT 'PLAIN'", + "oauth_flow" => "ENUM('client_credentials','authorization_code') NOT NULL DEFAULT 'client_credentials'", + "oauth_token_endpoint" => "VARCHAR(500) DEFAULT NULL", + "oauth_authorize_endpoint" => "VARCHAR(500) DEFAULT NULL", + "oauth_userinfo_endpoint" => "VARCHAR(500) DEFAULT NULL", + "oauth_client_id" => "VARCHAR(500) DEFAULT NULL", + "oauth_client_secret" => "VARCHAR(500) DEFAULT NULL", + "oauth_scope" => "VARCHAR(500) DEFAULT NULL", + "oauth_extra_params" => "TEXT DEFAULT NULL", + "oauth_access_token" => "TEXT DEFAULT NULL", + "oauth_token_expires" => "BIGINT UNSIGNED DEFAULT NULL", + "oauth_last_refresh_error" => "TEXT DEFAULT NULL", + "oauth_last_refresh_attempt" => "DATETIME NULL DEFAULT NULL", + "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", + "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP", + "active" => "TINYINT(1) NOT NULL DEFAULT '1'" + ), + "keys" => array( + "primary" => array( + "" => array("id") + ), + "unique" => array( + "uniq_creator_name" => array("created_by", "name") + ), + "key" => array( + "idx_created_by" => array("created_by"), + "idx_scope" => array("scope") + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + "imapsync_source_domain" => array( + "cols" => array( + "source_id" => "INT NOT NULL", + "domain" => "VARCHAR(255) NOT NULL" + ), + "keys" => array( + "primary" => array( + "" => array("source_id", "domain") + ), + "key" => array( + "idx_domain" => array("domain") + ), + "fkey" => array( + "fk_imapsync_source_domain" => array( + "col" => "source_id", + "ref" => "imapsync_source.id", + "delete" => "CASCADE", + "update" => "NO ACTION" + ) + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + "imapsync_source_user" => array( + "cols" => array( + "source_id" => "INT NOT NULL", + "username" => "VARCHAR(255) NOT NULL" + ), + "keys" => array( + "primary" => array( + "" => array("source_id", "username") + ), + "key" => array( + "idx_username" => array("username") + ), + "fkey" => array( + "fk_imapsync_source_user" => array( + "col" => "source_id", + "ref" => "imapsync_source.id", + "delete" => "CASCADE", + "update" => "NO ACTION" + ) + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + "imapsync_source_oauth_token" => array( + "cols" => array( + "source_id" => "INT NOT NULL", + "owner" => "VARCHAR(255) NOT NULL", + "username" => "VARCHAR(255) NOT NULL", + "access_token" => "TEXT DEFAULT NULL", + "refresh_token" => "TEXT DEFAULT NULL", + "token_expires" => "BIGINT UNSIGNED DEFAULT NULL", + "last_refresh_error" => "TEXT DEFAULT NULL", + "oauth_last_refresh_attempt" => "DATETIME NULL DEFAULT NULL", + "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", + "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP" + ), + "keys" => array( + "primary" => array( + "" => array("source_id", "owner", "username") + ), + "key" => array( + "idx_owner" => array("owner") + ), + "fkey" => array( + "fk_imapsync_source_oauth_token" => array( + "col" => "source_id", + "ref" => "imapsync_source.id", + "delete" => "CASCADE", + "update" => "NO ACTION" + ) + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), "imapsync" => array( "cols" => array( "id" => "INT NOT NULL AUTO_INCREMENT", "user2" => "VARCHAR(255) NOT NULL", - "host1" => "VARCHAR(255) NOT NULL", - "authmech1" => "ENUM('PLAIN','LOGIN','CRAM-MD5') DEFAULT 'PLAIN'", + "source_id" => "INT NOT NULL", "regextrans2" => "VARCHAR(255) DEFAULT ''", "authmd51" => "TINYINT(1) NOT NULL DEFAULT 0", "domain2" => "VARCHAR(255) NOT NULL DEFAULT ''", "subfolder2" => "VARCHAR(255) NOT NULL DEFAULT ''", "user1" => "VARCHAR(255) NOT NULL", - "password1" => "VARCHAR(255) NOT NULL", + "password1" => "VARCHAR(255) NOT NULL DEFAULT ''", "exclude" => "VARCHAR(500) NOT NULL DEFAULT ''", "maxage" => "SMALLINT NOT NULL DEFAULT '0'", "mins_interval" => "SMALLINT UNSIGNED NOT NULL DEFAULT '0'", "maxbytespersecond" => "VARCHAR(50) NOT NULL DEFAULT '0'", - "port1" => "SMALLINT UNSIGNED NOT NULL", - "enc1" => "ENUM('TLS','SSL','PLAIN') DEFAULT 'TLS'", "delete2duplicates" => "TINYINT(1) NOT NULL DEFAULT '1'", "delete1" => "TINYINT(1) NOT NULL DEFAULT '0'", "delete2" => "TINYINT(1) NOT NULL DEFAULT '0'", "automap" => "TINYINT(1) NOT NULL DEFAULT '0'", "skipcrossduplicates" => "TINYINT(1) NOT NULL DEFAULT '0'", - "custom_params" => "VARCHAR(512) NOT NULL DEFAULT ''", + "custom_params" => "TEXT", + // original legacy custom_params string, only set when options were dropped during migration (never used by the runner) + "custom_params_legacy" => "TEXT DEFAULT NULL", "timeout1" => "SMALLINT NOT NULL DEFAULT '600'", "timeout2" => "SMALLINT NOT NULL DEFAULT '600'", "subscribeall" => "TINYINT(1) NOT NULL DEFAULT '1'", "dry" => "TINYINT(1) NOT NULL DEFAULT '0'", "is_running" => "TINYINT(1) NOT NULL DEFAULT '0'", + "prio" => "INT NOT NULL DEFAULT 0", "returned_text" => "LONGTEXT", "last_run" => "TIMESTAMP NULL DEFAULT NULL", "success" => "TINYINT(1) UNSIGNED DEFAULT NULL", @@ -795,6 +913,32 @@ function init_db_schema() "keys" => array( "primary" => array( "" => array("id") + ), + "key" => array( + "idx_source_id" => array("source_id"), + "idx_prio" => array("prio") + ), + "fkey" => array( + "fk_imapsync_source" => array( + "col" => "source_id", + "ref" => "imapsync_source.id", + "delete" => "RESTRICT", + "update" => "NO ACTION" + ) + ) + ), + "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" + ), + "imapsync_settings" => array( + "cols" => array( + "name" => "VARCHAR(64) NOT NULL", + "value" => "VARCHAR(255) NOT NULL", + "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)", + "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP" + ), + "keys" => array( + "primary" => array( + "" => array("name") ) ), "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC" @@ -1184,11 +1328,128 @@ function init_db_schema() } } + // Migrate legacy imapsync jobs (host/port/enc per job) to domain-scoped legacy sources + // Re-entrant: only rows with source_id IS NULL are migrated, the legacy columns are kept while any are left + if ($table == 'imapsync') { + $stmt = $pdo->query("SHOW TABLES LIKE 'imapsync'"); + if (count($stmt->fetchAll(PDO::FETCH_ASSOC)) != 0) { + $imapsync_cols = array_column($pdo->query("SHOW COLUMNS FROM `imapsync`")->fetchAll(PDO::FETCH_ASSOC), 'Field', 'Field'); + if (isset($imapsync_cols['host1'])) { + if (!isset($imapsync_cols['source_id'])) { + $pdo->exec("ALTER TABLE `imapsync` ADD COLUMN `source_id` INT DEFAULT NULL"); + } + if (!isset($imapsync_cols['custom_params_legacy'])) { + $pdo->exec("ALTER TABLE `imapsync` ADD COLUMN `custom_params_legacy` TEXT DEFAULT NULL"); + } + // Converted JSON may not fit into the old VARCHAR(512) + $pdo->exec("ALTER TABLE `imapsync` MODIFY COLUMN `custom_params` TEXT"); + $imapsync_allow = \Mailcow\Imapsync\Model\CustomParams::allowlist($GLOBALS['IMAPSYNC_OPTIONS'] ?? [], $GLOBALS['IMAPSYNC_OPTIONS_DEFAULT'] ?? []); + $source_sel = $pdo->prepare("SELECT `id` FROM `imapsync_source` WHERE `created_by` = '' AND `name` = :name"); + $source_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source` (`name`, `description`, `created_by`, `scope`, `host1`, `port1`, `enc1`, `auth_type`, `active`) + VALUES (:name, '', '', 'domain', :host, :port, :enc, :auth, 1)"); + $source_domain_ins = $pdo->prepare("INSERT IGNORE INTO `imapsync_source_domain` (`source_id`, `domain`) VALUES (:id, :domain)"); + $job_upd = $pdo->prepare("UPDATE `imapsync` SET `source_id` = :source_id, `custom_params` = :cp, `custom_params_legacy` = :cp_legacy, + `dry` = GREATEST(`dry`, :dry) WHERE `id` = :id AND `source_id` IS NULL"); + $stmt = $pdo->query("SELECT `id`, `user2`, `host1`, `port1`, `enc1`, `custom_params` FROM `imapsync` WHERE `source_id` IS NULL"); + foreach ($stmt->fetchAll(PDO::FETCH_ASSOC) as $row) { + // Parse the legacy custom_params string like the old runner did (all tokens up to the next option are its value) + // into pairs valid under $IMAPSYNC_OPTIONS, a token after a flag is never its value. + // --authmech1 PLAIN|LOGIN|CRAM-MD5 becomes the source auth type, --dry the job's dry flag (never run a simulation for real). + // Anything else that does not fit is dropped, the original string is then kept in custom_params_legacy. + $pairs = array(); + $auth = null; + $dry = false; + $dropped = false; + $tokens = preg_split('/\s+/', trim((string)$row['custom_params']), -1, PREG_SPLIT_NO_EMPTY); + for ($i = 0, $n = count($tokens); $i < $n; $i++) { + if ($tokens[$i][0] !== '-') { + // stray value without option + $dropped = true; + continue; + } + @list($opt, $val) = explode('=', ltrim($tokens[$i], '-'), 2); + $opt = strtolower($opt); + $inline = ($val !== null); + $rest = array(); + while ($i + 1 < $n && $tokens[$i + 1][0] !== '-') { + $rest[] = $tokens[++$i]; + } + $rule = ($opt === 'authmech1') ? true : ($imapsync_allow[$opt] ?? null); + if ($opt === 'dry' || $rule === false) { + if ($inline || !empty($rest)) $dropped = true; + if ($opt === 'dry') { + $dry = true; + } elseif (!$inline) { + $pairs[] = array('o' => $opt, 'v' => ''); + } + continue; + } + // unknown option, or "--opt=a b" (the old runner cut the value at the first space) + if ($rule === null || ($inline && !empty($rest))) { + $dropped = true; + continue; + } + $val = $inline ? $val : implode(' ', $rest); + if ($opt === 'authmech1' && $auth === null && in_array(strtoupper($val), array('PLAIN', 'LOGIN', 'CRAM-MD5'), true)) { + $auth = strtoupper($val); + } elseif ($opt === 'authmech1' || trim($val) === '' + || (is_string($rule) && preg_match('~^(?:' . str_replace('~', '\~', $rule) . ')$~D', $val) !== 1)) { + $dropped = true; + } else { + $pairs[] = array('o' => $opt, 'v' => $val); + } + } + // Old runner ignored the authmech1 column, only a custom --authmech1 was effective + $auth = $auth ?? 'PLAIN'; + $host = strtolower(trim((string)$row['host1'])); + $port = (intval($row['port1']) >= 1 && intval($row['port1']) <= 65535) ? intval($row['port1']) : 143; + $enc = in_array($row['enc1'], array('TLS', 'SSL', 'PLAIN'), true) ? $row['enc1'] : 'TLS'; + $at = strrchr((string)$row['user2'], '@'); + $domain = ($at === false) ? '' : strtolower(substr($at, 1)); + // One source per (host, port, enc, auth, domain), name always unique and <= 100 chars + $name = 'legacy-' . mb_substr($host, 0, 60) . '-' . $port . '-' . substr(sha1($host . '|' . $port . '|' . $enc . '|' . $auth . '|' . $domain), 0, 8); + $source_sel->execute(array(':name' => $name)); + if (($source_id = $source_sel->fetchColumn()) === false) { + $source_ins->execute(array(':name' => $name, ':host' => $host, ':port' => $port, ':enc' => $enc, ':auth' => $auth)); + $source_sel->execute(array(':name' => $name)); + if (($source_id = $source_sel->fetchColumn()) === false) continue; + } + if ($domain !== '') { + $source_domain_ins->execute(array(':id' => $source_id, ':domain' => $domain)); + } + $job_upd->execute(array( + ':source_id' => $source_id, + ':cp' => json_encode($pairs, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), + ':cp_legacy' => $dropped ? $row['custom_params'] : null, + ':dry' => $dry ? 1 : 0, + ':id' => $row['id'] + )); + } + + // Never throw on leftovers: keep source_id nullable and the legacy columns, retried with the next db_version + $pending = intval($pdo->query("SELECT COUNT(*) FROM `imapsync` WHERE `source_id` IS NULL")->fetchColumn()); + if ($pending > 0) { + $properties['cols']['source_id'] = "INT DEFAULT NULL"; + $legacy_col_types = array( + 'host1' => "VARCHAR(255) DEFAULT NULL", + 'port1' => "SMALLINT UNSIGNED DEFAULT NULL", + 'enc1' => "ENUM('TLS','SSL','PLAIN') DEFAULT NULL", + 'authmech1' => "ENUM('PLAIN','LOGIN','CRAM-MD5') DEFAULT NULL" + ); + foreach ($legacy_col_types as $col => $type) { + if (isset($imapsync_cols[$col])) $properties['cols'][$col] = $type; + } + error_log('mailcow: imapsync migration: ' . $pending . ' sync job(s) could not be assigned to a source and will not run'); + } + } + } + } + $stmt = $pdo->query("SHOW TABLES LIKE '" . $table . "'"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); if ($num_results != 0) { $stmt = $pdo->prepare("SELECT CONCAT('ALTER TABLE `', `table_schema`, '`.', `table_name`, ' DROP FOREIGN KEY ', `constraint_name`, ';') AS `FKEY_DROP` FROM `information_schema`.`table_constraints` - WHERE `constraint_type` = 'FOREIGN KEY' AND `table_name` = :table;"); + WHERE `constraint_type` = 'FOREIGN KEY' AND `table_schema` = DATABASE() AND `table_name` = :table;"); $stmt->execute(array(':table' => $table)); $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); while ($row = array_shift($rows)) { @@ -1349,20 +1610,29 @@ function init_db_schema() AND (JSON_CONTAINS_PATH(`call`, 'one', '$[2].password') OR JSON_CONTAINS_PATH(`call`, 'one', '$[2].password2'));"); - // Mitigate imapsync argument injection issue - $pdo->query("UPDATE `imapsync` SET `custom_params` = '' - WHERE `custom_params` LIKE '%pipemess%' - OR custom_params LIKE '%skipmess%' - OR custom_params LIKE '%delete2foldersonly%' - OR custom_params LIKE '%delete2foldersbutnot%' - OR custom_params LIKE '%regexflag%' - OR custom_params LIKE '%pipemess%' - OR custom_params LIKE '%regextrans2%' - OR custom_params LIKE '%maxlinelengthcmd%';"); + // Mask syncjob secrets in old log entries (data and attr arguments of the logged call) + $mask_paths = array(); + $unmasked = array(); + foreach (array(3, 4) as $idx) { + foreach (array('password1', 'password2', 'oauth_client_secret', 'oauth_extra_params') as $key) { + $mask_paths[] = "'$[" . $idx . "]." . $key . "'"; + // only rows still holding a value other than the string '*' (objects/arrays/numbers included), missing paths are NULL + $unmasked[] = "JSON_UNQUOTE(JSON_EXTRACT(`call`, '$[" . $idx . "]." . $key . "')) <> '*'"; + } + } + $pdo->exec("UPDATE `logs` SET `call` = JSON_REPLACE(`call`, " . implode(", '*', ", $mask_paths) . ", '*') + WHERE JSON_VALID(`call`) + AND (JSON_VALUE(`call`, '$[0]') = 'syncjob' + OR JSON_VALUE(`call`, '$[0]') LIKE 'imapsync%' + OR (JSON_VALUE(`call`, '$[0]') = 'mailbox' AND JSON_VALUE(`call`, '$[2]') = 'syncjob')) + AND (" . implode(" OR ", $unmasked) . ");"); // Migrate webauthn tfa $stmt = $pdo->query("ALTER TABLE `tfa` MODIFY COLUMN `authmech` ENUM('yubi_otp', 'u2f', 'hotp', 'totp', 'webauthn')"); + // Syncjobs: seed global settings + $pdo->query("INSERT IGNORE INTO `imapsync_settings` (`name`, `value`) VALUES ('max_parallel', '1')"); + // Inject admin if not exists $stmt = $pdo->query("SELECT NULL FROM `admin`"); $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC)); @@ -1544,6 +1814,10 @@ function init_db_schema() } if (php_sapi_name() == "cli") { include '/web/inc/vars.inc.php'; + if (file_exists('/web/inc/vars.local.inc.php')) { + include '/web/inc/vars.local.inc.php'; + } + require_once '/web/inc/lib/vendor/autoload.php'; include '/web/inc/functions.inc.php'; include '/web/inc/functions.docker.inc.php'; // $now = new DateTime(); @@ -1561,7 +1835,7 @@ if (php_sapi_name() == "cli") { //PDO::MYSQL_ATTR_INIT_COMMAND => "SET time_zone = '" . $offset . "', group_concat_max_len = 3423543543;", ]; $pdo = new PDO($dsn, $database_user, $database_pass, $opt); - $stmt = $pdo->query("SELECT COUNT('OK') AS OK_C FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'sogo_view' OR TABLE_NAME = '_sogo_static_view';"); + $stmt = $pdo->query("SELECT COUNT('OK') AS OK_C FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = DATABASE() AND (TABLE_NAME = 'sogo_view' OR TABLE_NAME = '_sogo_static_view');"); $res = $stmt->fetch(PDO::FETCH_ASSOC); if (intval($res['OK_C']) === 2) { // Be more precise when replacing into _sogo_static_view, col orders may change diff --git a/data/web/inc/vars.inc.php b/data/web/inc/vars.inc.php index 72c5da67a..bdf0d3a34 100644 --- a/data/web/inc/vars.inc.php +++ b/data/web/inc/vars.inc.php @@ -287,131 +287,98 @@ $RSPAMD_MAPS = array( ); -$IMAPSYNC_OPTIONS = array( - 'whitelist' => array( - 'abort', - 'authmd51', - 'authmd52', - 'authmech1', - 'authmech2', - 'authuser1', - 'authuser2', - 'debug', - 'debugcontent', - 'debugcrossduplicates', - 'debugflags', - 'debugfolders', - 'debugimap', - 'debugimap1', - 'debugimap2', - 'debugmemory', - 'debugssl', - 'delete1emptyfolders', - 'delete2folders', - 'disarmreadreceipts', - 'domain1', - 'domain2', - 'domino1', - 'domino2', - 'dry', - 'errorsmax', - 'exchange1', - 'exchange2', - 'exitwhenover', - 'expunge1', - 'f1f2', - 'filterbuggyflags', - 'folder', - 'folderfirst', - 'folderlast', - 'folderrec', - 'gmail1', - 'gmail2', - 'idatefromheader', - 'include', - 'inet4', - 'inet6', - 'justconnect', - 'justfolders', - 'justfoldersizes', - 'justlogin', - 'keepalive1', - 'keepalive2', - 'log', - 'logdir', - 'logfile', - 'maxbytesafter', - 'maxlinelength', - 'maxmessagespersecond', - 'maxsize', - 'maxsleep', - 'minage', - 'minsize', - 'noabletosearch', - 'noabletosearch1', - 'noabletosearch2', - 'noexpunge1', - 'noexpunge2', - 'nofoldersizesatend', - 'noid', - 'nolog', - 'nomixfolders', - 'noresyncflags', - 'nossl1', - 'nossl2', - 'nosyncacls', - 'notls1', - 'notls2', - 'nouidexpunge2', - 'nousecache', - 'oauthaccesstoken1', - 'oauthaccesstoken2', - 'oauthdirect1', - 'oauthdirect2', - 'office1', - 'office2', - 'pidfile', - 'pidfilelocking', - 'prefix1', - 'prefix2', - 'proxyauth1', - 'proxyauth2', - 'resyncflags', - 'resynclabels', - 'search', - 'search1', - 'search2', - 'sep1', - 'sep2', - 'showpasswords', - 'skipemptyfolders', - 'ssl2', - 'sslargs1', - 'sslargs2', - 'subfolder1', - 'subscribe', - 'subscribed', - 'syncacls', - 'syncduplicates', - 'syncinternaldates', - 'synclabels', - 'tests', - 'testslive', - 'testslive6', - 'tls2', - 'truncmess', - 'usecache', - 'useheader', - 'useuid' - ), - 'blacklist' => array( - 'skipmess', - 'delete2foldersonly', - 'delete2foldersbutnot', - 'regexflag', - 'regexmess', - 'pipemess', - 'regextrans2', - 'maxlinelengthcmd' - ) +// Allowed imapsync options for a sync job's custom_params (key = option name, lowercase, without dashes): +// true = option requires a non-empty value +// false = pure flag, must NOT have a value +// string = PCRE (no delimiters) the whole non-empty value must match +// Anything not listed here is rejected when a job is saved (the runner passes stored options as they are). +// Override in vars.local.inc.php per option, e.g. $IMAPSYNC_OPTIONS['someoption'] = true; or unset($IMAPSYNC_OPTIONS['debug']); +// The old format $IMAPSYNC_OPTIONS['whitelist'] (list of names) is no longer supported and ignored (logged). +// NEVER add (RCE): 'delete2foldersonly', 'delete2foldersbutnot', 'regexflag', 'regexmess', 'pipemess', 'regextrans2', 'maxlinelengthcmd' +// NEVER add options taking local file paths (e.g. 'logfile', 'logdir', 'pidfile', 'passfile1', 'sslargs1' with SSL_ca_file) +$IMAPSYNC_OPTIONS_DEFAULT = array( + 'abort' => false, + 'authmd51' => false, + 'authuser1' => true, + 'debug' => false, + 'debugcontent' => false, + 'debugcrossduplicates' => false, + 'debugflags' => false, + 'debugfolders' => false, + 'debugimap' => false, + 'debugimap1' => false, + 'debugmemory' => false, + 'debugssl' => true, + 'delete1emptyfolders' => false, + 'delete2folders' => false, + 'disarmreadreceipts' => false, + 'domain1' => true, + 'domino1' => false, + 'errorsmax' => true, + 'exchange1' => false, + 'exitwhenover' => true, + 'expunge1' => false, + 'f1f2' => true, + 'filterbuggyflags' => false, + 'folder' => true, + 'folderfirst' => true, + 'folderlast' => true, + 'folderrec' => true, + 'gmail1' => false, + 'idatefromheader' => false, + 'include' => true, + 'inet4' => false, + 'inet6' => false, + 'justconnect' => false, + 'justfolders' => false, + 'justfoldersizes' => false, + 'justlogin' => false, + 'keepalive1' => false, + 'log' => false, + 'maxbytesafter' => true, + 'maxlinelength' => true, + 'maxmessagespersecond' => true, + 'maxsize' => true, + 'maxsleep' => true, + 'minage' => true, + 'minsize' => true, + 'noabletosearch' => false, + 'noabletosearch1' => false, + 'noexpunge1' => false, + 'nofoldersizesatend' => false, + 'noid' => false, + 'nolog' => false, + 'nomixfolders' => false, + 'noresyncflags' => false, + 'nossl1' => false, + 'nosyncacls' => false, + 'notls1' => false, + 'nousecache' => false, + 'office1' => false, + 'prefix1' => true, + 'prefix2' => true, + 'proxyauth1' => false, + 'resyncflags' => false, + 'resynclabels' => false, + 'search' => true, + 'search1' => true, + 'sep1' => true, + 'sep2' => true, + 'skipemptyfolders' => false, + 'sslargs1' => '^SSL_verify_mode=[01]$', + 'subfolder1' => true, + 'subscribe' => false, + 'subscribed' => false, + 'syncacls' => false, + 'syncduplicates' => false, + 'syncinternaldates' => false, + 'synclabels' => false, + 'tests' => false, + 'testslive' => false, + 'testslive6' => false, + 'truncmess' => true, + 'usecache' => false, + 'useheader' => true, + 'useuid' => false, ); +$IMAPSYNC_OPTIONS = $IMAPSYNC_OPTIONS_DEFAULT; diff --git a/data/web/js/build/013-mailcow.js b/data/web/js/build/013-mailcow.js index 9586af33d..d9182316a 100644 --- a/data/web/js/build/013-mailcow.js +++ b/data/web/js/build/013-mailcow.js @@ -73,7 +73,8 @@ $(document).ready(function() { } // form cache - $('[data-cached-form="true"]').formcache({key: $(this).data('id')}); + // fields marked with data-formcache-skip (secrets) are never written to localStorage + $('[data-cached-form="true"]').formcache({key: $(this).data('id'), controls: ['select', 'textarea', 'input:not([data-formcache-skip])']}); // tooltips $(function () { @@ -113,6 +114,8 @@ $(document).ready(function() { } localStorage.setItem(key, tab_id); + // remember which card of the tab was opened + localStorage.setItem(key + ':collapse', $(this).attr('data-bs-target').substring(1)); }); // open last tab $('[role="tablist"]').each(function (idx, elem) { @@ -128,7 +131,10 @@ $(document).ready(function() { dropdown.find('.dropdown-menu').removeClass('show'); dropdown.find('.dropdown-toggle').attr('aria-expanded', 'false'); var tab = $('[id^="' + lastTab + '"]'); - $(tab).find('.card-body.collapse:first').collapse('show'); + var lastCollapse = localStorage.getItem(key + ':collapse'); + var $card = lastCollapse ? $(tab).find('.card-body.collapse').filter(function() { return this.id === lastCollapse; }) : $(); + if (!$card.length) $card = $(tab).find('.card-body.collapse:first'); + $card.collapse('show'); } }); })(); diff --git a/data/web/js/site/imapsync.js b/data/web/js/site/imapsync.js new file mode 100644 index 000000000..bcf321e00 --- /dev/null +++ b/data/web/js/site/imapsync.js @@ -0,0 +1,707 @@ +// IMAP sync jobs and sync sources (admin/domainadmin mailbox page, user page, edit page). +// Loaded via $js_minifier only on pages that show sync jobs. +var mailcowImapsync = (function($) { + 'use strict'; + + // Lang strings: lang.syncjobs (mailbox/user page) or lang_syncjobs (edit page), '' if missing + function L(key) { + var src = (typeof lang_syncjobs !== 'undefined' && lang_syncjobs) ? lang_syncjobs : + ((typeof lang !== 'undefined' && lang && lang.syncjobs) ? lang.syncjobs : {}); + return (src[key] !== undefined && src[key] !== null) ? src[key] : ''; + } + function esc(s) { + return escapeHtml(s === null || s === undefined ? '' : s); + } + // same format as "last run" (YYYY-MM-DD HH:mm:ss), independent of the browser locale + function formatDate(ts) { + if (!ts) return ''; + var d = new Date(ts * 1000); + var p = function(n) { return (n < 10 ? '0' : '') + n; }; + return d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + + p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds()); + } + function isAdmin() { + return typeof mailcow_cc_role !== 'undefined' && mailcow_cc_role === 'admin'; + } + + // bootstrap-select: refresh an existing instance, or init with the mailcow defaults + function refreshSelect($sel) { + if (typeof $sel.selectpicker !== 'function') return; + $sel.each(function() { + var $s = $(this); + if ($s.data('selectpicker')) { + $s.selectpicker('refresh'); + } else { + $s.selectpicker({ + 'styleBase': 'btn btn-xs-lg', + 'noneSelectedText': (typeof lang_footer !== 'undefined') ? lang_footer.nothing_selected : '' + }); + } + }); + } + + // Long source labels must not overflow the modal on small screens + function injectStyle() { + if ($('#imapsync-style').length) return; + $('head').append(''); + } + + // ===== Display helpers (tables) ===== + + // Visibility label of a source row + function scopeDisplay(item) { + if (item.scope === 'all') return ' ' + esc(L('source_scope_all')); + if (item.scope === 'domain') return esc(L('source_scope_domain') + ': ' + (item.domains || []).join(', ')); + var us = item.users || []; + if (us.length === 0) return esc(item.created_by || ''); + return esc(L('source_scope_user') + ': ' + us.join(', ')); + } + + // Token badge of a client_credentials source: based on token existence/error, expiry in title + function sourceTokenBadge(item) { + if (item.oauth_last_refresh_error) { + return '' + esc(L('token_state_failed')) + ''; + } + if (item.has_access_token) { + var title = item.oauth_token_expires ? L('source_token_status') + ': ' + formatDate(item.oauth_token_expires) : ''; + return '' + esc(L('token_state_issued')) + ''; + } + return '' + esc(L('token_state_none')) + ''; + } + + // Auth column of a source row: auth type, and for XOAUTH2 the flow (+ token badge for client credentials) + function authDisplay(item) { + var out = esc(item.auth_type); + if (item.auth_type !== 'XOAUTH2') return out; + var flow = item.oauth_flow || 'client_credentials'; + out += ' (' + esc(flow === 'authorization_code' ? L('source_flow_badge_user') : L('source_flow_badge_app')) + ')'; + // authorization_code tokens live per job (local mailbox + remote user), not on the source + if (flow === 'authorization_code') return out; + return out + '
' + sourceTokenBadge(item) + '
'; + } + + // Per-job token badge for authorization_code sources ('' for everything else) + function jobTokenBadge(item) { + if (item.source_auth_type !== 'XOAUTH2' || item.source_oauth_flow !== 'authorization_code') return ''; + if (item.user_token_error) { + return '' + esc(L('token_state_failed')) + ''; + } + if (parseInt(item.user_token_exists) === 1) { + var title = item.user_token_expires ? L('source_token_status') + ': ' + formatDate(item.user_token_expires) : ''; + return '' + esc(L('token_state_issued')) + ''; + } + return '' + esc(L('token_state_pending')) + ''; + } + + // Warning badge for jobs with not migrated legacy custom parameters + function legacyBadge(item) { + if (!item.custom_params_legacy) return ''; + return '' + + ' ' + esc(L('custom_params_legacy_badge')) + ''; + } + + // Wrappable table cell content (long hosts, mailboxes, legacy source names) + function wrapCell(html) { + return '
' + html + '
'; + } + + // Server column: "remote user @ host:port", source name below, badges after + function serverDisplay(item) { + var out = esc(item.user1 || '') + ' @ ' + esc(item.source_host || '?') + ':' + esc(item.source_port || '?'); + if (item.source_name) out += '
' + esc(item.source_name) + '
'; + var badges = [jobTokenBadge(item), legacyBadge(item)].filter(Boolean); + if (badges.length) out += '
' + badges.join(' ') + '
'; + return wrapCell(out); + } + + // Prepares the shared fields of a job row (run state, last run, result) + function prepareJobRow(item) { + item.server_w_port = serverDisplay(item); + if (parseInt(item.is_running) === 1) { + item.is_running = '' + esc(L('status_running')) + ''; + } else if (parseInt(item.active) !== 1) { + item.is_running = '' + esc(L('status_inactive')) + ''; + } else { + item.is_running = '' + esc(L('status_waiting')) + ''; + } + if (!item.last_run) { + item.last_run = '' + esc(L('never_run')) + ''; + } + if (item.success == null) { + item.success = '-'; + item.exit_status = ''; + } else { + item.success = ''; + } + if (L('syncjob_' + item.exit_status)) { + item.exit_status = L('syncjob_' + item.exit_status); + } else if (item.success != '-') { + item.exit_status = L('syncjob_check_log'); + } + item.exit_status = item.success + ' ' + esc(item.exit_status); + return item; + } + + // Job tables: ignore DataTables state saved before the rework (old ID sort, column visibility) + var JOB_TABLE_STATE_VERSION = 2; + function jobTableStateSave(settings, data) { + data.imapsync_state = JOB_TABLE_STATE_VERSION; + } + function jobTableStateLoad(settings, data) { + if (!data || data.imapsync_state !== JOB_TABLE_STATE_VERSION) return false; + // column visibility depends on the role, never restore it + if (Array.isArray(data.columns)) { + data.columns.forEach(function(c) { if (c) delete c.visible; }); + } + } + + // Sync source table (mailbox + user page). canManage: session may add/edit sources at all. + function drawSourceTable(canManage) { + if ($.fn.DataTable.isDataTable('#imapsync_source_table')) { + $('#imapsync_source_table').DataTable().columns.adjust().responsive.recalc(); + return; + } + $('#imapsync_source_table').DataTable({ + responsive: true, + processing: true, + serverSide: false, + stateSave: true, + pageLength: pagination_size, + dom: "<'row'<'col-sm-12 col-md-6'f><'col-sm-12 col-md-6'l>>" + "tr" + + "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>", + language: lang_datatables, + order: [[2, 'asc']], + ajax: { + type: "GET", + url: "/api/v1/get/syncjob_source/all", + dataSrc: function(json) { + if (!Array.isArray(json)) return []; + $.each(json, function(i, item) { + item.endpoint = wrapCell(esc((item.host || item.host1) + ':' + (item.port || item.port1) + ' (' + (item.enc || item.enc1) + ')')); + item.name_display = esc(item.name); + if (item.description) item.name_display += '
' + esc(item.description) + '
'; + item.name_display = wrapCell(item.name_display); + item.auth_display = authDisplay(item); + item.visibility_display = wrapCell(scopeDisplay(item)); + if (canManage && item.can_edit) { + item.action = '
' + + ' ' + esc(lang.edit) + '' + + ' ' + esc(lang.remove) + '' + + '
'; + } else { + item.action = ' ' + esc(L('source_view')) + ''; + } + }); + return json; + }, + error: function() { + mailcow_alert_box(L('source_list_error'), 'danger'); + } + }, + columns: [ + { + // placeholder, so the responsive control does not cover the ID + title: '', + data: null, + searchable: false, + orderable: false, + defaultContent: '', + responsivePriority: 1 + }, + { title: L('id'), data: 'id', defaultContent: '', responsivePriority: 4 }, + { title: L('source_name'), data: 'name_display', defaultContent: '', responsivePriority: 2 }, + { title: L('source_scope'), data: 'visibility_display', defaultContent: '', responsivePriority: 6 }, + { title: L('source_server'), data: 'endpoint', defaultContent: '', responsivePriority: 5 }, + { title: L('source_auth_type'), data: 'auth_display', defaultContent: '', responsivePriority: 3 }, + { + title: lang.active, + data: 'active', + defaultContent: '', + responsivePriority: 7, + render: function(d) { return d == 1 ? '' : ''; } + }, + { + title: lang.action, + data: 'action', + orderable: false, + className: 'dt-sm-head-hidden dt-data-w100 dtr-col-md dt-text-right', + responsivePriority: 3, + defaultContent: '' + } + ] + }); + } + + // ===== Source dropdown in the job modal / edit page ===== + + function setNoSourceState($sel, empty) { + $sel.closest('form').find('.imapsync-no-source').toggleClass('d-none', !empty); + } + + // Replace the options of a source dropdown (array of jQuery options, kept in order) and select keepId (if present and enabled). + // bootstrap-select 1.14 keeps a detached title option (incl. its selectedness) across empty()/refresh, + // which overrides the selected option and leaves stale list entries, so destroy and re-init instead. + function rebuildSourceSelect($sel, $options, keepId) { + if (typeof $sel.selectpicker === 'function' && $sel.data('selectpicker')) $sel.selectpicker('destroy'); + $sel.find('option').remove(); + $sel.append($options); + var $keep = (keepId !== '') ? $sel.find('option').filter(function() { + return $(this).val() === keepId && !$(this).prop('disabled'); + }).first() : $(); + $sel.find('option').prop('selected', false).removeAttr('selected'); + if ($keep.length) { + $keep.attr('selected', 'selected').prop('selected', true); + } else { + // nothing to keep: select nothing, the required check makes the user pick a source + $sel.prop('selectedIndex', -1); + } + $sel.prop('disabled', false); + refreshSelect($sel); + if ($keep.length && $sel.data('selectpicker')) $sel.selectpicker('val', keepId); + return $keep.length > 0; + } + + // (Re)build every source dropdown from the ACL-filtered API + function populateSourceSelects() { + var $selects = $('select.imapsync-source-select'); + if (!$selects.length) return; + $selects.each(function() { + var $sel = $(this); + if ($sel.val()) $sel.data('keep-id', String($sel.val())); + $sel.data('saved-options', $sel.find('option:not(.bs-title-option)').clone()); + // disable while loading instead of a transient placeholder option + $sel.prop('disabled', true); + refreshSelect($sel); + }); + $.get('/api/v1/get/syncjob_source/all').done(function(sources) { + if (!Array.isArray(sources)) sources = []; + sources.sort(function(a, b) { return String(a.name).localeCompare(String(b.name)); }); + $selects.each(function() { + var $sel = $(this); + var cur = String($sel.data('current-source-id') || ''); + var keepId = String($sel.data('keep-id') || cur); + var opts = []; + var selectable = 0; + var listed = false; + $.each(sources, function(_, src) { + var id = String(src.id); + var $opt = sourceOption(src.id, src.name, src.host || src.host1, src.port || src.port1, src.auth_type, src.oauth_flow); + if (src.active != 1) { + $opt.attr('data-subtext', $opt.attr('data-subtext') + ' · ' + L('status_inactive')); + // the backend rejects inactive sources, only the assigned one stays selectable on the edit page + if (id !== cur) $opt.prop('disabled', true); + } + if (!$opt.prop('disabled')) selectable++; + if (id === cur) listed = true; + opts.push($opt); + }); + // edit page: the assigned source stays selectable even if the viewer cannot see it + if (cur !== '' && !listed) { + var $cur = sourceOption(cur, $sel.data('current-source-name'), $sel.data('current-source-host'), + $sel.data('current-source-port'), $sel.data('current-source-auth-type'), $sel.data('current-source-oauth-flow')); + $cur.text($cur.text() + ' ' + L('source_not_visible')); + opts.unshift($cur); + selectable++; + } + rebuildSourceSelect($sel, opts, keepId); + setNoSourceState($sel, selectable === 0); + $sel.trigger('change'); + }); + }).fail(function() { + // keep what was there before (e.g. the assigned source on the edit page) + $selects.each(function() { + var $sel = $(this); + rebuildSourceSelect($sel, $sel.data('saved-options') || $(), String($sel.data('keep-id') || $sel.data('current-source-id') || '')); + $sel.trigger('change'); + }); + mailcow_alert_box(L('source_list_error'), 'danger'); + }); + } + + function sourceOption(id, name, host, port, authType, flow) { + var sub = (host || '?') + ':' + (port || '?') + ' · ' + (authType || ''); + if (authType === 'XOAUTH2') sub += ' (' + (flow === 'authorization_code' ? L('source_flow_badge_user') : L('source_flow_badge_app')) + ')'; + return $('').val(id).text(name || ('#' + id)) + .attr('data-subtext', sub) + .attr('data-auth-type', authType || '') + .attr('data-oauth-flow', flow || ''); + } + + // Toggle password / user1 / connect rows by the selected source's auth type + function applySourceSelection($sel) { + var $opt = $sel.find('option:selected'); + var $form = $sel.closest('form'); + var isOauth = ($opt.attr('data-auth-type') === 'XOAUTH2'); + var isAuthCode = isOauth && ($opt.attr('data-oauth-flow') === 'authorization_code'); + var isEdit = ($form.data('id') === 'editsyncjob'); + var $user1 = $form.find('input[name="user1"]'); + var prevSource = $sel.data('prev-source'); + var curSource = $sel.val() || ''; + // a token belongs to (source, mailbox, remote user): switching the source drops the connect result + if (prevSource !== undefined && prevSource !== curSource && $user1.prop('readonly')) { + $user1.prop('readonly', false).val(isEdit ? $user1.data('initial') || '' : ''); + setConnectStatus($form, '', ''); + } + $sel.data('prev-source', curSource); + $form.find('.password1-row').toggle(!isOauth); + // edit: an empty password keeps the stored one + $form.find('input[name="password1"]').prop('required', !isOauth && !isEdit); + $form.find('.imapsync-oauth-connect-row').toggle(isAuthCode); + $form.find('.imapsync-user1-row').toggle(!isAuthCode); + $user1.prop('required', !isAuthCode && !isEdit); + } + + // ===== OAuth connect (authorization_code) ===== + + var oauthPending = null; + + function setConnectStatus($form, type, text, busy) { + var $status = $form.find('.imapsync-oauth-connect-status'); + $status.attr('class', 'imapsync-oauth-connect-status ms-2' + (type ? ' text-' + type : '')).empty(); + if (busy) $status.append(''); + $status.append(document.createTextNode(text || '')); + } + + function clearPending() { + if (oauthPending && oauthPending.timer) clearInterval(oauthPending.timer); + oauthPending = null; + } + + // Local mailbox the token is stored for (C1) + function connectMailbox($form) { + var $mb = $form.find('[name="username"]'); + if ($mb.length) return $mb.val() || ''; + if ($form.data('mailbox')) return String($form.data('mailbox')); + return (typeof mailcow_cc_username !== 'undefined') ? mailcow_cc_username : ''; + } + + function startConnect($form) { + var sourceId = $form.find('select.imapsync-source-select').val(); + if (!sourceId) return; + var mailbox = connectMailbox($form); + if (!mailbox) { + setConnectStatus($form, 'danger', L('oauth_select_mailbox')); + return; + } + var url = '/syncjob-oauth?action=start&source_id=' + encodeURIComponent(sourceId) + '&mailbox=' + encodeURIComponent(mailbox); + var popup = window.open(url, 'syncjob_oauth', 'width=600,height=700'); + if (!popup || popup.closed || typeof popup.closed === 'undefined') { + setConnectStatus($form, 'danger', L('oauth_popup_blocked')); + return; + } + clearPending(); + var pending = { form: $form, sourceId: String(sourceId), mailbox: mailbox.toLowerCase() }; + oauthPending = pending; + setConnectStatus($form, 'muted', L('oauth_waiting'), true); + pending.timer = setInterval(function() { + if (!popup.closed) return; + clearInterval(pending.timer); + pending.timer = null; + // stays pending: a result message that arrives after the close still wins + if (oauthPending === pending) setConnectStatus($form, 'muted', L('oauth_window_closed')); + }, 500); + } + + function onOauthMessage(ev) { + if (ev.origin !== window.location.origin || !ev.data || ev.data.type !== 'syncjob_oauth') return; + var p = oauthPending; + if (!p) return; + // only the row that started the flow is updated + if (ev.data.ok && (String(ev.data.source_id) !== p.sourceId || String(ev.data.mailbox || '').toLowerCase() !== p.mailbox)) return; + clearPending(); + if (ev.data.ok) { + p.form.find('input[name="user1"]').val(ev.data.user1).prop('readonly', true); + setConnectStatus(p.form, 'success', L('syncjob_oauth_connected') + ': ' + ev.data.user1); + } else { + var code = String(ev.data.error || '').replace(/[^a-z_]/g, ''); + setConnectStatus(p.form, 'danger', L('oauth_error_' + code) || L('oauth_error_unknown')); + } + } + + // ===== Custom params editor ===== + + // Allowlist { name: true (value) | false (flag) | 'regex' (value) }, loaded once on demand + var cpOptions = null; + var cpLoading = null; + function cpEnsureOptions(cb) { + if (cpOptions !== null) { cb(); return; } + if (!cpLoading) { + cpLoading = $.get('/api/v1/get/syncjob_options').done(function(opts) { + cpOptions = (opts && typeof opts === 'object' && !Array.isArray(opts)) ? opts : {}; + }).fail(function() { + cpLoading = null; + mailcow_alert_box(L('custom_params_options_error'), 'danger'); + }); + } + cpLoading.done(function() { cb(); }); + } + + function cpApplyValueState($row) { + var name = $row.find('.imapsync-cp-opt').val() || ''; + var isFlag = (name !== '') && !!cpOptions && cpOptions[name] === false; + var $val = $row.find('.imapsync-cp-val'); + $val.prop('disabled', isFlag); + if (isFlag) $val.val(''); + $val.prop('required', (name !== '') && !isFlag); + $val.attr('placeholder', isFlag ? L('custom_param_no_value') : L('custom_param_value')); + } + + var cpRowSeq = 0; + function cpAddRow($editor, o, v) { + var n = ++cpRowSeq; + var $row = $( + '
' + + '' + + '' + + '' + + '
' + ); + // aria-label: label elements inside .input-group break its first/last child styling + $row.find('.imapsync-cp-opt').attr('aria-label', L('custom_param_option')); + $row.find('.imapsync-cp-val').attr('aria-label', L('custom_param_value')); + $row.find('.imapsync-cp-remove').attr({ 'title': L('custom_param_remove'), 'aria-label': L('custom_param_remove') }); + var $sel = $row.find('.imapsync-cp-opt'); + $sel.append($('').attr('value', '').text('— ' + L('custom_param_option') + ' —')); + Object.keys(cpOptions || {}).sort().forEach(function(name) { + $sel.append($('').attr('value', name).text('--' + name)); + }); + // keep a stored option that is no longer allowlisted visible (the backend rejects it on save) + if (o && !(cpOptions && (o in cpOptions))) { + $sel.append($('').attr('value', o).text('--' + o)); + } + $sel.val(o || ''); + $row.find('.imapsync-cp-val').val(v || ''); + $editor.find('.imapsync-cp-rows').append($row); + cpApplyValueState($row); + } + + function cpBuild($editor) { + var $hidden = $editor.siblings('.imapsync-cp-value'); + $editor.find('.imapsync-cp-rows').empty(); + var pairs = []; + try { pairs = JSON.parse($hidden.val() || '[]'); } catch (e) { pairs = []; } + if (!Array.isArray(pairs)) pairs = []; + pairs.forEach(function(p) { + if (p && typeof p === 'object') cpAddRow($editor, p.o, p.v); + }); + } + + function cpSerialize($editor) { + var out = []; + $editor.find('.imapsync-cp-row').each(function() { + var o = $.trim($(this).find('.imapsync-cp-opt').val() || '').replace(/^-+/, ''); + if (o === '') return; + out.push({ o: o, v: $(this).find('.imapsync-cp-val').val() || '' }); + }); + $editor.siblings('.imapsync-cp-value').val(JSON.stringify(out)); + } + + function cpInit($scope) { + var $editors = $scope.find('.imapsync-cp-editor'); + if (!$editors.length) return; + cpEnsureOptions(function() { + $editors.each(function() { cpBuild($(this)); }); + }); + } + + // ===== Source editor (add modal + edit page) ===== + + function sourceFormToggle($form) { + var isOauth = $form.find('select.imapsync-source-auth-type').val() === 'XOAUTH2'; + var isAuthCode = $form.find('select.imapsync-source-oauth-flow').val() === 'authorization_code'; + var isCc = isOauth && !isAuthCode; + $form.find('.imapsync-source-oauth-block').toggle(isOauth); + $form.find('.imapsync-source-authcode-block').toggle(isOauth && isAuthCode); + $form.find('.imapsync-source-cc-block').toggle(isCc); + // client credentials sources are forced private by the backend + $form.find('.imapsync-source-scope-block').toggle(!isCc); + $form.find('.imapsync-source-scope-cc-hint').toggle(isCc); + } + + function sourceScopeToggle($form) { + var scope = $form.find('select.imapsync-source-scope').val(); + $form.find('.imapsync-source-domains-row').toggle(scope === 'domain'); + $form.find('.imapsync-source-users-row').toggle(scope === 'user'); + if (scope === 'domain') fillScopeSelect($form.find('select.imapsync-source-domains'), 'domain/all', 'domain_name'); + if (scope === 'user') fillScopeSelect($form.find('select.imapsync-source-users'), 'mailbox/all', 'username'); + } + + // Add the selectable domains/mailboxes to a scope multiselect (keeps selected options) + function fillScopeSelect($sel, url, field) { + if (!$sel.length || $sel.prop('disabled') || $sel.data('filled')) return; + $sel.data('filled', true); + $.get('/api/v1/get/' + url).done(function(rows) { + var have = {}; + $sel.find('option').each(function() { have[$(this).val()] = true; }); + $.each(Array.isArray(rows) ? rows : [], function(_, r) { + var v = r[field]; + if (!v || have[v]) return; + $sel.append($('').val(v).text(v)); + }); + // refresh duplicates preselected entries in the button label, re-init instead + if (typeof $sel.selectpicker === 'function' && $sel.data('selectpicker')) { + var keep = $sel.val(); + $sel.selectpicker('destroy'); + refreshSelect($sel); + $sel.selectpicker('val', keep); + } else { + refreshSelect($sel); + } + }).fail(function() { + $sel.data('filled', false); + mailcow_alert_box(lang_danger.access_denied, 'danger'); + }); + } + + // Extra params must be a JSON object; checked before the form is posted + function extraParamsValid($form) { + var $f = $form.find('input[name="oauth_extra_params"]'); + if (!$f.length || $f.prop('disabled') || !$f.is(':visible')) return true; + var v = $.trim($f.val() || ''); + if (v === '') { $f.removeClass('inputMissingAttr'); return true; } + var ok = false; + try { + var o = JSON.parse(v); + ok = (o !== null && typeof o === 'object' && !Array.isArray(o)); + } catch (e) { ok = false; } + $f.toggleClass('inputMissingAttr', !ok); + if (!ok) mailcow_alert_box(lang_danger.imapsync_source_oauth_extra_invalid, 'danger'); + return ok; + } + + function copyField(id) { + var el = document.getElementById(id); + if (!el) return; + var done = function() { mailcow_alert_box(L('copied'), 'success'); }; + if (navigator.clipboard && window.isSecureContext) { + navigator.clipboard.writeText(el.value).then(done); + } else { + el.select(); + try { document.execCommand('copy'); done(); } catch (e) {} + } + } + + // ===== Wiring ===== + + function bind() { + injectStyle(); + + // custom params editor + $(document).on('change', '.imapsync-cp-opt', function() { + cpApplyValueState($(this).closest('.imapsync-cp-row')); + cpSerialize($(this).closest('.imapsync-cp-editor')); + }); + $(document).on('input', '.imapsync-cp-val', function() { + cpSerialize($(this).closest('.imapsync-cp-editor')); + }); + $(document).on('click', '.imapsync-cp-add', function() { + var $editor = $(this).closest('.imapsync-cp-editor'); + cpEnsureOptions(function() { cpAddRow($editor); }); + }); + $(document).on('click', '.imapsync-cp-remove', function() { + var $editor = $(this).closest('.imapsync-cp-editor'); + $(this).closest('.imapsync-cp-row').remove(); + cpSerialize($editor); + }); + + // job form + $(document).on('change', 'select.imapsync-source-select', function() { + applySourceSelection($(this)); + }); + // a token is bound to the local mailbox: changing it drops the connect result + $(document).on('change', 'form[data-id="add_syncjob"] select[name="username"]', function() { + var $form = $(this).closest('form'); + var $user1 = $form.find('input[name="user1"]'); + if ($user1.prop('readonly')) { + $user1.prop('readonly', false).val(''); + setConnectStatus($form, '', ''); + } + }); + $(document).on('click', '.imapsync-oauth-connect-btn', function() { + startConnect($(this).closest('form')); + }); + window.addEventListener('message', onOauthMessage); + $(document).on('click', '.imapsync-open-source-modal', function(e) { + e.preventDefault(); + var $current = $(this).closest('.modal'); + var open = function() { $('#addImapsyncSourceModal').modal('show'); }; + if ($current.length) { + $current.one('hidden.bs.modal', open).modal('hide'); + } else { + open(); + } + }); + // authorization_code: user1 comes from the provider sign-in, require it before posting + $('[data-action="add_item"][data-id="add_syncjob"], [data-action="edit_selected"][data-id="editsyncjob"]').on('click', function(e) { + var $form = $(this).closest('form'); + var $opt = $form.find('select.imapsync-source-select option:selected'); + if ($opt.attr('data-auth-type') === 'XOAUTH2' && $opt.attr('data-oauth-flow') === 'authorization_code' && + !$.trim($form.find('input[name="user1"]').val() || '')) { + e.preventDefault(); + e.stopImmediatePropagation(); + setConnectStatus($form, 'danger', lang_danger.imapsync_source_oauth_connect_required); + return false; + } + }); + + // source editor + $(document).on('change', 'select.imapsync-source-auth-type, select.imapsync-source-oauth-flow', function() { + sourceFormToggle($(this).closest('form')); + }); + $(document).on('change', 'select.imapsync-source-scope', function() { + sourceScopeToggle($(this).closest('form')); + }); + $('[data-action="add_item"][data-id="add_imapsync_source"], [data-action="edit_selected"][data-id="edit_imapsync_source"]').on('click', function(e) { + if (!extraParamsValid($(this).closest('form'))) { + e.preventDefault(); + e.stopImmediatePropagation(); + return false; + } + }); + $(document).on('click', '.imapsync-copy-btn', function(e) { + e.preventDefault(); + copyField($(this).data('copy-target')); + }); + + // modals + $(document).on('shown.bs.modal', '#addImapsyncSourceModal', function() { + var $form = $(this).find('form'); + sourceFormToggle($form); + sourceScopeToggle($form); + }); + $(document).on('shown.bs.modal', '#addSyncJobModalAdmin, #addSyncJobModal', function() { + populateSourceSelects(); + cpInit($(this)); + }); + + // edit page: forms are already rendered + $('form[data-id="edit_imapsync_source"]').each(function() { + sourceFormToggle($(this)); + sourceScopeToggle($(this)); + }); + $('form[data-id="editsyncjob"]').each(function() { + var $form = $(this); + $form.find('input[name="user1"]').data('initial', $form.find('input[name="user1"]').val()); + populateSourceSelects(); + cpInit($form); + }); + } + + $(document).ready(bind); + + return { + drawSourceTable: drawSourceTable, + jobTableStateSave: jobTableStateSave, + jobTableStateLoad: jobTableStateLoad, + prepareJobRow: prepareJobRow, + wrapCell: wrapCell, + L: L + }; +})(jQuery); diff --git a/data/web/js/site/mailbox.js b/data/web/js/site/mailbox.js index 5ad5cceab..e073650d3 100644 --- a/data/web/js/site/mailbox.js +++ b/data/web/js/site/mailbox.js @@ -2169,7 +2169,11 @@ jQuery(function($){ "tr" + "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>", language: lang_datatables, - order: [[2, 'desc']], + // keep the backend order (C4: prio desc, never run first, least recently run) + order: [], + // drop state saved before the rework (it would restore the old ID sort / column visibility) + stateSaveParams: mailcowImapsync.jobTableStateSave, + stateLoadParams: mailcowImapsync.jobTableStateLoad, initComplete: function(){ hideTableExpandCollapseBtn('#tab-syncjobs', '#sync_job_table'); }, @@ -2177,40 +2181,21 @@ jQuery(function($){ type: "GET", url: "/api/v1/get/syncjobs/all/no_log", dataSrc: function(json){ + if (!Array.isArray(json)) return []; $.each(json, function (i, item) { - item.log = '' + lang.open_logs + '' - item.user2 = escapeHtml(item.user2); if (!item.exclude > 0) { item.exclude = '-'; } else { item.exclude = '' + escapeHtml(item.exclude) + ''; } - item.server_w_port = escapeHtml(item.user1) + '@' + escapeHtml(item.host1) + ':' + escapeHtml(item.port1); + mailcowImapsync.prepareJobRow(item); + item.user2 = mailcowImapsync.wrapCell(escapeHtml(item.user2)); item.action = '
' + ' ' + lang.edit + '' + + ' ' + lang.open_logs + '' + ' ' + lang.remove + '' + '
'; item.chkbox = ''; - if (item.is_running == 1) { - item.is_running = '' + lang.running + ''; - } else { - item.is_running = '' + lang.waiting + ''; - } - if (!item.last_run > 0) { - item.last_run = lang.waiting; - } - if (item.success == null) { - item.success = '-'; - item.exit_status = ''; - } else { - item.success = ''; - } - if (lang['syncjob_'+item.exit_status]) { - item.exit_status = lang['syncjob_'+item.exit_status]; - } else if (item.success != '-') { - item.exit_status = lang.syncjob_check_log; - } - item.exit_status = item.success + ' ' + item.exit_status; }); return json; @@ -2235,40 +2220,46 @@ jQuery(function($){ responsivePriority: 2 }, { - title: 'ID', + title: mailcowImapsync.L('id'), data: 'id', responsivePriority: 3, defaultContent: '' }, { - title: lang.owner, + title: lang.syncjobs.prio, + data: 'prio', + visible: role === 'admin', + responsivePriority: 3, + defaultContent: '' + }, + { + title: lang.syncjobs.user2_label, data: 'user2', responsivePriority: 4, defaultContent: '' }, { - title: 'Server', + title: lang.syncjobs.source_server, data: 'server_w_port', + responsivePriority: 6, defaultContent: '' }, { title: lang.last_run, data: 'last_run', + responsivePriority: 9, defaultContent: '' }, { - title: lang.syncjob_last_run_result, + title: lang.syncjobs.syncjob_last_run_result, data: 'exit_status', - defaultContent: '' - }, - { - title: 'Log', - data: 'log', + responsivePriority: 8, defaultContent: '' }, { title: lang.active, data: 'active', + responsivePriority: 10, defaultContent: '', render: function (data, type) { return 1==data?'1':0==data&&'0'; @@ -2277,6 +2268,7 @@ jQuery(function($){ { title: lang.status, data: 'is_running', + responsivePriority: 7, defaultContent: '' }, { @@ -2486,6 +2478,7 @@ jQuery(function($){ onVisible("[id^=alias_table]", () => draw_alias_table()); onVisible("[id^=aliasdomain_table]", () => draw_aliasdomain_table()); onVisible("[id^=sync_job_table]", () => draw_sync_job_table()); + onVisible("[id^=imapsync_source_table]", () => mailcowImapsync.drawSourceTable(role === 'admin' || acl_data.syncjobs === 1)); onVisible("[id^=filter_table]", () => draw_filter_table()); onVisible("[id^=bcc_table]", () => draw_bcc_table()); onVisible("[id^=recipient_map_table]", () => draw_recipient_map_table()); diff --git a/data/web/js/site/user.js b/data/web/js/site/user.js index 288e7abd2..af6521f9e 100644 --- a/data/web/js/site/user.js +++ b/data/web/js/site/user.js @@ -272,22 +272,27 @@ jQuery(function($){ "tr" + "<'row'<'col-sm-12 col-md-5'i><'col-sm-12 col-md-7'p>>", language: lang_datatables, + // keep the backend order (C4: prio desc, never run first, least recently run) + order: [], + // drop state saved before the rework (it would restore the old ID sort / column visibility) + stateSaveParams: mailcowImapsync.jobTableStateSave, + stateLoadParams: mailcowImapsync.jobTableStateLoad, ajax: { type: "GET", url: '/api/v1/get/syncjobs/' + encodeURIComponent(mailcow_cc_username) + '/no_log', dataSrc: function(data){ + if (!Array.isArray(data)) return []; $.each(data, function (i, item) { - item.user1 = escapeHtml(item.user1); - item.log = '' + lang.open_logs + '' if (!item.exclude > 0) { item.exclude = '-'; } else { item.exclude = '' + escapeHtml(item.exclude) + ''; } - item.server_w_port = escapeHtml(item.user1 + '@' + item.host1 + ':' + item.port1); + mailcowImapsync.prepareJobRow(item); if (acl_data.syncjobs === 1) { item.action = '
' + ' ' + lang.edit + '' + + ' ' + lang.open_logs + '' + ' ' + lang.remove + '' + '
'; item.chkbox = ''; @@ -296,26 +301,6 @@ jQuery(function($){ item.action = '-'; item.chkbox = ''; } - if (item.is_running == 1) { - item.is_running = '' + lang.running + ''; - } else { - item.is_running = '' + lang.waiting + ''; - } - if (!item.last_run > 0) { - item.last_run = lang.waiting; - } - if (item.success == null) { - item.success = '-'; - item.exit_status = ''; - } else { - item.success = ''; - } - if (lang['syncjob_'+item.exit_status]) { - item.exit_status = lang['syncjob_'+item.exit_status]; - } else if (item.success != '-') { - item.exit_status = lang.syncjob_check_log; - } - item.exit_status = item.success + ' ' + item.exit_status; }); return data; @@ -340,41 +325,34 @@ jQuery(function($){ responsivePriority: 2 }, { - title: 'ID', + title: mailcowImapsync.L('id'), data: 'id', defaultContent: '', responsivePriority: 3 }, { - title: 'Server', + title: lang.syncjobs.source_server, data: 'server_w_port', - defaultContent: '' - }, - { - title: lang.username, - data: 'user1', defaultContent: '', - responsivePriority: 3 + responsivePriority: 4 }, { title: lang.last_run, data: 'last_run', - defaultContent: '' + defaultContent: '', + responsivePriority: 7 }, { - title: lang.syncjob_last_run_result, + title: lang.syncjobs.syncjob_last_run_result, data: 'exit_status', - defaultContent: '' - }, - { - title: 'Log', - data: 'log', - defaultContent: '' + defaultContent: '', + responsivePriority: 6 }, { title: lang.active, data: 'active', defaultContent: '', + responsivePriority: 8, render: function (data, type) { return 1==data?'':0==data&&'' } @@ -385,11 +363,6 @@ jQuery(function($){ defaultContent: '', responsivePriority: 5 }, - { - title: lang.encryption, - data: 'enc1', - defaultContent: '' - }, { title: lang.excludes, data: 'exclude', @@ -699,6 +672,7 @@ jQuery(function($){ onVisible("[id^=bl_policy_mailbox_table]", () => draw_bl_policy_mailbox_table()); onVisible("[id^=wl_policy_mailbox_table]", () => draw_wl_policy_mailbox_table()); onVisible("[id^=sync_job_table]", () => draw_sync_job_table()); + onVisible("[id^=imapsync_source_table]", () => mailcowImapsync.drawSourceTable(acl_data.syncjobs === 1)); onVisible("[id^=app_passwd_table]", () => draw_app_passwd_table()); onVisible("[id^=recent-logins]", () => last_logins('get')); }); diff --git a/data/web/json_api.php b/data/web/json_api.php index fc34b2595..71c82f1d9 100644 --- a/data/web/json_api.php +++ b/data/web/json_api.php @@ -19,7 +19,7 @@ function api_log($_data) { if ($value) { if (is_array($value)) unset($value["csrf_token"]); foreach ($value as $key => &$val) { - if(preg_match("/pass/i", $key)) { + if(preg_match("/pass|secret|oauth_extra_params/i", $key)) { $val = '*'; } } @@ -317,7 +317,10 @@ if (isset($_GET['query'])) { process_add_return(admin('add', $attr)); break; case "syncjob": - process_add_return(mailbox('add', 'syncjob', $attr)); + process_add_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->addJob($attr)); + break; + case "syncjob_source": + process_add_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->addSource($attr)); break; case "bcc": process_add_return(bcc('add', $attr)); @@ -1046,61 +1049,37 @@ if (isset($_GET['query'])) { } process_get_return($data); break; + case "syncjob_options": + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getOptions()); + break; + case "imapsync_settings": + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings()); + break; + case "syncjob_source": + switch ($object) { + case "all": + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSources()); + break; + default: + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSource($object)); + break; + } + break; case "syncjobs": switch ($object) { case "all": - $domains = mailbox('get', 'domains'); - if (!empty($domains)) { - foreach ($domains as $domain) { - $mailboxes = mailbox('get', 'mailboxes', $domain); - if (!empty($mailboxes)) { - foreach ($mailboxes as $mailbox) { - $syncjobs = mailbox('get', 'syncjobs', $mailbox); - if (!empty($syncjobs)) { - foreach ($syncjobs as $syncjob) { - if (isset($extra)) { - $details = mailbox('get', 'syncjob_details', $syncjob, explode(',', $extra)); - } - else { - $details = mailbox('get', 'syncjob_details', $syncjob); - } - if ($details) { - $data[] = $details; - } - else { - continue; - } - } - } - } - } + // Admins see every job (incl. orphaned ones), others the jobs of the mailboxes they manage + $mailboxes = array(); + if ($_SESSION['mailcow_cc_role'] != 'admin') { + foreach (mailbox('get', 'domains') ?: array() as $domain) { + $mailboxes = array_merge($mailboxes, mailbox('get', 'mailboxes', $domain) ?: array()); } - process_get_return($data); - } - else { - echo '{}'; } + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getAllJobs($extra, $mailboxes)); break; default: - $syncjobs = mailbox('get', 'syncjobs', $object); - if (!empty($syncjobs)) { - foreach ($syncjobs as $syncjob) { - if (isset($extra)) { - $details = mailbox('get', 'syncjob_details', $syncjob, explode(',', $extra)); - } - else { - $details = mailbox('get', 'syncjob_details', $syncjob); - } - if ($details) { - $data[] = $details; - } - else { - continue; - } - } - } - process_get_return($data); + process_get_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getJobs($object, $extra)); break; } break; @@ -1742,7 +1721,10 @@ if (isset($_GET['query'])) { process_delete_return(rsettings('delete', array('id' => $items))); break; case "syncjob": - process_delete_return(mailbox('delete', 'syncjob', array('id' => $items))); + process_delete_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->deleteJobs($items)); + break; + case "syncjob_source": + process_delete_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->deleteSources($items)); break; case "filter": process_delete_return(mailbox('delete', 'filter', array('id' => $items))); @@ -1968,7 +1950,27 @@ if (isset($_GET['query'])) { } break; case "syncjob": - process_edit_return(mailbox('edit', 'syncjob', array_merge(array('id' => $items), $attr))); + switch ($object) { + case "run_next": + process_edit_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->runNext($items)); + break; + default: + process_edit_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->editJobs($items, $attr)); + break; + } + break; + case "imapsync_settings": + process_edit_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->editSettings($attr)); + break; + case "syncjob_source": + switch ($object) { + case "refresh_token": + process_edit_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->refreshSourceToken($items)); + break; + default: + process_edit_return(\Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->editSources($items, $attr)); + break; + } break; case "filter": process_edit_return(mailbox('edit', 'filter', array_merge(array('id' => $items), $attr))); diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index 1f1c1f273..1849caf40 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -45,24 +45,16 @@ "app_name": "App name", "app_password": "Add app password", "app_passwd_protocols": "Allowed protocols for app password", - "automap": "Try to automap folders (\"Sent items\", \"Sent\" => \"Sent\" etc.)", "backup_mx_options": "Relay options", "bcc_dest_format": "BCC destination must be a single valid email address.
If you need to send a copy to multiple addresses, create an alias and use it here.", "comment_info": "A private comment is not visible to the user, while a public comment is shown as tooltip when hovering it in a user's overview", - "custom_params": "Custom parameters", - "custom_params_hint": "Right: --param=xy, wrong: --param xy", - "delete1": "Delete from source when completed", - "delete2": "Delete messages on destination that are not on source", - "delete2duplicates": "Delete duplicates on destination", "description": "Description", "destination": "Destination", "disable_login": "Disallow login (incoming mail is still accepted)", "domain": "Domain", "domain_matches_hostname": "Domain %s matches hostname", "domain_quota_m": "Total domain quota (MiB)", - "dry": "Simulate synchronization", "enc_method": "Encryption method", - "exclude": "Exclude objects (regex)", "full_name": "Full name", "gal": "Global Address List", "gal_info": "The GAL contains all objects of a domain and cannot be edited by any user. Free/busy information in SOGo is missing, if disabled! Restart SOGo to apply changes.", @@ -81,7 +73,6 @@ "mailbox_username": "Username (left part of an email address)", "max_aliases": "Max. possible aliases", "max_mailboxes": "Max. possible mailboxes", - "mins_interval": "Polling interval (minutes)", "multiple_bookings": "Multiple bookings", "nexthop": "Next hop", "password": "Password", @@ -101,16 +92,10 @@ "select_domain": "Please select a domain first", "sieve_desc": "Short description", "sieve_type": "Filter type", - "skipcrossduplicates": "Skip duplicate messages across folders (first come, first serve)", - "subscribeall": "Subscribe all folders", - "syncjob": "Add sync job", - "syncjob_hint": "Be aware that passwords need to be saved plain-text!", "tags": "Tags", "target_address": "Goto addresses", "target_address_info": "Full email address/es (comma-separated).", "target_domain": "Target domain", - "timeout1": "Timeout for connection to remote host", - "timeout2": "Timeout for connection to local host", "username": "Username", "validate": "Validate", "validation_success": "Validated successfully" @@ -418,7 +403,12 @@ "user_quicklink": "Hide Quicklink to User Login Page", "validate_license_now": "Validate GUID against license server", "verify": "Verify", - "yes": "✓" + "yes": "✓", + "syncjobs": "Sync jobs", + "syncjob_max_parallel": "Max. parallel sync processes", + "syncjob_max_parallel_info": "How many sync jobs may run at the same time (1-32).", + "syncjob_max_kb_per_second": "Bandwidth limit per process (KB/s)", + "syncjob_max_kb_per_second_info": "0 = unlimited. Caps every sync process in KB/s; a per-job limit (bytes/s) is capped by this. Total ≈ this × max parallel." }, "danger": { "access_denied": "Access denied or invalid form data", @@ -567,7 +557,30 @@ "validity_missing": "Please assign a period of validity", "value_missing": "Please provide all values", "version_invalid": "Version %s is invalid", - "yotp_verification_failed": "Yubico OTP verification failed: %s" + "yotp_verification_failed": "Yubico OTP verification failed: %s", + "imapsync_source_in_use": "Sync source %s is still used by a sync job and cannot be deleted", + "imapsync_source_token_refresh_failed": "OAuth token refresh for %s failed", + "imapsync_source_scope_empty": "Select at least one domain for the chosen visibility", + "imapsync_source_unavailable": "The selected sync source is unavailable", + "imapsync_source_oauth_connect_required": "Sign in with the provider before saving this sync job", + "imapsync_source_name_invalid": "Invalid sync source name", + "imapsync_source_host_invalid": "Invalid host name or IP address", + "imapsync_source_port_invalid": "Invalid port", + "imapsync_source_enc_invalid": "Invalid encryption method", + "imapsync_source_auth_invalid": "Invalid authentication method", + "imapsync_custom_param_invalid": "Custom parameters contain a disallowed imapsync option: %s", + "imapsync_source_oauth_required": "OAuth2 requires token endpoint, client ID, client secret and scope", + "imapsync_source_oauth_endpoint_invalid": "Invalid OAuth2 endpoint: an https:// URL with a valid host name is required (sources not created by an administrator need a publicly reachable host)", + "imapsync_source_oauth_extra_invalid": "OAuth2 extra parameters must be a JSON object with simple values, e.g. {\"resource\": \"https://example.com\"}", + "imapsync_source_oauth_invalid": "Invalid OAuth2 configuration", + "imapsync_max_parallel_invalid": "Parallel process count must be between 1 and 32", + "imapsync_source_description_invalid": "The description must not exceed 255 characters", + "imapsync_source_refresh_unsupported": "Manual token refresh is only available for client credentials sources", + "imapsync_custom_params_format": "Custom parameters must be sent as a list of option/value pairs", + "imapsync_custom_param_value": "Invalid value for imapsync option %s (missing value, value on a flag or disallowed value)", + "imapsync_job_running": "Sync job %s is currently running and cannot be scheduled next", + "imapsync_user1_invalid": "Invalid remote username", + "imapsync_max_kb_per_second_invalid": "Bandwidth limit must be between 0 and 10485760 KB/s" }, "datatables": { "collapse_all": "Collapse All", @@ -656,7 +669,6 @@ "app_name": "App name", "app_passwd": "App password", "app_passwd_protocols": "Allowed protocols for app password", - "automap": "Try to automap folders (\"Sent items\", \"Sent\" => \"Sent\" etc.)", "backup_mx_options": "Relay options", "bcc_dest_format": "BCC destination must be a single valid email address.
If you need to send a copy to multiple addresses, create an alias and use it here.", "client_id": "Client ID", @@ -664,9 +676,6 @@ "comment_info": "A private comment is not visible to the user, while a public comment is shown as tooltip when hovering it in a user's overview", "created_on": "Created on", "custom_attributes": "Custom attributes", - "delete1": "Delete from source when completed", - "delete2": "Delete messages on destination that are not on source", - "delete2duplicates": "Delete duplicates on destination", "delete_ays": "Please confirm the deletion process.", "description": "Description", "disable_login": "Disallow login (incoming mail is still accepted)", @@ -690,7 +699,6 @@ "dont_check_sender_acl": "Disable sender check for domain %s (+ alias domains)", "edit_alias_domain": "Edit Alias domain", "encryption": "Encryption", - "exclude": "Exclude objects (regex)", "extended_sender_acl": "External sender addresses", "extended_sender_acl_info": "A DKIM domain key should be imported, if available.
\r\n Remember to add this server to the corresponding SPF TXT record.
\r\n Whenever a domain or alias domain is added to this server, that overlaps with an external address, the external address is removed.
\r\n Use @domain.tld to allow to send as *@domain.tld.", "force_pw_update": "Force password update at next login", @@ -721,10 +729,7 @@ "max_aliases": "Max. aliases", "max_mailboxes": "Max. possible mailboxes", "max_quota": "Max. quota per mailbox (MiB)", - "maxage": "Maximum age of messages in days that will be polled from remote
(0 = ignore age)", - "maxbytespersecond": "Max. bytes per second
(0 = unlimited)", "mbox_rl_info": "This rate limit is applied on the SASL login name, it matches any \"from\" address used by the logged-in user. A mailbox rate limit overrides a domain-wide rate limit.", - "mins_interval": "Interval (min)", "mta_sts": "MTA-STS", "mta_sts_info": "MTA-STS is a standard that enforces email delivery between mail servers to use TLS with valid certificates.
It is used when DANE is not possible due to missing or unsupported DNSSEC.
Note: If the receiving domain supports DANE with DNSSEC, DANE is always preferred – MTA-STS only acts as a fallback.", "mta_sts_version": "Version", @@ -777,7 +782,6 @@ "sender_acl_info": "If mailbox user A is allowed to send as mailbox user B, the sender address is not automatically displayed as selectable \"from\" field in SOGo.
\r\n Mailbox user B needs to create a delegation in SOGo to allow mailbox user A to select their address as sender. To delegate a mailbox in SOGo, use the menu (three dots) to the right of your mailbox name in the upper left while in mail view. This behaviour does not apply to alias addresses.", "sieve_desc": "Short description", "sieve_type": "Filter type", - "skipcrossduplicates": "Skip duplicate messages across folders (first come, first serve)", "sogo_access": "Direct forwarding to SOGo", "sogo_access_info": "After logging in, the user is automatically redirected to SOGo.", "sogo_visible": "Alias is visible in SOGo", @@ -786,12 +790,8 @@ "spam_filter": "Spam filter", "spam_policy": "Add or remove items to allow-/denylist", "spam_score": "Set a custom spam score", - "subfolder2": "Sync into subfolder on destination
(empty = do not use subfolder)", - "syncjob": "Edit sync job", "target_address": "Goto address/es (comma-separated)", "target_domain": "Target domain", - "timeout1": "Timeout for connection to remote host", - "timeout2": "Timeout for connection to local host", "title": "Edit object", "unchanged_if_empty": "If unchanged leave blank", "username": "Username", @@ -1013,16 +1013,6 @@ "spam_aliases": "Temp. alias", "stats": "Statistics", "status": "Status", - "sync_jobs": "Sync jobs", - "syncjob_check_log": "Check log", - "syncjob_last_run_result": "Last run result", - "syncjob_EX_OK": "Success", - "syncjob_EXIT_CONNECTION_FAILURE": "Connection problem", - "syncjob_EXIT_TLS_FAILURE": "Problem with encrypted connection", - "syncjob_EXIT_AUTHENTICATION_FAILURE": "Authentication problem", - "syncjob_EXIT_OVERQUOTA": "Target mailbox is over quota", - "syncjob_EXIT_CONNECTION_FAILURE_HOST1": "Can't connect to remote server", - "syncjob_EXIT_AUTHENTICATION_FAILURE_USER1": "Wrong username or password", "table_size": "Table size", "table_size_show_n": "Show %s items", "target_address": "Goto address", @@ -1225,7 +1215,13 @@ "verified_fido2_login": "Verified FIDO2 login", "verified_totp_login": "Verified TOTP login", "verified_webauthn_login": "Verified WebAuthn login", - "verified_yotp_login": "Verified Yubico OTP login" + "verified_yotp_login": "Verified Yubico OTP login", + "imapsync_source_added": "Sync source %s has been added", + "imapsync_source_modified": "Sync source %s has been saved", + "imapsync_source_deleted": "Sync source %s has been deleted", + "imapsync_source_token_refreshed": "OAuth token for %s has been refreshed", + "max_parallel_saved": "Sync job settings saved", + "imapsync_run_next_done": "%s sync job(s) moved to the front of the queue" }, "tfa": { "authenticators": "Authenticators", @@ -1260,6 +1256,126 @@ "waiting_usb_register": "Waiting for USB device...

Please enter your password above and confirm your registration by tapping the button on your USB device.", "yubi_otp": "Yubico OTP authentication" }, + "syncjobs": { + "sources_panel": "Sync sources", + "sources_hint": "Sync sources define IMAP servers (and their authentication method) that sync jobs can reuse. Define each source once, then pick it from the dropdown when creating sync jobs.", + "source_add": "Add source", + "source": "Source", + "source_name": "Name", + "source_description": "Description", + "source_scope": "Visibility", + "source_scope_all": "All users", + "source_scope_domain": "Specific domains", + "source_scope_user": "Specific users", + "source_domains": "Domains", + "source_users": "Users", + "source_select": "Select source", + "source_auth_type": "Authentication", + "source_oauth_block": "OAuth2 configuration", + "source_token_status": "Token valid until", + "source_token_error": "Last refresh error", + "source_refresh_now": "Refresh token now", + "source_secret_unchanged_hint": "Leave blank to keep the stored secret.", + "source_oauth_flow": "OAuth2 flow", + "source_oauth_flow_app": "App (client credentials)", + "source_oauth_flow_user": "User login (authorization code)", + "source_flow_badge_app": "Client credentials", + "source_flow_badge_user": "User login", + "token_state_issued": "Token issued", + "token_state_pending": "Token pending", + "token_state_failed": "Token failed", + "source_authorize_endpoint": "Authorize endpoint", + "source_userinfo_endpoint": "Userinfo endpoint", + "source_redirect_uri": "Redirect URI", + "source_redirect_uri_hint": "Register this URI in the provider app (e.g. Azure). Provider sign-in only works when mailcow is opened via this host name.", + "syncjob_oauth_connect": "Sign in with the provider", + "syncjob_oauth_connected": "Connected", + "sync_jobs": "Sync jobs", + "syncjob_check_log": "Check log", + "syncjob_last_run_result": "Last run result", + "syncjob_EX_OK": "Success", + "syncjob_EXIT_CONNECTION_FAILURE": "Connection problem", + "syncjob_EXIT_TLS_FAILURE": "Problem with encrypted connection", + "syncjob_EXIT_AUTHENTICATION_FAILURE": "Authentication problem", + "syncjob_EXIT_OVERQUOTA": "Target mailbox is over quota", + "syncjob_EXIT_CONNECTION_FAILURE_HOST1": "Can't connect to remote server", + "syncjob_EXIT_AUTHENTICATION_FAILURE_USER1": "Wrong username or password", + "syncjob_OAUTH_TOKEN_NOT_READY": "OAuth token not ready (sign in again or wait for the token refresh)", + "add": "Add sync job", + "add_hint": "Be aware that passwords need to be saved plain-text!", + "edit": "Edit sync job", + "create": "Create new sync job", + "dry": "Simulate synchronization", + "subfolder2": "Sync into subfolder on destination
(empty = do not use subfolder)", + "mins_interval": "Polling interval (minutes)", + "maxage": "Maximum age of messages in days that will be polled from remote
(0 = ignore age)", + "exclude": "Exclude objects (regex)", + "automap": "Try to automap folders (\"Sent items\", \"Sent\" => \"Sent\" etc.)", + "delete1": "Delete from source when completed", + "delete2": "Delete messages on destination that are not on source", + "delete2duplicates": "Delete duplicates on destination", + "skipcrossduplicates": "Skip duplicate messages across folders (first come, first serve)", + "subscribeall": "Subscribe all folders", + "timeout1": "Timeout for connection to remote host", + "timeout2": "Timeout for connection to local host", + "maxbytespersecond": "Max. bandwidth in bytes/s
(0 = unlimited, additionally capped by the global limit per process in KB/s)", + "custom_params": "Custom parameters", + "custom_params_hint": "Add one imapsync option per row. Only allowlisted options are accepted; the value may contain any character (spaces, regex, …) and is passed as a single argument.", + "custom_param_option": "Option", + "custom_param_value": "Value", + "custom_param_no_value": "no value needed", + "custom_param_add": "Add parameter", + "prio": "Priority", + "prio_hint": "Jobs with a higher priority run first. Within the same priority, jobs that never ran come first, then the least recently run ones (0 = normal).", + "oauth_popup_title": "Sign in with the provider", + "oauth_popup_success": "Connected. You can close this window.", + "oauth_popup_failed": "Sign-in failed. You can close this window.", + "oauth_error_access_denied": "You are not allowed to manage sync jobs", + "oauth_error_mailbox_denied": "No access to the selected local mailbox", + "oauth_error_source_unavailable": "The sync source is unavailable or not configured for provider sign-in", + "oauth_error_provider_denied": "The provider refused the sign-in", + "oauth_error_invalid_state": "The sign-in session expired or is invalid, please try again", + "oauth_error_token_failed": "The provider did not issue a token (check client ID, secret and redirect URI)", + "oauth_error_no_identity": "The provider did not return an e-mail address for the signed-in account", + "user2_label": "Local mailbox (destination)", + "user1_label": "Remote username (source)", + "password1_label": "Remote password (source)", + "no_sources": "No sync source is available. Please add a sync source first.", + "source_list_error": "Sync sources could not be loaded", + "source_not_visible": "(not visible)", + "oauth_waiting": "Waiting for the provider…", + "oauth_popup_blocked": "The sign-in window was blocked. Please allow pop-ups for this site and try again.", + "oauth_select_mailbox": "Please select the local mailbox first", + "oauth_error_unknown": "Sign-in failed", + "oauth_hint": "The remote username is taken from the account used to sign in with the provider.", + "token_state_none": "Token pending", + "custom_params_legacy": "Legacy parameters (not migrated)", + "custom_params_legacy_hint": "These custom parameters of the original sync job could not be converted and are ignored. Add the options you still need with the parameter editor above.", + "custom_params_legacy_badge": "Legacy parameters", + "custom_param_remove": "Remove parameter", + "never_run": "Never", + "status_inactive": "Inactive", + "source_server": "Source server", + "id": "ID", + "source_created_by": "Created by", + "source_oauth_token_endpoint": "Token endpoint", + "source_oauth_client_id": "Client ID", + "source_oauth_client_secret": "Client secret", + "source_oauth_scope": "Scope", + "source_oauth_extra_params": "Extra parameters (JSON)", + "source_oauth_extra_params_hint": "Optional JSON object with additional parameters for the token request, e.g. {\"resource\": \"https://provider.example.com\"}", + "source_scope_cc_hint": "Sources using the client credentials flow are always private to their creator.", + "source_readonly_hint": "This sync source is shared with you and can only be viewed.", + "copy": "Copy", + "copied": "Copied to clipboard", + "source_view": "Show", + "custom_params_options_error": "The list of allowed imapsync options could not be loaded", + "source_missing": "No sync source is assigned to this job. Please select a source.", + "oauth_window_closed": "The sign-in window was closed before a result arrived. If you completed the sign-in, reload the page or try again.", + "custom_params_legacy_badge_title": "Legacy custom parameters that could not be converted are ignored. Open the edit page for details.", + "status_waiting": "Waiting", + "status_running": "Running" + }, "user": { "action": "Action", "active": "Active", @@ -1290,7 +1406,6 @@ "clear_recent_successful_connections": "Clear seen successful connections", "client_configuration": "Show configuration guides for email clients and smartphones", "create_app_passwd": "Create app password", - "create_syncjob": "Create new sync job", "created_on": "Created on", "daily": "Daily", "day": "day", @@ -1400,16 +1515,6 @@ "spamfilter_wl_desc": "Allowlisted email addresses are programmed to never classify as spam. Wildcards may be used. A filter is only applied to direct aliases (aliases with a single target mailbox) excluding catch-all aliases and a mailbox itself.", "spamfilter_yellow": "Yellow: this message may be spam, will be tagged as spam and moved to your junk folder", "status": "Status", - "sync_jobs": "Sync jobs", - "syncjob_check_log": "Check log", - "syncjob_last_run_result": "Last run result", - "syncjob_EX_OK": "Success", - "syncjob_EXIT_CONNECTION_FAILURE": "Connection problem", - "syncjob_EXIT_TLS_FAILURE": "Problem with encrypted connection", - "syncjob_EXIT_AUTHENTICATION_FAILURE": "Authentication problem", - "syncjob_EXIT_OVERQUOTA": "Target mailbox is over quota", - "syncjob_EXIT_CONNECTION_FAILURE_HOST1": "Can't connect to remote server", - "syncjob_EXIT_AUTHENTICATION_FAILURE_USER1": "Wrong username or password", "tag_handling": "Set handling for tagged mail", "tag_help_example": "Example for a tagged email address: me+Facebook@example.org", "tag_help_explain": "In subfolder: a new subfolder named after the tag will be created below INBOX (\"INBOX/Facebook\").
\r\nIn subject: the tags name will be prepended to the mails subject, example: \"[Facebook] My News\".", diff --git a/data/web/syncjob-oauth.php b/data/web/syncjob-oauth.php new file mode 100644 index 000000000..1046dc97a --- /dev/null +++ b/data/web/syncjob-oauth.php @@ -0,0 +1,103 @@ + redirect the logged-in user to the provider consent screen +// (callback, ?code&state) -> exchange code, store the token for mailbox M, message the opener +// The token is bound to the local mailbox M (owner) and the authenticated remote identity (syncjob user1). +// The redirect URI is always https://MAILCOW_HOSTNAME/syncjob-oauth, so the UI must be used on that host. +// postMessage only carries short error codes, the UI translates them: +// access_denied, mailbox_denied, source_unavailable, provider_denied, invalid_state, token_failed, no_identity +require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php'; + +function syncjob_oauth_finish($ok, $payload) { + // Renders a tiny page that hands the result back to the opener modal and closes the popup. + global $lang; + header('Content-Type: text/html; charset=utf-8'); + header('Cache-Control: no-store'); + $json = json_encode(array_merge(array('type' => 'syncjob_oauth', 'ok' => $ok), $payload), + JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT); + $title = $lang['syncjobs']['oauth_popup_title'] ?? 'Sign in with the provider'; + $text = $ok + ? ($lang['syncjobs']['oauth_popup_success'] ?? 'Connected. You can close this window.') + : ($lang['syncjobs']['oauth_popup_failed'] ?? 'Authentication failed. You can close this window.'); + echo '' . htmlspecialchars($title) . '' + . '

' . htmlspecialchars($text) . '

'; + exit; +} + +// Must be an authenticated session with the syncjobs ACL (user_acl / da_acl; always set for admins). +if (empty($_SESSION['mailcow_cc_username']) || empty($_SESSION['mailcow_cc_role'])) { + header('Location: /'); + exit; +} +$context = \Mailcow\Core\Context::fromSession($_SESSION); +$imapsync = \Mailcow\Imapsync\ImapsyncModule::fromGlobals($context); +if (!$context->hasAcl('syncjobs')) { + syncjob_oauth_finish(false, array('error' => 'access_denied')); +} + +$action = isset($_GET['action']) ? $_GET['action'] : ((isset($_GET['code']) || isset($_GET['error'])) ? 'callback' : ''); + +if ($action == 'start') { + $source_id = intval($_GET['source_id'] ?? 0); + // C1: the token is stored for this local mailbox; default is the own mailbox (role user) + $mailbox = (isset($_GET['mailbox']) && is_string($_GET['mailbox']) && $_GET['mailbox'] !== '') + ? strtolower(trim($_GET['mailbox'])) : $_SESSION['mailcow_cc_username']; + if (!filter_var($mailbox, FILTER_VALIDATE_EMAIL) || !$context->canAccessMailbox($mailbox)) { + syncjob_oauth_finish(false, array('error' => 'mailbox_denied')); + } + // findConnectable enforces visibility for the current session + $state = bin2hex(random_bytes(16)); + $source = $imapsync->sources->findConnectable($source_id); + $url = $source ? $imapsync->oauth->authorizeUrl($source, $state) : null; + if ($url === null) { + syncjob_oauth_finish(false, array('error' => 'source_unavailable')); + } + $_SESSION['syncjob_oauth2state'] = $state; + $_SESSION['syncjob_oauth_source_id'] = $source_id; + $_SESSION['syncjob_oauth_mailbox'] = $mailbox; + header('Location: ' . $url); + exit; +} + +// --- callback --- +// CSRF: state must match the one stored at 'start' +$state_ok = !empty($_GET['state']) && is_string($_GET['state']) + && hash_equals((string)($_SESSION['syncjob_oauth2state'] ?? ''), $_GET['state']); +$source_id = intval($_SESSION['syncjob_oauth_source_id'] ?? 0); +$mailbox = (string)($_SESSION['syncjob_oauth_mailbox'] ?? ''); +if ($state_ok) { + unset($_SESSION['syncjob_oauth2state'], $_SESSION['syncjob_oauth_source_id'], $_SESSION['syncjob_oauth_mailbox']); +} +if (isset($_GET['error'])) { + // Provider error codes are not passed through (only a fixed code) + syncjob_oauth_finish(false, array('error' => 'provider_denied')); +} +if (!$state_ok || empty($_GET['code']) || !is_string($_GET['code']) || $source_id <= 0 || $mailbox === '') { + syncjob_oauth_finish(false, array('error' => 'invalid_state')); +} +// Re-check access to the owner mailbox (session could have changed in between) +if (!$context->canAccessMailbox($mailbox)) { + syncjob_oauth_finish(false, array('error' => 'mailbox_denied')); +} + +$source = $imapsync->sources->findConnectable($source_id); +if (!$source) { + syncjob_oauth_finish(false, array('error' => 'source_unavailable')); +} + +// C1: the token is bound to the local owner mailbox and the remote identity (id_token claims, else userinfo). +// Sanitized details go to the php-fpm log only, the opener gets a fixed code. +try { + $user1 = $imapsync->oauth->connectUser($source, $mailbox, $_GET['code']); +} +catch (\Mailcow\Core\UserError $e) { + if ($e->args) { + error_log('syncjob-oauth: ' . $e->args[0]); + } + syncjob_oauth_finish(false, array('error' => $e->key)); +} +syncjob_oauth_finish(true, array('source_id' => $source_id, 'mailbox' => $mailbox, 'user1' => $user1)); diff --git a/data/web/templates/admin.twig b/data/web/templates/admin.twig index 651107ccf..0c487c4ea 100644 --- a/data/web/templates/admin.twig +++ b/data/web/templates/admin.twig @@ -22,6 +22,7 @@
    • +
    • @@ -51,6 +52,7 @@ {% include 'admin/tab-config-f2b.twig' %} {% include 'admin/tab-config-quarantine.twig' %} {% include 'admin/tab-config-quota.twig' %} + {% include 'admin/tab-config-syncjob.twig' %} {% include 'admin/tab-config-rsettings.twig' %} {% include 'admin/tab-config-customize.twig' %} {% include 'admin/tab-config-password-settings.twig' %} diff --git a/data/web/templates/admin/tab-config-syncjob.twig b/data/web/templates/admin/tab-config-syncjob.twig new file mode 100644 index 000000000..21479d48e --- /dev/null +++ b/data/web/templates/admin/tab-config-syncjob.twig @@ -0,0 +1,35 @@ +
    +
    +
    + + {{ lang.admin.syncjobs }} +
    +
    +
    +
    +
    + + + {{ lang.admin.syncjob_max_parallel_info }} +
    +
    + + + {{ lang.admin.syncjob_max_kb_per_second_info }} +
    +
    +
    +
    + {{ lang.user.save_changes }} +
    +
    +
    +
    +
    +
    diff --git a/data/web/templates/edit.twig b/data/web/templates/edit.twig index 375018c66..a5c2cc154 100644 --- a/data/web/templates/edit.twig +++ b/data/web/templates/edit.twig @@ -26,6 +26,7 @@ var lang_user = {{ lang_user|raw }}; var lang_admin = {{ lang_admin|raw }}; var lang_datatables = {{ lang_datatables|raw }}; + var lang_syncjobs = {{ lang_syncjobs|raw }}; var csrf_token = '{{ csrf_token }}'; var pagination_size = Math.trunc('{{ pagination_size }}'); var table_for_domain = '{{ domain }}'; diff --git a/data/web/templates/edit/imapsync_source.twig b/data/web/templates/edit/imapsync_source.twig new file mode 100644 index 000000000..d6ad9d601 --- /dev/null +++ b/data/web/templates/edit/imapsync_source.twig @@ -0,0 +1,233 @@ +{% extends 'edit.twig' %} + +{% block inner_content %} +{% if result %} + {# XOAUTH2 sources are managed by admins and domain admins only #} + {% set ro = not result.can_edit or (mailcow_cc_role == 'user' and result.auth_type == 'XOAUTH2') %} + {% set show_oauth = mailcow_cc_role != 'user' or result.auth_type == 'XOAUTH2' %} + {% set is_cc = result.auth_type == 'XOAUTH2' and result.oauth_flow != 'authorization_code' %} +

    {{ lang.syncjobs.source }}: {{ result.name }}

    + {% if ro %} + + {% endif %} +
    + +
    + +
    + +
    +
    +
    + +
    + +
    +
    + {% if result.created_by %} +
    + +
    + +
    +
    + {% endif %} + {% if (mailcow_cc_role == 'admin' or mailcow_cc_role == 'domainadmin') and not ro %} + + + {% else %} +
    + +
    + {% set scope_targets = result.scope == 'domain' ? result.domains : (result.scope == 'user' ? result.users : []) %} + +
    +
    + {% endif %} +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    + {% if show_oauth %} +
    + {{ lang.syncjobs.source_oauth_block }} +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    + +
    + +
    +
    +
    + +
    +
    + + +
    + {% if not ro %} + {{ lang.syncjobs.source_redirect_uri_hint }} + {% endif %} +
    +
    +
    +
    + +
    + +
    +
    + {% if not ro %} +
    + +
    + + {{ lang.syncjobs.source_secret_unchanged_hint }} +
    +
    + {% endif %} +
    + +
    + +
    +
    + {% if not ro %} +
    + +
    + + {{ lang.syncjobs.source_oauth_extra_params_hint }} +
    +
    + {% endif %} + {% if is_cc %} + {# token state of the stored client credentials token (authorization_code tokens live per sync job) #} +
    +
    +

    + {% if result.oauth_last_refresh_error %} + {{ lang.syncjobs.token_state_failed }} + {% elseif result.has_access_token %} + {{ lang.syncjobs.token_state_issued }} + {% if result.oauth_token_expires %}{{ lang.syncjobs.source_token_status }}: {{ result.oauth_token_expires|date('Y-m-d H:i:s') }}{% endif %} + {% else %} + {{ lang.syncjobs.token_state_none }} + {% endif %} +

    + {% if result.oauth_last_refresh_error %} +

    {{ lang.syncjobs.source_token_error }}: {{ result.oauth_last_refresh_error }}

    + {% endif %} + {% if not ro %} + {# own data-id: must not harvest/post the form fields #} + + {% endif %} +
    +
    + {% endif %} +
    + {% endif %} +
    +
    +
    + +
    +
    +
    + {% if not ro %} +
    +
    + +
    +
    + {% endif %} +
    +{% else %} + {{ parent() }} +{% endif %} +{% endblock %} diff --git a/data/web/templates/edit/syncjob.twig b/data/web/templates/edit/syncjob.twig index 801009849..8b958079b 100644 --- a/data/web/templates/edit/syncjob.twig +++ b/data/web/templates/edit/syncjob.twig @@ -2,8 +2,8 @@ {% block inner_content %} {% if result %} -

    {{ lang.edit.syncjob }}

    -
    +

    {{ lang.syncjobs.edit }}

    + @@ -13,139 +13,180 @@
    - +
    - +
    - -
    - -
    -
    -
    - -
    - -
    -
    -
    - -
    - -
    -
    -
    - -
    - + {% if result.source_id %} + + {% endif %} + {% if not result.source_id %} + + {% endif %} +
    {{ lang.syncjobs.no_sources }}
    +
    +
    +
    + +
    + +
    +
    +
    +
    + + {% if result.user_token_exists == 1 and not result.user_token_error %} + {{ lang.syncjobs.syncjob_oauth_connected }}: {{ result.user1 }} + {% elseif result.user_token_error %} + {{ lang.syncjobs.token_state_failed }}: {{ result.user1 }} + {% else %} + {{ lang.syncjobs.token_state_pending }} + {% endif %} +
    {{ lang.syncjobs.oauth_hint }}
    +
    +
    +
    + +
    + + {{ lang.edit.unchanged_if_empty }}
    - +
    - + 1-43800
    + {% if mailcow_cc_role == 'admin' %}
    - + +
    + + {{ lang.syncjobs.prio_hint }} +
    +
    + {% endif %} +
    +
    - +
    0-32000
    - +
    0-125000000
    - +
    1-32000
    - +
    1-32000
    - +
    - +
    - - {{ lang.add.custom_params_hint }} +
    +
    + +
    + + {{ lang.syncjobs.custom_params_hint }}
    + {% if result.custom_params_legacy %} +
    + +
    +
    {{ lang.syncjobs.custom_params_legacy_hint }}
    + +
    +
    + {% endif %}
    - +
    - +
    - +
    - +
    - +
    - +
    - +
    diff --git a/data/web/templates/mailbox.twig b/data/web/templates/mailbox.twig index 1265d00cc..f330a76b9 100644 --- a/data/web/templates/mailbox.twig +++ b/data/web/templates/mailbox.twig @@ -33,7 +33,7 @@
    • - + @@ -67,6 +67,7 @@