mirror of
https://github.com/mailcow/mailcow-dockerized.git
synced 2026-10-24 19:48:54 +00:00
[Web][Dovecot] Rework imapsync: sync sources, OAuth2 (XOAUTH2), worker pool, Python runner
This commit is contained in:
39 files changed
+3582
-1358
No files matched your search
@@ -68,7 +68,6 @@ RUN addgroup -g 5000 vmail \
|
||||
perl-package-stash-xs \
|
||||
perl-par-packer \
|
||||
perl-parse-recdescent \
|
||||
perl-lockfile-simple \
|
||||
libproc2 \
|
||||
perl-readonly \
|
||||
perl-regexp-common \
|
||||
@@ -84,7 +83,6 @@ RUN addgroup -g 5000 vmail \
|
||||
perl-test-warn \
|
||||
perl-try-tiny \
|
||||
perl-unicode-string \
|
||||
perl-proc-processtable \
|
||||
perl-app-cpanminus \
|
||||
procps \
|
||||
python3 \
|
||||
@@ -120,7 +118,7 @@ COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
|
||||
COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
|
||||
COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
|
||||
COPY imapsync /usr/local/bin/imapsync
|
||||
COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
|
||||
COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
|
||||
COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
|
||||
COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
|
||||
COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham
|
||||
|
||||
@@ -279,7 +279,7 @@ chmod g+rw /dev/console
|
||||
chown root:tty /dev/console
|
||||
chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
|
||||
/usr/lib/dovecot/sieve/rspamd-pipe-spam \
|
||||
/usr/local/bin/imapsync_runner.pl \
|
||||
/usr/local/bin/imapsync_runner.py \
|
||||
/usr/local/bin/imapsync \
|
||||
/usr/local/bin/trim_logs.sh \
|
||||
/usr/local/bin/sa-rules.sh \
|
||||
|
||||
@@ -1,196 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use DBI;
|
||||
use LockFile::Simple qw(lock trylock unlock);
|
||||
use Proc::ProcessTable;
|
||||
use Data::Dumper qw(Dumper);
|
||||
use IPC::Run 'run';
|
||||
use File::Temp;
|
||||
use Try::Tiny;
|
||||
use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT);
|
||||
|
||||
sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
|
||||
my $t = Proc::ProcessTable->new;
|
||||
my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table};
|
||||
if ($imapsync_running ge 1)
|
||||
{
|
||||
print "imapsync is active, exiting...";
|
||||
exit;
|
||||
}
|
||||
|
||||
sub qqw($) {
|
||||
my @params = ();
|
||||
my @values = split(/(?=--)/, $_[0]);
|
||||
foreach my $val (@values) {
|
||||
my @tmpparam = split(/ /, $val, 2);
|
||||
foreach my $tmpval (@tmpparam) {
|
||||
if ($tmpval ne '') {
|
||||
push @params, $tmpval;
|
||||
}
|
||||
}
|
||||
}
|
||||
foreach my $val (@params) {
|
||||
$val=trim($val);
|
||||
}
|
||||
return @params;
|
||||
}
|
||||
|
||||
$run_dir="/tmp";
|
||||
$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
|
||||
$lock_file = $run_dir . "/imapsync_busy";
|
||||
$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
|
||||
$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
|
||||
$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
|
||||
mysql_auto_reconnect => 1,
|
||||
mysql_enable_utf8mb4 => 1
|
||||
});
|
||||
$dbh->do("UPDATE imapsync SET is_running = 0");
|
||||
|
||||
sub sig_handler {
|
||||
# Send die to force exception in "run"
|
||||
die "sig_handler received signal, preparing to exit...\n";
|
||||
};
|
||||
|
||||
open my $file, '<', "/etc/sogo/sieve.creds";
|
||||
my $creds = <$file>;
|
||||
close $file;
|
||||
my ($master_user, $master_pass) = split /:/, $creds;
|
||||
my $sth = $dbh->prepare("SELECT id,
|
||||
user1,
|
||||
user2,
|
||||
host1,
|
||||
authmech1,
|
||||
password1,
|
||||
exclude,
|
||||
port1,
|
||||
enc1,
|
||||
delete2duplicates,
|
||||
maxage,
|
||||
subfolder2,
|
||||
delete1,
|
||||
delete2,
|
||||
automap,
|
||||
skipcrossduplicates,
|
||||
maxbytespersecond,
|
||||
custom_params,
|
||||
subscribeall,
|
||||
timeout1,
|
||||
timeout2,
|
||||
dry
|
||||
FROM imapsync
|
||||
WHERE active = 1
|
||||
AND is_running = 0
|
||||
AND (
|
||||
UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
|
||||
OR
|
||||
last_run IS NULL)
|
||||
ORDER BY last_run");
|
||||
|
||||
$sth->execute();
|
||||
my $row;
|
||||
|
||||
while ($row = $sth->fetchrow_arrayref()) {
|
||||
|
||||
$id = @$row[0];
|
||||
$user1 = @$row[1];
|
||||
$user2 = @$row[2];
|
||||
$host1 = @$row[3];
|
||||
$authmech1 = @$row[4];
|
||||
$password1 = @$row[5];
|
||||
$exclude = @$row[6];
|
||||
$port1 = @$row[7];
|
||||
$enc1 = @$row[8];
|
||||
$delete2duplicates = @$row[9];
|
||||
$maxage = @$row[10];
|
||||
$subfolder2 = @$row[11];
|
||||
$delete1 = @$row[12];
|
||||
$delete2 = @$row[13];
|
||||
$automap = @$row[14];
|
||||
$skipcrossduplicates = @$row[15];
|
||||
$maxbytespersecond = @$row[16];
|
||||
$custom_params = @$row[17];
|
||||
$subscribeall = @$row[18];
|
||||
$timeout1 = @$row[19];
|
||||
$timeout2 = @$row[20];
|
||||
$dry = @$row[21];
|
||||
|
||||
if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
|
||||
|
||||
my $template = $run_dir . '/imapsync.XXXXXXX';
|
||||
my $passfile1 = File::Temp->new(TEMPLATE => $template);
|
||||
my $passfile2 = File::Temp->new(TEMPLATE => $template);
|
||||
|
||||
binmode( $passfile1, ":utf8" );
|
||||
|
||||
print $passfile1 "$password1\n";
|
||||
print $passfile2 trim($master_pass) . "\n";
|
||||
|
||||
my @custom_params_a = qqw($custom_params);
|
||||
my $custom_params_ref = \@custom_params_a;
|
||||
|
||||
my $generated_cmds = [ "/usr/local/bin/imapsync",
|
||||
"--tmpdir", "/tmp",
|
||||
"--nofoldersizes",
|
||||
"--addheader",
|
||||
($timeout1 le "0" ? () : ('--timeout1', $timeout1)),
|
||||
($timeout2 le "0" ? () : ('--timeout2', $timeout2)),
|
||||
($exclude eq "" ? () : ("--exclude", $exclude)),
|
||||
($subfolder2 eq "" ? () : ('--subfolder2', $subfolder2)),
|
||||
($maxage eq "0" ? () : ('--maxage', $maxage)),
|
||||
($maxbytespersecond eq "0" ? () : ('--maxbytespersecond', $maxbytespersecond)),
|
||||
($delete2duplicates ne "1" ? () : ('--delete2duplicates')),
|
||||
($subscribeall ne "1" ? () : ('--subscribeall')),
|
||||
($delete1 ne "1" ? () : ('--delete')),
|
||||
($delete2 ne "1" ? () : ('--delete2')),
|
||||
($automap ne "1" ? () : ('--automap')),
|
||||
($skipcrossduplicates ne "1" ? () : ('--skipcrossduplicates')),
|
||||
(!defined($enc1) ? () : ($enc1)),
|
||||
"--host1", $host1,
|
||||
"--user1", $user1,
|
||||
"--passfile1", $passfile1->filename,
|
||||
"--port1", $port1,
|
||||
"--host2", "localhost",
|
||||
"--user2", $user2 . '*' . trim($master_user),
|
||||
"--passfile2", $passfile2->filename,
|
||||
($dry eq "1" ? ('--dry') : ()),
|
||||
'--no-modulesversion',
|
||||
'--noreleasecheck'];
|
||||
|
||||
try {
|
||||
$is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
|
||||
$is_running->bind_param( 1, ${id} );
|
||||
$is_running->execute();
|
||||
|
||||
run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
|
||||
|
||||
# check exit code and status
|
||||
($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
|
||||
|
||||
$success = 0;
|
||||
if (defined $exit_code && $exit_code == 0) {
|
||||
$success = 1;
|
||||
}
|
||||
|
||||
$update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
|
||||
$update->bind_param( 1, ${stdout} );
|
||||
$update->bind_param( 2, ${success} );
|
||||
$update->bind_param( 3, ${exit_status} );
|
||||
$update->bind_param( 4, ${id} );
|
||||
$update->execute();
|
||||
} catch {
|
||||
$update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
|
||||
$update->bind_param( 1, ${id} );
|
||||
$update->execute();
|
||||
} finally {
|
||||
$update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
|
||||
$update->bind_param( 1, ${id} );
|
||||
$update->execute();
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
$sth->finish();
|
||||
$dbh->disconnect();
|
||||
|
||||
$lockmgr->unlock($lock_file);
|
||||
Executable
+274
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import MySQLdb
|
||||
import MySQLdb.cursors
|
||||
|
||||
run_dir = '/tmp'
|
||||
# Max size of imapsync output stored in returned_text (the tail holds the summary)
|
||||
max_output = 16 * 1024 * 1024
|
||||
lock_file = run_dir + '/imapsync_busy.lock'
|
||||
|
||||
# Next due job: higher prio first, never-run jobs first, then oldest run
|
||||
next_sql = """SELECT i.id, i.user1, i.user2, i.password1, i.exclude, i.delete2duplicates, i.maxage,
|
||||
i.subfolder2, i.delete1, i.delete2, i.automap, i.skipcrossduplicates, i.maxbytespersecond, i.custom_params,
|
||||
i.subscribeall, i.timeout1, i.timeout2, i.dry, i.source_id,
|
||||
s.host1, s.port1, s.enc1, s.auth_type, s.oauth_flow, s.oauth_access_token, s.oauth_token_expires
|
||||
FROM imapsync i JOIN imapsync_source s ON i.source_id = s.id
|
||||
WHERE i.active = 1
|
||||
AND s.active = 1
|
||||
AND i.is_running = 0
|
||||
AND (UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(i.last_run) > i.mins_interval * 60 OR i.last_run IS NULL)
|
||||
ORDER BY i.prio DESC, i.last_run IS NOT NULL, i.last_run ASC, i.id ASC
|
||||
LIMIT 1"""
|
||||
# Atomic claim, a job is only started when this flips is_running 0 -> 1
|
||||
claim_sql = """UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL
|
||||
WHERE id = %s AND is_running = 0 AND active = 1"""
|
||||
# Token of authorization_code sources: source + local mailbox (user2) + remote user (user1)
|
||||
token_sql = """SELECT access_token, token_expires FROM imapsync_source_oauth_token
|
||||
WHERE source_id = %s AND owner = %s AND username = %s"""
|
||||
|
||||
conn = None
|
||||
# Popen => job of running workers
|
||||
running = {}
|
||||
stopping = False
|
||||
|
||||
def num(value):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
def unlink(files):
|
||||
for path in files:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def imapsync_running():
|
||||
for pid in filter(str.isdigit, os.listdir('/proc')):
|
||||
try:
|
||||
with open('/proc/' + pid + '/cmdline', 'rb') as f:
|
||||
cmdline = f.read().replace(b'\0', b' ')
|
||||
except OSError:
|
||||
continue
|
||||
if re.search(rb'imapsync\s', cmdline, re.I):
|
||||
return True
|
||||
return False
|
||||
|
||||
def connect():
|
||||
return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
|
||||
passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
|
||||
|
||||
# Reconnect when the server dropped the connection (syncs can outlast wait_timeout)
|
||||
def query(sql, args=None):
|
||||
global conn
|
||||
try:
|
||||
conn.ping()
|
||||
except MySQLdb.Error:
|
||||
conn = connect()
|
||||
cur = conn.cursor(MySQLdb.cursors.DictCursor)
|
||||
cur.execute(sql, args)
|
||||
return cur
|
||||
|
||||
# Store a result and release the job
|
||||
def finish_job(job_id, text, success, exit_status):
|
||||
try:
|
||||
query("UPDATE imapsync SET returned_text = %s, success = %s, exit_status = %s, last_run = NOW(), is_running = 0 WHERE id = %s",
|
||||
(text, success, exit_status, job_id))
|
||||
except MySQLdb.Error:
|
||||
query("UPDATE imapsync SET success = 0, last_run = NOW(), is_running = 0 WHERE id = %s", (job_id,))
|
||||
|
||||
# Max concurrent imapsync processes and global per-process bandwidth cap in bytes/s (0 = off),
|
||||
# re-read before every claim round so changes apply to long runs
|
||||
def read_settings():
|
||||
settings = {r['name']: r['value'] for r in query("SELECT name, value FROM imapsync_settings")}
|
||||
return max(num(settings.get('max_parallel')), 1), max(num(settings.get('max_bytes_per_second')), 0)
|
||||
|
||||
# Build temp files and argv for a claimed job. Returns None when the job was finished without running imapsync.
|
||||
def prepare_job(j, global_bw, files):
|
||||
def tmpfile(content):
|
||||
fd, path = tempfile.mkstemp(prefix='imapsync.', dir=run_dir)
|
||||
files.append(path)
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
return path
|
||||
|
||||
auth_type = j['auth_type'] or 'PLAIN'
|
||||
if auth_type == 'XOAUTH2':
|
||||
# authorization_code: token of this local mailbox + remote user, client_credentials: shared source token
|
||||
token, expires = j['oauth_access_token'], j['oauth_token_expires']
|
||||
if j['oauth_flow'] == 'authorization_code':
|
||||
row = query(token_sql, (j['source_id'], j['user2'], j['user1'])).fetchone() or {}
|
||||
token, expires = row.get('access_token'), row.get('token_expires')
|
||||
# Skip sync if the cached token is missing or about to expire (<2 min)
|
||||
if not token or expires is None or expires - time.time() < 120:
|
||||
finish_job(j['id'], "OAuth access token for the configured source is missing or expires too soon; refresh cron will retry.",
|
||||
0, 'OAUTH_TOKEN_NOT_READY')
|
||||
return None
|
||||
auth_args = ['--authmech1', 'XOAUTH2', '--oauthaccesstoken1', tmpfile(token + '\n')]
|
||||
else:
|
||||
auth_args = ['--passfile1', tmpfile(j['password1'] + '\n')]
|
||||
# imapsync auto-picks PLAIN, only pass --authmech1 for LOGIN/CRAM-MD5
|
||||
if auth_type != 'PLAIN':
|
||||
auth_args += ['--authmech1', auth_type]
|
||||
|
||||
# Custom params: JSON array of {o,v}, validated by the backend on write. One argv token per pair,
|
||||
# "--opt" or "--opt=value" (value glued to its option, no shell)
|
||||
custom_params = []
|
||||
try:
|
||||
pairs = json.loads(j['custom_params'] or '[]')
|
||||
except ValueError:
|
||||
pairs = []
|
||||
for p in pairs if isinstance(pairs, list) else []:
|
||||
if not isinstance(p, dict) or not isinstance(p.get('o'), str) or not re.fullmatch(r'[a-z0-9]+', p['o']):
|
||||
continue
|
||||
value = '' if p.get('v') is None else str(p['v'])
|
||||
custom_params.append('--' + p['o'] if value == '' else '--' + p['o'] + '=' + value)
|
||||
|
||||
# Effective per-process bandwidth cap (bytes/s): per-job value, capped by the global limit
|
||||
eff_bw = num(j['maxbytespersecond'])
|
||||
if global_bw > 0 and (eff_bw <= 0 or eff_bw > global_bw):
|
||||
eff_bw = global_bw
|
||||
|
||||
cmd = ['/usr/local/bin/imapsync', '--tmpdir', '/tmp', '--nofoldersizes', '--addheader']
|
||||
if num(j['timeout1']) > 0:
|
||||
cmd += ['--timeout1', str(j['timeout1'])]
|
||||
if num(j['timeout2']) > 0:
|
||||
cmd += ['--timeout2', str(j['timeout2'])]
|
||||
if j['exclude']:
|
||||
cmd += ['--exclude', j['exclude']]
|
||||
if j['subfolder2']:
|
||||
cmd += ['--subfolder2', j['subfolder2']]
|
||||
if num(j['maxage']) != 0:
|
||||
cmd += ['--maxage', str(j['maxage'])]
|
||||
if eff_bw > 0:
|
||||
cmd += ['--maxbytespersecond', str(eff_bw)]
|
||||
for col, opt in (('delete2duplicates', '--delete2duplicates'), ('subscribeall', '--subscribeall'),
|
||||
('delete1', '--delete'), ('delete2', '--delete2'), ('automap', '--automap'),
|
||||
('skipcrossduplicates', '--skipcrossduplicates')):
|
||||
if j[col] == 1:
|
||||
cmd.append(opt)
|
||||
enc1 = {'TLS': '--tls1', 'SSL': '--ssl1'}.get(j['enc1'])
|
||||
if enc1:
|
||||
cmd.append(enc1)
|
||||
cmd += ['--host1', j['host1'], '--user1', j['user1']] + auth_args + ['--port1', str(j['port1'])]
|
||||
cmd += ['--host2', 'localhost', '--user2', j['user2'] + '*' + master_user, '--passfile2', tmpfile(master_pass + '\n')]
|
||||
if j['dry'] == 1:
|
||||
cmd.append('--dry')
|
||||
cmd += ['--no-modulesversion', '--noreleasecheck'] + custom_params
|
||||
|
||||
return {'id': j['id'], 'files': files, 'outfile': tmpfile(''), 'cmd': cmd}
|
||||
|
||||
# Worker: imapsync with output to a file, no inherited fds (DB socket, lock)
|
||||
def start_job(job):
|
||||
with open(job['outfile'], 'wb') as out:
|
||||
return subprocess.Popen(job['cmd'], stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
|
||||
|
||||
# Store the result of a finished worker and remove its temp files
|
||||
def reap_job(job):
|
||||
stdout = b''
|
||||
try:
|
||||
with open(job['outfile'], 'rb') as f:
|
||||
size = os.fstat(f.fileno()).st_size
|
||||
if size > max_output:
|
||||
f.seek(size - max_output)
|
||||
stdout = b'[output truncated, last %d bytes]\n' % max_output
|
||||
stdout += f.read()
|
||||
except OSError:
|
||||
pass
|
||||
unlink(job['files'])
|
||||
text = stdout.decode('utf-8', 'replace')
|
||||
# Last match: the echoed command line at the top may contain user controlled values
|
||||
m = re.findall(r'Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)', text)
|
||||
exit_code, exit_status = m[-1] if m else (None, None)
|
||||
if exit_code is None:
|
||||
text = 'Could not start or finish imapsync\n' + text
|
||||
finish_job(job['id'], text, 1 if exit_code is not None and int(exit_code) == 0 else 0, exit_status)
|
||||
|
||||
# Stop claiming new jobs, pass the signal on to the running imapsync processes
|
||||
def stop(signum, frame):
|
||||
global stopping
|
||||
stopping = True
|
||||
for proc in list(running):
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# Released by the kernel when the process ends, a crashed run leaves no stale lock
|
||||
lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
sys.exit("can't lock " + lock_file)
|
||||
|
||||
if imapsync_running():
|
||||
print('imapsync is active, exiting...')
|
||||
sys.exit(0)
|
||||
|
||||
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGQUIT):
|
||||
signal.signal(sig, stop)
|
||||
|
||||
conn = connect()
|
||||
# Stale flags from crashed runs, safe only here: under the lock and with no imapsync running
|
||||
query("UPDATE imapsync SET is_running = 0")
|
||||
|
||||
with open('/etc/sogo/sieve.creds') as f:
|
||||
master_user, master_pass = [s.strip() for s in f.readline().split(':', 1)]
|
||||
|
||||
# Worker pool: keep up to max_parallel workers busy, claim the next due job whenever a slot is free
|
||||
# (after every worker exit and at least every 10 seconds while workers run)
|
||||
try:
|
||||
while True:
|
||||
for proc, job in list(running.items()):
|
||||
if proc.poll() is not None:
|
||||
del running[proc]
|
||||
reap_job(job)
|
||||
|
||||
max_parallel, global_bw = read_settings()
|
||||
attempts = 0
|
||||
while not stopping and len(running) < max_parallel and attempts < 100:
|
||||
attempts += 1
|
||||
j = query(next_sql).fetchone()
|
||||
if not j:
|
||||
break
|
||||
if query(claim_sql, (j['id'],)).rowcount != 1:
|
||||
continue
|
||||
|
||||
files = []
|
||||
try:
|
||||
job = prepare_job(j, global_bw, files)
|
||||
except Exception as e:
|
||||
print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
|
||||
finish_job(j['id'], 'Could not start or finish imapsync', 0, None)
|
||||
job = None
|
||||
if not job:
|
||||
unlink(files)
|
||||
continue
|
||||
try:
|
||||
running[start_job(job)] = job
|
||||
except OSError as e:
|
||||
# Release the claim, the job is picked up again later
|
||||
print('imapsync job %d: %s' % (j['id'], e), file=sys.stderr)
|
||||
unlink(files)
|
||||
query("UPDATE imapsync SET is_running = 0 WHERE id = %s", (j['id'],))
|
||||
break
|
||||
|
||||
if not running:
|
||||
break
|
||||
# A worker exit ends the wait early
|
||||
for _ in range(50):
|
||||
if any(proc.poll() is not None for proc in running):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
finally:
|
||||
# Never leave password/token files behind
|
||||
for job in running.values():
|
||||
unlink(job['files'])
|
||||
Reference in new issue
Block a user