diff --git a/ChangeLog b/ChangeLog index 64fb4a900..9c444503e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2010-07-22 Ludovic Marcotte + + * Added Migration/Horde/* - scripts used to migrate + the address books and email signatures from Horde to + the SOGo database. + 2010-07-21 Wolfgang Sourdeau * UI/WebServerResources/ContactsUI.js: (dropSelectedContacts): diff --git a/Migration/Horde/HordeSignatureConverter.py b/Migration/Horde/HordeSignatureConverter.py new file mode 100644 index 000000000..29040f37a --- /dev/null +++ b/Migration/Horde/HordeSignatureConverter.py @@ -0,0 +1,84 @@ +import PHPDeserializer +import sys + +class HordeSignatureConverter: + def __init__(self, user, domain): + self.user = user + self.domain = domain + self.domainLen = len(domain) + + def fetchSignatures(self, conn): + self.signatures = None + self.conn = conn + self.fetchIdentities() + + return self.signatures + + def fetchIdentities(self): + self.users = {} + + cursor = self.conn.cursor() + if self.user == "ALL": + userClause = "" + else: + userClause = "AND pref_uid = '%s'" % self.user + query = "SELECT pref_uid, pref_value" \ + " FROM horde_prefs" \ + " WHERE pref_scope = 'horde'" \ + " AND pref_name = 'identities'" \ + " %s" % userClause + cursor.execute(query) + + self.signatures = {} + records = cursor.fetchall() + max = len(records) + if max > 0: + for record in records: + user = record[0] + signature = self.decodeSignature(record[1], user) + if signature is None or len(signature.strip()) == 0: + print "No useful signature found for %s" % user + else: + self.signatures[user] = signature + + print "%d useful signature(s) found in %d record(s)" % (len(self.signatures), max) + else: + print "No record found" + + cursor.close() + + def decodeSignature(self, prefs, user): + des = PHPDeserializer.PHPDeserializer(prefs) + identities = des.deserialize() + nbrEntries = len(identities) + signatures = [] + for identity in identities: + fromAddr = identity["from_addr"] + if (len(fromAddr) > self.domainLen + and fromAddr[-self.domainLen:] == self.domain): + if identity.has_key("signature"): + signatures.append(identity["signature"]) + + if len(signatures) > 0: + signature = self.chooseSignature(signatures) + else: + signature = None + + return signature + + def chooseSignature(self, signatures): + biggest = -1 + length = -1 + count = 0 + for signature in signatures: + thisLength = len(signature) + if thisLength > 0 and thisLength > length: + biggest = count + count = count + 1 + + if biggest == -1: + signature = None + else: + signature = signatures[biggest] + + return signature diff --git a/Migration/Horde/PHPDeserializer.py b/Migration/Horde/PHPDeserializer.py new file mode 100644 index 000000000..675b8a7be --- /dev/null +++ b/Migration/Horde/PHPDeserializer.py @@ -0,0 +1,135 @@ +# we perform no validation +class PHPDeserializer: + def __init__(self, string): + self.string = string + if string is None: + self.length = 0 + else: + self.length = len(string) + self.cursor = 0 + + def deserializeInteger(self): + start = self.cursor + + done = False + while not done: + if self.cursor < self.length: + currentChar = self.string[self.cursor] + if currentChar.isdigit(): + self.cursor = self.cursor + 1 + else: + done = True + else: + done = True + + length = self.cursor - start + if length > 0: + dInteger = int(self.string[start:self.cursor]) + else: + dInteger = 0 + + return dInteger + + def deserializeBoolean(self): + start = self.cursor + + if self.cursor < self.length: + currentChar = self.string[self.cursor] + if currentChar == "0": + dBoolean = False + else: + dBoolean = True + self.cursor = self.cursor + 1 + else: + dBoolean = None + + return dBoolean + + def deserializeString(self): + length = self.deserializeInteger() + start = self.cursor + 2 + end = start+length + value = self.string[start:end] + self.cursor = end + 1 + + return value + + def deserializeArray(self): + isHash = False + max = self.deserializeInteger() + dArray = [ None ] * max + + self.cursor = self.cursor + 2 + count = 0 + while count < max: + elementIndex = self.deserialize() + self.cursor = self.cursor + 1 + element = self.deserialize() + if isHash: + if type(elementIndex) == int: + elementIndex = "%d" % elementIndex + else: + if type(elementIndex) != int: + dArray = self._arrayToHash(dArray) + isHash = True + dArray[elementIndex] = element + self.cursor = self.cursor + 1 + count = count + 1 + + if self.string[self.cursor] != "}": + raise Exception, \ + ("inconsistency detected in serialized string at character %d:\n%s" + % (self.cursor, self._exceptionSample())) + + return dArray + + def _exceptionSample(self): + if self.cursor > 30: + start = self.cursor - 30 + prefix = "..." + else: + start = 0 + prefix = "" + carret = self.cursor + len(prefix) - start + length = len(self.string) + if self.cursor + 30 < length: + end = self.cursor + 30 + suffix = "..." + else: + end = length + suffix = "" + + sample = self.string[start:end] + while sample[0] == " ": + carret = carret - 1 + sample = sample[1:] + + return "%s%s%s\n%s^" % (prefix, sample, suffix, carret * " ") + + def _arrayToHash(self, array): + dHash = {} + count = 0 + for element in array: + dHash["%d" % count] = element + count = count + 1 + + return dHash + + def deserialize(self): + dObject = None + + if self.string is not None and self.length > 0: + currentChar = self.string[self.cursor] + self.cursor = self.cursor + 2 + if currentChar == 'a': + dObject = self.deserializeArray() + elif currentChar == 's': + dObject = self.deserializeString() + elif currentChar == 'i': + dObject = self.deserializeInteger() + elif currentChar == 'b': + dObject = self.deserializeBoolean() + elif currentChar == 'N': + dObject = None + + return dObject diff --git a/Migration/Horde/README b/Migration/Horde/README new file mode 100644 index 000000000..4d04218a8 --- /dev/null +++ b/Migration/Horde/README @@ -0,0 +1,7 @@ +1) copy config.py.in to config.py and personalize the settings +2) personalize the domain setting in signature.py (look for DOMAIN.COM) +3) the scripts require "webdavlib.py", which is located in /Tests/Integration/. Therefore, in order to use the scripts, we must set the environment variable "PYTHONPATH" to that directory on our system. +4) invoke "signature " for importing signatures +5) invoke "turba " for importing signatures +6) "" can have the special "ALL" value + diff --git a/Migration/Horde/TurbaConverter.py b/Migration/Horde/TurbaConverter.py new file mode 100644 index 000000000..cfe32b503 --- /dev/null +++ b/Migration/Horde/TurbaConverter.py @@ -0,0 +1,351 @@ +import PHPDeserializer +import webdavlib +import sys + +commonMappings = { "owner_id": "owner", + "object_id": "filename", + "object_uid": "uid", + "object_name": "fn" } +cardMappings = { "object_alias": "nickname", + "object_email": "email", + "object_homeaddress": "homeaddress", + "object_homephone": "homephone", + "object_workaddress": "workaddress", + "object_workphone": "workphone", + "object_cellphone": "cellphone", + "object_fax": "fax", + "object_title": "title", + "object_company": "org", + "object_notes": "notes", + "object_freebusyurl": "fburl" } + +prodid = "-//Inverse inc.//SOGo Turba Importer 1.0//EN" + +# a managed type of template where each line is put only if at least one field +# has been filled +cardTemplate = u"""BEGIN:VCARD\r +VERSION:3.0\r +PRODID:%s\r +UID:${uid}\r +FN:${fn}\r +TITLE:${title}\r +ORG:${org};\r +NICKNAME:${nickname}\r +EMAIL:${email}\r +ADR;TYPE=work:;;${workaddress};;;;\r +ADR;TYPE=home:;;${homeaddress};;;;\r +TEL;TYPE=work:${workphone}\r +TEL;TYPE=home:${homephone}\r +TEL;TYPE=fax:${fax}\r +NOTE:${notes}\r +FBURL:${fburl}\r +END:VCARD""" % prodid + +class TurbaConverter: + def __init__(self, user, webdavConfig): + self.user = user + self.webdavConfig = webdavConfig + + def start(self, conn): + self.conn = conn + self.readUsers() + self.missing = [] + for user in self.users.keys(): + self.hasCards = False + self.hasLists = False + self.currentUser = user + self.readUserRecords() + if self.hasCards or self.hasLists: + print "Converting addressbook of '%s'" % user + self.prepareCards() + self.uploadCards() + self.prepareLists() + self.uploadLists() + else: + self.missing.append(user) + + if len(self.missing) > 0: + print "No information extracted for: %s" % ", ".join(self.missing) + + print "Done" + + def readUsers(self): + self.users = {} + + cursor = self.conn.cursor() + query = "SELECT user_uid, datatree_name FROM horde_datatree" + if self.user != "ALL": + query = query + " WHERE user_uid = '%s'" % self.user + cursor.execute(query) + + records = cursor.fetchall() + count = 0 + max = len(records) + for record in records: + record_user = record[0].lower() + if not self.users.has_key(record_user): + self.users[record_user] = [] + self.users[record_user].append(record[1]) + count = count + 1 + cursor.close() + + def readUserRecords(self): + self.cards = {} + self.lists = {} + + cursor = self.conn.cursor() + owner_ids = self.users[self.currentUser] + whereClause = "owner_id = '%s'" % "' or owner_id = '".join(owner_ids) + query = "SELECT * FROM turba_objects WHERE %s" % whereClause + cursor.execute(query) + self.prepareColumns(cursor) + + records = cursor.fetchall() + count = 0 + max = len(records) + while count < max: + self.parseRecord(records[count]) + count = count + 1 + + cursor.close() + + def prepareColumns(self, cursor): + self.columns = {} + count = 0 + for dbColumn in cursor.description: + columnId = dbColumn[0] + self.columns[columnId] = count + count = count + 1 + + def parseRecord(self, record): + typeCol = self.columns["object_type"] + meta = {} + self.applyRecordMappings(meta, record, commonMappings) + + if record[typeCol] == "Object": + meta["type"] = "card" + self.hasCards = True + self.applyRecordMappings(meta, record, cardMappings) + elif record[typeCol] == "Group": + meta["type"] = "list" + self.hasLists = True + self.fillListMembers(meta, record) + else: + raise Exception, "UNKNOWN TYPE: %s" % record[type] + + self.dispatchMeta(meta) + + def applyRecordMappings(self, meta, record, mappings): + for k in mappings.keys(): + metaKey = mappings[k] + meta[metaKey] = self.recordColumn(record, k) + + def recordColumn(self, record, columnName): + columnIndex = self.columns[columnName] + value = record[columnIndex] + if value is None: + value = u"" + else: + value = self.deUTF8Ize(value) + + return value + + def deUTF8Ize(self, value): + # unicode -> repeat(utf-8 str -> iso-8859-1 str) -> unicode + oldValue = value + + done = False + while not done: + try: + utf8Value = value.encode("iso-8859-1") + value = utf8Value.decode("utf-8") + except: + done = True + if value == oldValue: + done = True + + return value + + def fillListMembers(self, meta, record): + members = self.recordColumn(record, "object_members") + if members is not None and len(members) > 0: + deserializer = PHPDeserializer.PHPDeserializer(members) + dMembers = deserializer.deserialize() + else: + dMembers = [] + meta["members"] = dMembers + + def dispatchMeta(self, meta): + owner = meta["owner"] + if meta["type"] == "card": + repository = self.cards + else: + repository = self.lists + filename = meta["filename"] + repository[filename] = meta + + def prepareCards(self): + count = 0 + for filename in self.cards.keys(): + card = self.cards[filename] + card["data"] = self.buildVCard(card).encode("utf-8") + count = count + 1 + if count > 0: + print " prepared %d cards" % count + + def buildVCard(self, card): + vcardArray = [] + tmplArray = cardTemplate.split("\r\n") + for line in tmplArray: + keyPos = line.find("${") + if keyPos > -1: + keyEndPos = line.find("}") + key = line[keyPos+2:keyEndPos] + if card.has_key(key): + value = card[key] + if len(value) > 0: + newLine = "%s%s%s" % (line[0:keyPos], + value.replace(";", "\;"), + line[keyEndPos + 1:]) + vcardArray.append(self.foldLineIfNeeded(newLine)) + else: + vcardArray.append(self.foldLineIfNeeded(line)) + + return "\r\n".join(vcardArray) + + def foldLineIfNeeded(self, line): + wasFolded = False + newLine = line\ + .replace("\\", "\\\\") \ + .replace("\r", "\\r") \ + .replace("\n", "\\n") + lines = [] + while len(newLine) > 73: + wasFolded = True + lines.append(newLine[0:73]) + newLine = newLine[73:] + lines.append(newLine) + + newLine = "\r\n ".join(lines) + if wasFolded: + print "line was folded: '%s' ->\n\n%s\n\n" % (line, newLine) + + return newLine + + def uploadCards(self): + self.uploadEntries(self.cards, + "vcf", "text/x-vcard; charset=utf-8"); + + def prepareLists(self): + count = 0 + skipped = 0 + for filename in self.lists.keys(): + list = self.lists[filename] + vlist = self.buildVList(list) + if vlist is None: + skipped = skipped + 1 + else: + list["data"] = vlist.encode("utf-8") + count = count + 1 + + if (count + skipped) > 0: + print " prepared %d lists. %d were skipped." % (count, skipped) + + def buildVList(self, list): + vlist = None + + members = list["members"] + if len(members) > 0: + cardMembers = [] + for member in members: + card = self.getListCard(member) + if card is not None: + cardMembers.append(card) + if len(cardMembers) > 0: + vlist = self.assembleVList(list, cardMembers) + else: + print " list '%s' skipped because of lack of usable" \ + " members" % list["filename"] + + return vlist + + def getListCard(self, cardRef): + card = None + if len(cardRef) != 0 and not cardRef.startswith("localldap:"): + if cardRef.startswith("localsql:"): + cardRef = cardRef[9:] + if self.cards.has_key(cardRef): + card = self.cards[cardRef] + else: + print "card reference does not exist: '%s'" % cardRef + + return card + + def assembleVList(self, list, cardMembers): + entries = [] + for cardMember in cardMembers: + if cardMember.has_key("fn") and len(cardMember["fn"]) > 0: + fn = ";FN=%s" % cardMember["fn"] + else: + fn = "" + if cardMember.has_key("email") and len(cardMember["email"]) > 0: + email = ";EMAIL=%s" % cardMember["email"] + else: + email = "" + entries.append("CARD%s%s:%s.vcf" + % (fn, email, cardMember["filename"])) + if list.has_key("fn") and len(list["fn"]) > 0: + listfn = "FN:%s\r\n" % list["fn"] + else: + listfn = "" + vlist = """BEGIN:VLIST\r +PRODID:%s\r +VERSION:1.0\r +UID:%s\r +%s%s\r +END:VLIST""" % (prodid, list["uid"], listfn, "\r\n".join(entries)) + + return vlist + + def uploadLists(self): + self.uploadEntries(self.lists, + "vlf", "text/x-vcard; charset=utf-8"); + + def uploadEntries(self, entries, extension, mimeType): + isatty = sys.stdout.isatty() # enable progressive display of summary + success = 0 + failure = 0 + client = webdavlib.WebDAVClient(self.webdavConfig["hostname"], + self.webdavConfig["port"], + self.webdavConfig["username"], + self.webdavConfig["password"]) + collection = '/SOGo/dav/%s/Contacts/personal' % self.currentUser + + mkcol = webdavlib.WebDAVMKCOL(collection) + client.execute(mkcol) + + for entryName in entries.keys(): + entry = entries[entryName] + if entry.has_key("data"): + fullFilename = "%s.%s" % (entry["filename"], extension) + url = "%s/%s" % (collection, fullFilename) + put = webdavlib.HTTPPUT(url, entry["data"]) + put.content_type = mimeType + client.execute(put) + if (put.response["status"] < 200 + or put.response["status"] > 399): + failure = failure + 1 + print " error uploading '%s': %d" \ + % (fullFilename, put.response["status"]) + else: + success = success + 1 + if isatty: + print "\r successes: %d; failures: %d" % (success, failure), + if (success + failure) % 5 == 0: + sys.stdout.flush() + if isatty: + print "" + else: + if (success + failure) > 0: + print " successes: %d; failures: %d\n" % (success, failure) + sys.stdout.flush() diff --git a/Migration/Horde/config.py.in b/Migration/Horde/config.py.in new file mode 100644 index 000000000..05aa6b60c --- /dev/null +++ b/Migration/Horde/config.py.in @@ -0,0 +1,11 @@ +#!/usr/bin/python + +webdavConfig = { "hostname": "sogo.hostname.com", + "port": 80, + "username": "username", + "password": "password" } + +dbConfig = { "host": "localhost", + "username": "username", + "password": "password", + "db": "database" } diff --git a/Migration/Horde/signature.py b/Migration/Horde/signature.py new file mode 100755 index 000000000..a8ce024e1 --- /dev/null +++ b/Migration/Horde/signature.py @@ -0,0 +1,48 @@ +#!/usr/bin/python + +import sys +import MySQLdb +import webdavlib + +import HordeSignatureConverter + +from config import webdavConfig, dbConfig + +xmlns_inversedav = "urn:inverse:params:xml:ns:inverse-dav" + +def UploadSignature(client, user, signature): + collection = '/SOGo/dav/%s/' % user + proppatch \ + = webdavlib.WebDAVPROPPATCH(collection, + { "{%s}signature" % xmlns_inversedav: \ + signature.encode("utf-8") }) + client.execute(proppatch) + if (proppatch.response["status"] < 200 + or proppatch.response["status"] > 399): + print "Failure uploading signature for user '%s': %d" \ + % (user, proppatch.response["status"]) + +if __name__ == "__main__": + if len(sys.argv) > 1: + user = sys.argv[1] + else: + raise Exception, " argument must be specified" \ + " (use 'ALL' for everyone)" + + conn = MySQLdb.connect(host = dbConfig["hostname"], + user = dbConfig["username"], + passwd = dbConfig["password"], + db = dbConfig["database"], + use_unicode = True) + + cnv = HordeSignatureConverter.HordeSignatureConverter(user, "DOMAIN.COM") + signatures = cnv.fetchSignatures(conn) + conn.close() + + client = webdavlib.WebDAVClient(webdavConfig["hostname"], + webdavConfig["port"], + webdavConfig["username"], + webdavConfig["password"]) + for user in signatures: + signature = signatures[user] + UploadSignature(client, user, signature) diff --git a/Migration/Horde/turba.py b/Migration/Horde/turba.py new file mode 100755 index 000000000..22247a275 --- /dev/null +++ b/Migration/Horde/turba.py @@ -0,0 +1,24 @@ +#!/usr/bin/python + +import sys +import MySQLdb + +import TurbaConverter + +from config import webdavConfig, dbConfig + +if __name__ == "__main__": + if len(sys.argv) > 1: + user = sys.argv[1] + else: + raise Exception, " argument must be specified" \ + " (use 'ALL' for everyone)" + + conn = MySQLdb.connect(host = dbConfig["hostname"], + user = dbConfig["username"], + passwd = dbConfig["password"], + db = dbConfig["database"], + use_unicode = True) + cnv = TurbaConverter.TurbaConverter(user, webdavConfig) + cnv.start(conn) + conn.close()