See ChangeLog

Monotone-Parent: 027083e4ddfcc14898e7dd8b6e08c73058f18778
Monotone-Revision: 1475912d0e20ad5f1180b0921306270039272161

Monotone-Author: ludovic@Sophos.ca
Monotone-Date: 2010-07-22T15:13:09
Monotone-Branch: ca.inverse.sogo

ChangeLog

@@ -1,3 +1,9 @@
+2010-07-22  Ludovic Marcotte  <lmarcotte@inverse.ca>
+
+	* Added Migration/Horde/* - scripts used to migrate
+	the address books and email signatures from Horde to
+	the SOGo database.
+
 2010-07-21  Wolfgang Sourdeau  <wsourdeau@inverse.ca>
 
 	* UI/WebServerResources/ContactsUI.js: (dropSelectedContacts):

Migration/Horde/HordeSignatureConverter.py

@@ -0,0 +1,84 @@
+import PHPDeserializer
+import sys
+
+class HordeSignatureConverter:
+    def __init__(self, user, domain):
+        self.user = user
+        self.domain = domain
+        self.domainLen = len(domain)
+
+    def fetchSignatures(self, conn):
+        self.signatures = None
+        self.conn = conn
+        self.fetchIdentities()
+
+        return self.signatures
+
+    def fetchIdentities(self):
+        self.users = {}
+
+        cursor = self.conn.cursor()
+        if self.user == "ALL":
+            userClause = ""
+        else:
+            userClause = "AND pref_uid = '%s'" % self.user
+        query = "SELECT pref_uid, pref_value" \
+            "      FROM horde_prefs" \
+            "     WHERE pref_scope = 'horde'" \
+            "       AND pref_name = 'identities'" \
+            "        %s" % userClause
+        cursor.execute(query)
+
+        self.signatures = {}
+        records = cursor.fetchall()
+        max = len(records)
+        if max > 0:
+            for record in records:
+                user = record[0]
+                signature = self.decodeSignature(record[1], user)
+                if signature is None or len(signature.strip()) == 0:
+                    print "No useful signature found for %s" % user
+                else:
+                    self.signatures[user] = signature
+
+            print "%d useful signature(s) found in %d record(s)" % (len(self.signatures), max)
+        else:
+            print "No record found"
+
+        cursor.close()
+
+    def decodeSignature(self, prefs, user):
+        des = PHPDeserializer.PHPDeserializer(prefs)
+        identities = des.deserialize()
+        nbrEntries = len(identities)
+        signatures = []
+        for identity in identities:
+            fromAddr = identity["from_addr"]
+            if (len(fromAddr) > self.domainLen
+                and fromAddr[-self.domainLen:] == self.domain):
+                if identity.has_key("signature"):
+                    signatures.append(identity["signature"])
+
+        if len(signatures) > 0:
+            signature = self.chooseSignature(signatures)
+        else:
+            signature = None
+
+        return signature
+
+    def chooseSignature(self, signatures):
+        biggest = -1
+        length = -1
+        count = 0
+        for signature in signatures:
+            thisLength = len(signature)
+            if thisLength > 0 and thisLength > length:
+                biggest = count
+            count = count + 1
+
+        if biggest == -1:
+            signature = None
+        else:
+            signature = signatures[biggest]
+
+        return signature

Migration/Horde/PHPDeserializer.py

@@ -0,0 +1,135 @@
+# we perform no validation
+class PHPDeserializer:
+    def __init__(self, string):
+        self.string = string
+        if string is None:
+            self.length = 0
+        else:
+            self.length = len(string)
+        self.cursor = 0
+
+    def deserializeInteger(self):
+        start = self.cursor
+
+        done = False
+        while not done:
+            if self.cursor < self.length:
+                currentChar = self.string[self.cursor]
+                if currentChar.isdigit():
+                    self.cursor = self.cursor + 1
+                else:
+                    done = True
+            else:
+                done = True
+
+        length = self.cursor - start
+        if length > 0:
+            dInteger = int(self.string[start:self.cursor])
+        else:
+            dInteger = 0
+
+        return dInteger
+
+    def deserializeBoolean(self):
+        start = self.cursor
+
+        if self.cursor < self.length:
+            currentChar = self.string[self.cursor]
+            if currentChar == "0":
+                dBoolean = False
+            else:
+                dBoolean = True
+            self.cursor = self.cursor + 1
+        else:
+            dBoolean = None
+
+        return dBoolean
+
+    def deserializeString(self):
+        length = self.deserializeInteger()
+        start = self.cursor + 2
+        end = start+length
+        value = self.string[start:end]
+        self.cursor = end + 1
+
+        return value
+
+    def deserializeArray(self):
+        isHash = False
+        max = self.deserializeInteger()
+        dArray = [ None ] * max
+
+        self.cursor = self.cursor + 2
+        count = 0
+        while count < max:
+            elementIndex = self.deserialize()
+            self.cursor = self.cursor + 1
+            element = self.deserialize()
+            if isHash:
+                if type(elementIndex) == int:
+                    elementIndex = "%d" % elementIndex
+            else:
+                if type(elementIndex) != int:
+                    dArray = self._arrayToHash(dArray)
+                    isHash = True
+            dArray[elementIndex] = element
+            self.cursor = self.cursor + 1
+            count = count + 1
+
+        if self.string[self.cursor] != "}":
+            raise Exception, \
+                ("inconsistency detected in serialized string at character %d:\n%s"
+                 % (self.cursor, self._exceptionSample()))
+
+        return dArray
+
+    def _exceptionSample(self):
+        if self.cursor > 30:
+            start = self.cursor - 30
+            prefix = "..."
+        else:
+            start = 0
+            prefix = ""
+        carret = self.cursor + len(prefix) - start
+        length = len(self.string)
+        if self.cursor + 30 < length:
+            end = self.cursor + 30
+            suffix = "..."
+        else:
+            end = length
+            suffix = ""
+
+        sample = self.string[start:end]
+        while sample[0] == " ":
+            carret = carret - 1
+            sample = sample[1:]
+
+        return "%s%s%s\n%s^" % (prefix, sample, suffix, carret * " ")
+
+    def _arrayToHash(self, array):
+        dHash = {}
+        count = 0
+        for element in array:
+            dHash["%d" % count] = element
+            count = count + 1
+
+        return dHash
+
+    def deserialize(self):
+        dObject = None
+
+        if self.string is not None and self.length > 0:
+            currentChar = self.string[self.cursor]
+            self.cursor = self.cursor + 2
+            if currentChar == 'a':
+                dObject = self.deserializeArray()
+            elif currentChar == 's':
+                dObject = self.deserializeString()
+            elif currentChar == 'i':
+                dObject = self.deserializeInteger()
+            elif currentChar == 'b':
+                dObject = self.deserializeBoolean()
+            elif currentChar == 'N':
+                dObject = None
+
+        return dObject

Migration/Horde/README

@@ -0,0 +1,7 @@
+1) copy config.py.in to config.py and personalize the settings
+2) personalize the domain setting in signature.py (look for DOMAIN.COM) 
+3) the scripts require "webdavlib.py", which is located in <SOGoDIR>/Tests/Integration/. Therefore, in order to use the scripts, we must set the environment variable "PYTHONPATH" to that directory on our system.
+4) invoke "signature <USER>" for importing signatures
+5) invoke "turba <USER>" for importing signatures
+6) "<USER>" can have the special "ALL" value
+

Migration/Horde/TurbaConverter.py

@@ -0,0 +1,351 @@
+import PHPDeserializer
+import webdavlib
+import sys
+
+commonMappings = { "owner_id": "owner",
+                   "object_id": "filename",
+                   "object_uid": "uid",
+                   "object_name": "fn" }
+cardMappings = { "object_alias": "nickname",
+                 "object_email": "email",
+                 "object_homeaddress": "homeaddress",
+                 "object_homephone": "homephone",
+                 "object_workaddress": "workaddress",
+                 "object_workphone": "workphone",
+                 "object_cellphone": "cellphone",
+                 "object_fax": "fax",
+                 "object_title": "title",
+                 "object_company": "org",
+                 "object_notes": "notes",
+                 "object_freebusyurl": "fburl" }
+
+prodid = "-//Inverse inc.//SOGo Turba Importer 1.0//EN"
+
+# a managed type of template where each line is put only if at least one field
+# has been filled
+cardTemplate = u"""BEGIN:VCARD\r
+VERSION:3.0\r
+PRODID:%s\r
+UID:${uid}\r
+FN:${fn}\r
+TITLE:${title}\r
+ORG:${org};\r
+NICKNAME:${nickname}\r
+EMAIL:${email}\r
+ADR;TYPE=work:;;${workaddress};;;;\r
+ADR;TYPE=home:;;${homeaddress};;;;\r
+TEL;TYPE=work:${workphone}\r
+TEL;TYPE=home:${homephone}\r
+TEL;TYPE=fax:${fax}\r
+NOTE:${notes}\r
+FBURL:${fburl}\r
+END:VCARD""" % prodid
+
+class TurbaConverter:
+    def __init__(self, user, webdavConfig):
+        self.user = user
+        self.webdavConfig = webdavConfig
+
+    def start(self, conn):
+        self.conn = conn
+        self.readUsers()
+        self.missing = []
+        for user in self.users.keys():
+            self.hasCards = False
+            self.hasLists = False
+            self.currentUser = user
+            self.readUserRecords()
+            if self.hasCards or self.hasLists:
+                print "Converting addressbook of '%s'" % user
+                self.prepareCards()
+                self.uploadCards()
+                self.prepareLists()
+                self.uploadLists()
+            else:
+                self.missing.append(user)
+
+        if len(self.missing) > 0:
+            print "No information extracted for: %s" % ", ".join(self.missing)
+
+        print "Done"
+
+    def readUsers(self):
+        self.users = {}
+
+        cursor = self.conn.cursor()
+        query = "SELECT user_uid, datatree_name FROM horde_datatree"
+        if self.user != "ALL":
+            query = query + " WHERE user_uid = '%s'" % self.user
+        cursor.execute(query)
+
+        records = cursor.fetchall()
+        count = 0
+        max = len(records)
+        for record in records:
+            record_user = record[0].lower()
+            if not self.users.has_key(record_user):
+                self.users[record_user] = []
+            self.users[record_user].append(record[1])
+            count = count + 1
+        cursor.close()
+
+    def readUserRecords(self):
+        self.cards = {}
+        self.lists = {}
+
+        cursor = self.conn.cursor()
+        owner_ids = self.users[self.currentUser]
+        whereClause = "owner_id = '%s'" % "' or owner_id = '".join(owner_ids)
+        query = "SELECT * FROM turba_objects WHERE %s" % whereClause
+        cursor.execute(query)
+        self.prepareColumns(cursor)
+
+        records = cursor.fetchall()
+        count = 0
+        max = len(records)
+        while count < max:
+            self.parseRecord(records[count])
+            count = count + 1
+
+        cursor.close()
+
+    def prepareColumns(self, cursor):
+        self.columns = {}
+        count = 0
+        for dbColumn in cursor.description:
+            columnId = dbColumn[0]
+            self.columns[columnId] = count
+            count = count + 1
+
+    def parseRecord(self, record):
+        typeCol = self.columns["object_type"]
+        meta = {}
+        self.applyRecordMappings(meta, record, commonMappings)
+
+        if record[typeCol] == "Object":
+            meta["type"] = "card"
+            self.hasCards = True
+            self.applyRecordMappings(meta, record, cardMappings)
+        elif record[typeCol] == "Group":
+            meta["type"] = "list"
+            self.hasLists = True
+            self.fillListMembers(meta, record)
+        else:
+            raise Exception, "UNKNOWN TYPE: %s" % record[type]
+
+        self.dispatchMeta(meta)
+
+    def applyRecordMappings(self, meta, record, mappings):
+        for k in mappings.keys():
+            metaKey = mappings[k]
+            meta[metaKey] = self.recordColumn(record, k)
+
+    def recordColumn(self, record, columnName):
+        columnIndex = self.columns[columnName]
+        value = record[columnIndex]
+        if value is None:
+            value = u""
+        else:
+            value = self.deUTF8Ize(value)
+
+        return value
+
+    def deUTF8Ize(self, value):
+        # unicode -> repeat(utf-8 str -> iso-8859-1 str) -> unicode
+        oldValue = value
+
+        done = False
+        while not done:
+            try:
+                utf8Value = value.encode("iso-8859-1")
+                value = utf8Value.decode("utf-8")
+            except:
+                done = True
+            if value == oldValue:
+                done = True
+
+        return value
+
+    def fillListMembers(self, meta, record):
+        members = self.recordColumn(record, "object_members")
+        if members is not None and len(members) > 0:
+            deserializer = PHPDeserializer.PHPDeserializer(members)
+            dMembers = deserializer.deserialize()
+        else:
+            dMembers = []
+        meta["members"] = dMembers
+
+    def dispatchMeta(self, meta):
+        owner = meta["owner"]
+        if meta["type"] == "card":
+            repository = self.cards
+        else:
+            repository = self.lists
+        filename = meta["filename"]
+        repository[filename] = meta
+
+    def prepareCards(self):
+        count = 0
+        for filename in self.cards.keys():
+            card = self.cards[filename]
+            card["data"] = self.buildVCard(card).encode("utf-8")
+            count = count + 1
+        if count > 0:
+            print "  prepared %d cards" % count
+
+    def buildVCard(self, card):
+        vcardArray = []
+        tmplArray = cardTemplate.split("\r\n")
+        for line in tmplArray:
+            keyPos = line.find("${")
+            if keyPos > -1:
+                keyEndPos = line.find("}")
+                key = line[keyPos+2:keyEndPos]
+                if card.has_key(key):
+                    value = card[key]
+                    if len(value) > 0:
+                        newLine = "%s%s%s" % (line[0:keyPos],
+                                              value.replace(";", "\;"),
+                                              line[keyEndPos + 1:])
+                        vcardArray.append(self.foldLineIfNeeded(newLine))
+            else:
+                vcardArray.append(self.foldLineIfNeeded(line))
+
+        return "\r\n".join(vcardArray)
+
+    def foldLineIfNeeded(self, line):
+        wasFolded = False
+        newLine = line\
+            .replace("\\", "\\\\") \
+            .replace("\r", "\\r") \
+            .replace("\n", "\\n")
+        lines = []
+        while len(newLine) > 73:
+            wasFolded = True
+            lines.append(newLine[0:73])
+            newLine = newLine[73:]
+        lines.append(newLine)
+
+        newLine = "\r\n ".join(lines)
+        if wasFolded:
+            print "line was folded: '%s' ->\n\n%s\n\n" % (line, newLine)
+
+        return newLine
+
+    def uploadCards(self):
+        self.uploadEntries(self.cards,
+                           "vcf", "text/x-vcard; charset=utf-8");
+
+    def prepareLists(self):
+        count = 0
+        skipped = 0
+        for filename in self.lists.keys():
+            list = self.lists[filename]
+            vlist = self.buildVList(list)
+            if vlist is None:
+                skipped = skipped + 1
+            else:
+                list["data"] = vlist.encode("utf-8")
+                count = count + 1
+
+        if (count + skipped) > 0:
+            print "  prepared %d lists. %d were skipped." % (count, skipped)
+
+    def buildVList(self, list):
+        vlist = None
+
+        members = list["members"]
+        if len(members) > 0:
+            cardMembers = []
+            for member in members:
+                card = self.getListCard(member)
+                if card is not None:
+                    cardMembers.append(card)
+            if len(cardMembers) > 0:
+                vlist = self.assembleVList(list, cardMembers)
+            else:
+                print "  list '%s' skipped because of lack of usable" \
+                    " members" % list["filename"]
+
+        return vlist
+
+    def getListCard(self, cardRef):
+        card = None
+        if len(cardRef) != 0 and not cardRef.startswith("localldap:"):
+            if cardRef.startswith("localsql:"):
+                cardRef = cardRef[9:]
+            if self.cards.has_key(cardRef):
+                card = self.cards[cardRef]
+            else:
+                print "card reference does not exist: '%s'" % cardRef
+
+        return card
+
+    def assembleVList(self, list, cardMembers):
+        entries = []
+        for cardMember in cardMembers:
+            if cardMember.has_key("fn") and len(cardMember["fn"]) > 0:
+                fn = ";FN=%s" % cardMember["fn"]
+            else:
+                fn = ""
+            if cardMember.has_key("email") and len(cardMember["email"]) > 0:
+                email = ";EMAIL=%s" % cardMember["email"]
+            else:
+                email = ""
+            entries.append("CARD%s%s:%s.vcf"
+                           % (fn, email, cardMember["filename"]))
+        if list.has_key("fn") and len(list["fn"]) > 0:
+            listfn = "FN:%s\r\n" % list["fn"]
+        else:
+            listfn = ""
+        vlist = """BEGIN:VLIST\r
+PRODID:%s\r
+VERSION:1.0\r
+UID:%s\r
+%s%s\r
+END:VLIST""" % (prodid, list["uid"], listfn, "\r\n".join(entries))
+
+        return vlist
+
+    def uploadLists(self):
+        self.uploadEntries(self.lists,
+                           "vlf", "text/x-vcard; charset=utf-8");
+
+    def uploadEntries(self, entries, extension, mimeType):
+        isatty = sys.stdout.isatty() # enable progressive display of summary
+        success = 0
+        failure = 0
+        client = webdavlib.WebDAVClient(self.webdavConfig["hostname"],
+                                        self.webdavConfig["port"],
+                                        self.webdavConfig["username"],
+                                        self.webdavConfig["password"])
+        collection = '/SOGo/dav/%s/Contacts/personal' % self.currentUser
+
+        mkcol = webdavlib.WebDAVMKCOL(collection)
+        client.execute(mkcol)
+        
+        for entryName in entries.keys():
+            entry = entries[entryName]
+            if entry.has_key("data"):
+                fullFilename = "%s.%s" % (entry["filename"], extension)
+                url = "%s/%s" % (collection, fullFilename)
+                put = webdavlib.HTTPPUT(url, entry["data"])
+                put.content_type = mimeType
+                client.execute(put)
+                if (put.response["status"] < 200
+                    or put.response["status"] > 399):
+                    failure = failure + 1
+                    print "  error uploading '%s': %d" \
+                        % (fullFilename, put.response["status"])
+                else:
+                    success = success + 1
+                if isatty:
+                    print "\r  successes: %d; failures: %d" % (success, failure),
+                    if (success + failure) % 5 == 0:
+                        sys.stdout.flush()
+        if isatty:
+            print ""
+        else:
+            if (success + failure) > 0:
+                print "  successes: %d; failures: %d\n" % (success, failure)
+                sys.stdout.flush()

Migration/Horde/config.py.in

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+webdavConfig = { "hostname": "sogo.hostname.com",
+                 "port": 80,
+                 "username": "username",
+                 "password": "password" }
+
+dbConfig = { "host": "localhost",
+             "username": "username",
+             "password": "password",
+             "db": "database" }

Migration/Horde/signature.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+
+import sys
+import MySQLdb
+import webdavlib
+
+import HordeSignatureConverter
+
+from config import webdavConfig, dbConfig
+
+xmlns_inversedav = "urn:inverse:params:xml:ns:inverse-dav"
+
+def UploadSignature(client, user, signature):
+    collection = '/SOGo/dav/%s/' % user
+    proppatch \
+        = webdavlib.WebDAVPROPPATCH(collection,
+                                    { "{%s}signature" % xmlns_inversedav: \
+                                          signature.encode("utf-8") })
+    client.execute(proppatch)
+    if (proppatch.response["status"] < 200
+        or proppatch.response["status"] > 399):
+        print "Failure uploading signature for user '%s': %d" \
+            % (user, proppatch.response["status"])
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        user = sys.argv[1]
+    else:
+        raise Exception, "<user> argument must be specified" \
+            " (use 'ALL' for everyone)"
+
+    conn = MySQLdb.connect(host = dbConfig["hostname"],
+                           user = dbConfig["username"],
+                           passwd = dbConfig["password"],
+                           db = dbConfig["database"],
+                           use_unicode = True)
+
+    cnv = HordeSignatureConverter.HordeSignatureConverter(user, "DOMAIN.COM")
+    signatures = cnv.fetchSignatures(conn)
+    conn.close()
+
+    client = webdavlib.WebDAVClient(webdavConfig["hostname"],
+                                    webdavConfig["port"],
+                                    webdavConfig["username"],
+                                    webdavConfig["password"])
+    for user in signatures:
+        signature = signatures[user]
+        UploadSignature(client, user, signature)

Migration/Horde/turba.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+import sys
+import MySQLdb
+
+import TurbaConverter
+
+from config import webdavConfig, dbConfig
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        user = sys.argv[1]
+    else:
+        raise Exception, "<user> argument must be specified" \
+            " (use 'ALL' for everyone)"
+
+    conn = MySQLdb.connect(host = dbConfig["hostname"],
+                           user = dbConfig["username"],
+                           passwd = dbConfig["password"],
+                           db = dbConfig["database"],
+                           use_unicode = True)
+    cnv = TurbaConverter.TurbaConverter(user, webdavConfig)
+    cnv.start(conn)
+    conn.close()