From 1dbfc183793b4c01357b95e8c3385ea2aec297a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Sat, 18 Jul 2015 08:47:55 +0200 Subject: [PATCH 01/13] oc: Update to new XID structure definition From ede986f commit from OpenChange repository. --- OpenChange/MAPIStoreGCSFolder.m | 4 ++-- OpenChange/MAPIStoreMailFolder.m | 4 ++-- OpenChange/NSData+MAPIStore.m | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index ce4bb870f..73bb647f5 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -270,8 +270,8 @@ static Class NSNumberK; NSMutableDictionary *changeList; xid = [changeKey asXIDInMemCtx: NULL]; - guid = [NSString stringWithGUID: &xid->GUID]; - globCnt = [NSData dataWithBytes: xid->Data length: xid->Size]; + guid = [NSString stringWithGUID: &xid->NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid->LocalId.data length: xid->LocalId.length]; talloc_free (xid); if (!inChangeListOnly) diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index d4defea1e..4f66fc2f4 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -526,8 +526,8 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) NSMutableDictionary *changeList; xid = [changeKey asXIDInMemCtx: NULL]; - guid = [NSString stringWithGUID: &xid->GUID]; - globCnt = [NSData dataWithBytes: xid->Data length: xid->Size]; + guid = [NSString stringWithGUID: &xid->NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid->LocalId.data length: xid->LocalId.length]; talloc_free (xid); /* 1. set change key association */ diff --git a/OpenChange/NSData+MAPIStore.m b/OpenChange/NSData+MAPIStore.m index 44b6dcf85..38ef1ccf4 100644 --- a/OpenChange/NSData+MAPIStore.m +++ b/OpenChange/NSData+MAPIStore.m @@ -136,11 +136,11 @@ static void _fillFlatUIDWithGUID (struct FlatUID_r *flatUID, const struct GUID * NSMutableData *xidData; struct FlatUID_r flatUID; - _fillFlatUIDWithGUID (&flatUID, &xid->GUID); + _fillFlatUIDWithGUID (&flatUID, &xid->NameSpaceGuid); - xidData = [NSMutableData dataWithCapacity: 16 + xid->Size]; + xidData = [NSMutableData dataWithCapacity: 16 + xid->LocalId.length]; [xidData appendBytes: flatUID.ab length: 16]; - [xidData appendBytes: xid->Data length: xid->Size]; + [xidData appendBytes: xid->LocalId.data length: xid->LocalId.length]; return xidData; } @@ -156,12 +156,12 @@ static void _fillFlatUIDWithGUID (struct FlatUID_r *flatUID, const struct GUID * { xid = talloc_zero (memCtx, struct XID); - [self _extractGUID: &xid->GUID]; + [self _extractGUID: &xid->NameSpaceGuid]; - xid->Size = max - 16; + xid->LocalId.length = max - 16; bytes = (uint8_t *) [self bytes]; - xid->Data = talloc_memdup (xid, (bytes+16), xid->Size); + xid->LocalId.data = talloc_memdup (xid, (bytes+16), xid->LocalId.length); } else { From 8d9b54815c5e35ccf46c8085a5f63c41f2e3c871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Sat, 18 Jul 2015 08:56:59 +0200 Subject: [PATCH 02/13] oc: Receive new predecessor change list parameter on move copy op This is to apply new API introduced by 85e2d7c commit in OpenChange repository. --- OpenChange/MAPIStoreFolder.h | 1 + OpenChange/MAPIStoreFolder.m | 17 +++++++++++++---- OpenChange/MAPIStoreMailFolder.m | 2 ++ OpenChange/MAPIStoreSOGo.m | 2 ++ 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/OpenChange/MAPIStoreFolder.h b/OpenChange/MAPIStoreFolder.h index b16e4b91f..3f3bfb204 100644 --- a/OpenChange/MAPIStoreFolder.h +++ b/OpenChange/MAPIStoreFolder.h @@ -121,6 +121,7 @@ fromFolder: (MAPIStoreFolder *) sourceFolder withMIDs: (uint64_t *) targetMids andChangeKeys: (struct Binary_r **) targetChangeKeys + andPredecessorChangeLists: (struct Binary_r **) targetPredecessorChangeLists wantCopy: (uint8_t) want_copy inMemCtx: (TALLOC_CTX *) memCtx; diff --git a/OpenChange/MAPIStoreFolder.m b/OpenChange/MAPIStoreFolder.m index 7ca80e749..ac3b19bf3 100644 --- a/OpenChange/MAPIStoreFolder.m +++ b/OpenChange/MAPIStoreFolder.m @@ -642,6 +642,7 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe fromFolder: (MAPIStoreFolder *) sourceFolder withMID: (uint64_t) targetMid andChangeKey: (struct Binary_r *) targetChangeKey + andPredecessorChangeList: (struct Binary_r *) targetPredecessorChangeList wantCopy: (uint8_t) wantCopy inMemCtx: (TALLOC_CTX *) memCtx { @@ -696,6 +697,7 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe fromFolder: (MAPIStoreFolder *) sourceFolder withMIDs: (uint64_t *) targetMids andChangeKeys: (struct Binary_r **) targetChangeKeys + andPredecessorChangeLists: (struct Binary_r **) targetPredecessorChangeLists wantCopy: (uint8_t) wantCopy inMemCtx: (TALLOC_CTX *) memCtx { @@ -705,7 +707,7 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe NSString *oldMessageURL; MAPIStoreMapping *mapping; SOGoUser *ownerUser; - struct Binary_r *targetChangeKey; + struct Binary_r *targetChangeKey, *targetPredecessorChangeList; //TALLOC_CTX *memCtx; //memCtx = talloc_zero (NULL, TALLOC_CTX); @@ -726,14 +728,21 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe if (oldMessageURL) { [oldMessageURLs addObject: oldMessageURL]; - if (targetChangeKeys) - targetChangeKey = targetChangeKeys[count]; + if (targetChangeKeys && targetPredecessorChangeList) + { + targetChangeKey = targetChangeKeys[count]; + targetPredecessorChangeList = targetPredecessorChangeLists[count]; + } else - targetChangeKey = NULL; + { + targetChangeKey = NULL; + targetPredecessorChangeList = NULL; + } rc = [self _moveCopyMessageWithMID: srcMids[count] fromFolder: sourceFolder withMID: targetMids[count] andChangeKey: targetChangeKey + andPredecessorChangeList: targetPredecessorChangeList wantCopy: wantCopy inMemCtx: memCtx]; } diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 4f66fc2f4..9026dcbf8 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -1217,6 +1217,7 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) fromFolder: (MAPIStoreFolder *) sourceFolder withMIDs: (uint64_t *) targetMids andChangeKeys: (struct Binary_r **) targetChangeKeys + andPredecessorChangeLists: (struct Binary_r **) targetPredecessorChangeLists wantCopy: (uint8_t) wantCopy inMemCtx: (TALLOC_CTX *) memCtx @@ -1237,6 +1238,7 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) return [super moveCopyMessagesWithMIDs: srcMids andCount: midCount fromFolder: sourceFolder withMIDs: targetMids andChangeKeys: targetChangeKeys + andPredecessorChangeLists: targetPredecessorChangeLists wantCopy: wantCopy inMemCtx: memCtx]; diff --git a/OpenChange/MAPIStoreSOGo.m b/OpenChange/MAPIStoreSOGo.m index c11967a0e..0923c44b5 100644 --- a/OpenChange/MAPIStoreSOGo.m +++ b/OpenChange/MAPIStoreSOGo.m @@ -674,6 +674,7 @@ sogo_folder_move_copy_messages(void *folder_object, uint32_t mid_count, uint64_t *src_mids, uint64_t *t_mids, struct Binary_r **target_change_keys, + struct Binary_r **target_predecessor_change_lists, uint8_t want_copy) { MAPIStoreFolder *sourceFolder, *targetFolder; @@ -698,6 +699,7 @@ sogo_folder_move_copy_messages(void *folder_object, fromFolder: sourceFolder withMIDs: t_mids andChangeKeys: target_change_keys + andPredecessorChangeLists: target_predecessor_change_lists wantCopy: want_copy inMemCtx: mem_ctx]; TRYCATCH_END(pool) From 321672e2c3b5ed8468c5f530a7240a4f72945f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Mon, 20 Jul 2015 11:17:00 +0200 Subject: [PATCH 03/13] oc: Update predecessor change list on saving There were cases where only the change key was updated (GCS) or others were the change key was updated with wrong info. This changeset has as goal to update the predecessor change list and, change key if required, on saving taking into account the latest information given by the client in high level ROPs such as ImportMessageMove or SetProperties, and merge it with information provided by the server backend (IMAP server, SOGo DB) using `synchroniseCache`. For more details about `PidTagChangeKey` and `PidTagPredecessorChangeList` property values check [MS-OXCFXICS] Section 2.2.1.2 --- OpenChange/MAPIStoreDBMessage.m | 104 +++++++++++++++++++++++++++++++ OpenChange/MAPIStoreFolder.m | 11 ++-- OpenChange/MAPIStoreGCSFolder.h | 3 +- OpenChange/MAPIStoreGCSFolder.m | 98 ++++++++++++++++++++++++----- OpenChange/MAPIStoreGCSMessage.m | 9 ++- OpenChange/MAPIStoreMailFolder.m | 63 ++++++++++++++++--- OpenChange/NSData+MAPIStore.h | 2 + OpenChange/NSData+MAPIStore.m | 34 ++++++++++ 8 files changed, 292 insertions(+), 32 deletions(-) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index 077f2c708..cdfe00cd8 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -22,6 +22,7 @@ #import #import +#import #import #import #import @@ -34,6 +35,7 @@ #import "MAPIStoreDBFolder.h" #import "MAPIStoreDBMessage.h" #import "MAPIStoreTypes.h" +#import "NSData+MAPIStore.h" #import "NSObject+MAPIStore.h" #import "NSString+MAPIStore.h" @@ -104,6 +106,105 @@ return objectVersion; } +- (void) _updatePredecessorChangeList +{ + BOOL updated; + enum mapistore_error rc; + NSData *currentChangeList, *changeKey; + NSMutableArray *changeKeys; + NSMutableData *newChangeList; + NSUInteger count, len; + struct SizedXid *changes; + struct SPropValue property; + struct SRow aRow; + struct XID *currentChangeKey; + TALLOC_CTX *localMemCtx; + uint32_t nChanges; + + localMemCtx = talloc_new (NULL); + if (!localMemCtx) + { + [self errorWithFormat: @"No more memory"]; + return; + } + + changeKey = [self getReplicaKeyFromGlobCnt: [self objectVersion]]; + + currentChangeList = [properties objectForKey: MAPIPropertyKey (PidTagPredecessorChangeList)]; + if (!currentChangeList) + { + /* Create a new PredecessorChangeList */ + len = [changeKey length]; + newChangeList = [NSMutableData dataWithCapacity: len + 1]; + [newChangeList appendUInt8: len]; + [newChangeList appendData: changeKey]; + } + else + { + /* Update current predecessor change list with new change key */ + changes = [currentChangeList asSizedXidArrayInMemCtx: localMemCtx + with: &nChanges]; + + updated = NO; + currentChangeKey = [changeKey asXIDInMemCtx: localMemCtx]; + for (count = 0; count < nChanges && !updated; count++) + { + if (GUID_equal(&changes[count].XID.NameSpaceGuid, ¤tChangeKey->NameSpaceGuid)) + { + NSData *globCnt, *oldGlobCnt; + oldGlobCnt = [NSData dataWithBytes: changes[count].XID.LocalId.data length: changes[count].XID.LocalId.length]; + globCnt = [NSData dataWithBytes: currentChangeKey->LocalId.data length: currentChangeKey->LocalId.length]; + if ([globCnt compare: oldGlobCnt] == NSOrderedDescending) + { + if ([globCnt length] != [oldGlobCnt length]) + { + [self errorWithFormat: @"Cannot compare globcnt with different length: %@ and %@", globCnt, oldGlobCnt]; + abort(); + } + memcpy (changes[count].XID.LocalId.data, currentChangeKey->LocalId.data, currentChangeKey->LocalId.length); + updated = YES; + } + } + } + + /* Serialise it */ + changeKeys = [NSMutableArray array]; + + if (!updated) + [changeKeys addObject: changeKey]; + + for (count = 0; count < nChanges; count++) + { + changeKey = [NSData dataWithXID: &changes[count].XID]; + [changeKeys addObject: changeKey]; + } + + [changeKeys sortUsingFunction: MAPIChangeKeyGUIDCompare context: localMemCtx]; + + newChangeList = [NSMutableData data]; + len = [changeKeys count]; + for (count = 0; count < len; count++) + { + changeKey = [changeKeys objectAtIndex: count]; + [newChangeList appendUInt8: [changeKey length]]; + [newChangeList appendData: changeKey]; + } + } + + if ([newChangeList length] > 0) + { + property.ulPropTag = PidTagPredecessorChangeList; + property.value.bin = *[newChangeList asBinaryInMemCtx: localMemCtx]; + aRow.cValues = 1; + aRow.lpProps = &property; + rc = [self addPropertiesFromRow: &aRow]; + if (rc != MAPISTORE_SUCCESS) + [self errorWithFormat: @"Impossible to add a new predecessor change list: %d", rc]; + } + + talloc_free (localMemCtx); +} + // // FIXME: how this can happen? // @@ -166,6 +267,9 @@ [properties setObject: [NSNumber numberWithUnsignedLongLong: newVersion] forKey: @"version"]; + /* Update PredecessorChangeList accordingly */ + [self _updatePredecessorChangeList]; + [self logWithFormat: @"%d props in dict", [properties count]]; [sogoObject save]; diff --git a/OpenChange/MAPIStoreFolder.m b/OpenChange/MAPIStoreFolder.m index ac3b19bf3..735c04cd6 100644 --- a/OpenChange/MAPIStoreFolder.m +++ b/OpenChange/MAPIStoreFolder.m @@ -670,15 +670,18 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe [sourceMsg copyToMessage: destMsg inMemCtx: memCtx]; - if (targetChangeKey) + if (targetPredecessorChangeList) { - property.ulPropTag = PidTagChangeKey; - property.value.bin = *targetChangeKey; + property.ulPropTag = PidTagPredecessorChangeList; + property.value.bin = *targetPredecessorChangeList; aRow.cValues = 1; aRow.lpProps = &property; rc = [destMsg addPropertiesFromRow: &aRow]; if (rc != MAPISTORE_SUCCESS) - goto end; + { + [self errorWithFormat: @"Cannot add PredecessorChangeList on move"]; + goto end; + } } [destMsg save: memCtx]; if (!wantCopy) diff --git a/OpenChange/MAPIStoreGCSFolder.h b/OpenChange/MAPIStoreGCSFolder.h index d4e3660f1..ecf9cee18 100644 --- a/OpenChange/MAPIStoreGCSFolder.h +++ b/OpenChange/MAPIStoreGCSFolder.h @@ -42,7 +42,8 @@ /* synchronisation */ - (BOOL) synchroniseCache; - (void) updateVersionsForMessageWithKey: (NSString *) messageKey - withChangeKey: (NSData *) newChangeKey; + withChangeKey: (NSData *) oldChangeKey + andPredecessorChangeList: (NSData *) pcl; - (NSNumber *) lastModifiedFromMessageChangeNumber: (NSString *) changeNumber; - (NSString *) changeNumberForMessageWithKey: (NSString *) messageKey; - (NSData *) changeKeyForMessageWithKey: (NSString *) messageKey; diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index 73bb647f5..55e4f52e1 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -261,7 +261,6 @@ static Class NSNumberK; */ - (void) _setChangeKey: (NSData *) changeKey forMessageEntry: (NSMutableDictionary *) messageEntry - inChangeListOnly: (BOOL) inChangeListOnly { struct XID *xid; NSString *guid; @@ -274,15 +273,11 @@ static Class NSNumberK; globCnt = [NSData dataWithBytes: xid->LocalId.data length: xid->LocalId.length]; talloc_free (xid); - if (!inChangeListOnly) - { - /* 1. set change key association */ - changeKeyDict = [NSDictionary dictionaryWithObjectsAndKeys: - guid, @"GUID", - globCnt, @"LocalId", - nil]; - [messageEntry setObject: changeKeyDict forKey: @"ChangeKey"]; - } + /* 1. set change key association */ + changeKeyDict = [NSDictionary dictionaryWithObjectsAndKeys: guid, @"GUID", + globCnt, @"LocalId", + nil]; + [messageEntry setObject: changeKeyDict forKey: @"ChangeKey"]; /* 2. append/update predecessor change list */ changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; @@ -296,6 +291,77 @@ static Class NSNumberK; [changeList setObject: globCnt forKey: guid]; } +- (void) _updatePredecessorChangeList: (NSData *) predecessorChangeList + forMessageEntry: (NSMutableDictionary *) messageEntry + withOldChangeKey: (NSData *) oldChangeKey +{ + NSData *globCnt, *oldGlobCnt; + NSDictionary *changeKeyDict; + NSString *guid; + NSMutableDictionary *changeList; + struct SizedXid *sizedXIDList; + struct XID xid, *givenChangeKey; + TALLOC_CTX *localMemCtx; + uint32_t i, length; + + localMemCtx = talloc_new (NULL); + if (!localMemCtx) + { + [self errorWithFormat: @"No more memory"]; + return; + } + + if (predecessorChangeList) + { + sizedXIDList = [predecessorChangeList asSizedXidArrayInMemCtx: localMemCtx with: &length]; + + changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; + if (!changeList) + { + changeList = [NSMutableDictionary new]; + [messageEntry setObject: changeList + forKey: @"PredecessorChangeList"]; + [changeList release]; + } + + if (sizedXIDList) { + for (i = 0; i < length; i++) + { + xid = sizedXIDList[i].XID; + guid = [NSString stringWithGUID: &xid.NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid.LocalId.data length: xid.LocalId.length]; + oldGlobCnt = [changeList objectForKey: guid]; + if (!oldGlobCnt || ([globCnt compare: oldGlobCnt] == NSOrderedDescending)) + [changeList setObject: globCnt forKey: guid]; + } + } + } + + if (oldChangeKey) + { + givenChangeKey = [oldChangeKey asXIDInMemCtx: localMemCtx]; + if (givenChangeKey) { + guid = [NSString stringWithGUID: &givenChangeKey->NameSpaceGuid]; + globCnt = [NSData dataWithBytes: givenChangeKey->LocalId.data length: givenChangeKey->LocalId.length]; + + changeKeyDict = [messageEntry objectForKey: @"ChangeKey"]; + if (!changeKeyDict || + ([guid isEqualToString: [changeKeyDict objectForKey: @"GUID"]] + && ([globCnt compare: [changeKeyDict objectForKey: @"LocalId"]] == NSOrderedDescending))) + { + /* The given change key is greater than current one stored in + metadata or it does not exist */ + [messageEntry setObject: [NSDictionary dictionaryWithObjectsAndKeys: guid, @"GUID", + globCnt, @"LocalId", + nil] + forKey: @"ChangeKey"]; + } + } + } + + talloc_free (localMemCtx); +} + - (EOQualifier *) componentQualifier { if (!componentQualifier) @@ -465,8 +531,7 @@ static Class NSNumberK; // A GLOBCNT structure is a 6-byte global namespace counter, // we strip the first 2 bytes. The first two bytes is the ReplicaId changeKey = [self getReplicaKeyFromGlobCnt: newChangeNum >> 16]; - [self _setChangeKey: changeKey forMessageEntry: messageEntry - inChangeListOnly: NO]; + [self _setChangeKey: changeKey forMessageEntry: messageEntry]; } now = [NSCalendarDate date]; @@ -483,12 +548,13 @@ static Class NSNumberK; } - (void) updateVersionsForMessageWithKey: (NSString *) messageKey - withChangeKey: (NSData *) newChangeKey + withChangeKey: (NSData *) oldChangeKey + andPredecessorChangeList: (NSData *) pcl { NSMutableDictionary *messages, *messageEntry; [self synchroniseCache]; - if (newChangeKey) + if (oldChangeKey || pcl) { messages = [[versionsMessage properties] objectForKey: @"Messages"]; messageEntry = [messages objectForKey: messageKey]; @@ -496,8 +562,8 @@ static Class NSNumberK; [NSException raise: @"MAPIStoreIOException" format: @"no version record found for message '%@'", messageKey]; - [self _setChangeKey: newChangeKey forMessageEntry: messageEntry - inChangeListOnly: YES]; + [self _updatePredecessorChangeList: pcl forMessageEntry: messageEntry + withOldChangeKey: oldChangeKey]; [versionsMessage save]; } } diff --git a/OpenChange/MAPIStoreGCSMessage.m b/OpenChange/MAPIStoreGCSMessage.m index 7ab05e5f1..747bb783a 100644 --- a/OpenChange/MAPIStoreGCSMessage.m +++ b/OpenChange/MAPIStoreGCSMessage.m @@ -209,13 +209,16 @@ - (void) updateVersions { - NSData *newChangeKey; + /* Update ChangeKey and PredecessorChangeList on message's save */ + NSData *newChangeKey, *predecessorChangeList; newChangeKey = [properties objectForKey: MAPIPropertyKey (PR_CHANGE_KEY)]; + predecessorChangeList = [properties objectForKey: MAPIPropertyKey (PR_PREDECESSOR_CHANGE_LIST)]; [(MAPIStoreGCSFolder *) container - updateVersionsForMessageWithKey: [self nameInContainer] - withChangeKey: newChangeKey]; + updateVersionsForMessageWithKey: [self nameInContainer] + withChangeKey: newChangeKey + andPredecessorChangeList: predecessorChangeList]; } @end diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 9026dcbf8..963fb7463 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -68,6 +68,8 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; +#include + #undef DEBUG #include #include @@ -516,6 +518,44 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) return [modseq1 compare: modseq2]; } +- (void) _updatePredecessorChangeListWith: (NSData *) predecessorChangeList + forMessageEntry: (NSMutableDictionary *) messageEntry +{ + NSData *globCnt, *oldGlobCnt; + NSMutableDictionary *changeList; + NSString *guid; + struct SizedXid *sizedXIDList; + struct XID xid; + uint32_t i, length; + + sizedXIDList = [predecessorChangeList asSizedXidArrayInMemCtx: NULL with: &length]; + + changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; + if (!changeList) + { + changeList = [NSMutableDictionary new]; + [messageEntry setObject: changeList + forKey: @"PredecessorChangeList"]; + [changeList release]; + } + + if (sizedXIDList) { + for (i = 0; i < length; i++) + { + xid = sizedXIDList[i].XID; + guid = [NSString stringWithGUID: &xid.NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid.LocalId.data length: xid.LocalId.length]; + oldGlobCnt = [changeList objectForKey: guid]; + if (!oldGlobCnt || ([globCnt compare: oldGlobCnt] == NSOrderedDescending)) + [changeList setObject: globCnt forKey: guid]; + } + + talloc_free (sizedXIDList); + } + + [versionsMessage save]; +} + - (void) _setChangeKey: (NSData *) changeKey forMessageEntry: (NSMutableDictionary *) messageEntry { @@ -924,8 +964,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) return changeNumber; } -- (void) setChangeKey: (NSData *) changeKey - forMessageWithKey: (NSString *) messageKey +- (NSMutableDictionary *) _messageEntryFromMessageKey: (NSString *) messageKey { NSMutableDictionary *messages, *messageEntry; NSString *messageUid; @@ -936,7 +975,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) messageEntry = [messages objectForKey: messageUid]; if (!messageEntry) { - [self warnWithFormat: @"attempting to synchronise to set the change key for " + [self warnWithFormat: @"attempting to synchronise to get the message entry for " @"this message %@", messageKey]; synced = [self synchroniseCacheForUID: messageUid]; if (synced) @@ -947,7 +986,15 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) abort (); } } - [self _setChangeKey: changeKey forMessageEntry: messageEntry]; + + return messageEntry; +} + +- (void) setChangeKey: (NSData *) changeKey + forMessageWithKey: (NSString *) messageKey +{ + [self _setChangeKey: changeKey + forMessageEntry: [self _messageEntryFromMessageKey: messageKey]]; [versionsMessage save]; } @@ -1232,7 +1279,7 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) NSDictionary *result; NSUInteger count; NSArray *a; - NSData *changeKey; + NSData *changeList; if (![sourceFolder isKindOfClass: [MAPIStoreMailFolder class]]) return [super moveCopyMessagesWithMIDs: srcMids andCount: midCount @@ -1327,11 +1374,11 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) [self synchroniseCache]; for (count = 0; count < midCount; count++) { - changeKey = [NSData dataWithBinary: targetChangeKeys[count]]; + changeList = [NSData dataWithBinary: targetPredecessorChangeLists[count]]; messageKey = [NSString stringWithFormat: @"%@.eml", [destUIDs objectAtIndex: count]]; - [self setChangeKey: changeKey - forMessageWithKey: messageKey]; + [self _updatePredecessorChangeListWith: changeList + forMessageEntry: [self _messageEntryFromMessageKey: messageKey]]; } } diff --git a/OpenChange/NSData+MAPIStore.h b/OpenChange/NSData+MAPIStore.h index 5715b9128..13daa8aef 100644 --- a/OpenChange/NSData+MAPIStore.h +++ b/OpenChange/NSData+MAPIStore.h @@ -41,6 +41,8 @@ + (id) dataWithXID: (const struct XID *) xid; - (struct XID *) asXIDInMemCtx: (void *) memCtx; +- (struct SizedXid *) asSizedXidArrayInMemCtx: (void *) memCtx + with: (uint32_t *) length; + (id) dataWithChangeKeyGUID: (NSString *) guidString andCnt: (NSData *) globCnt; diff --git a/OpenChange/NSData+MAPIStore.m b/OpenChange/NSData+MAPIStore.m index 38ef1ccf4..5a2afeb1d 100644 --- a/OpenChange/NSData+MAPIStore.m +++ b/OpenChange/NSData+MAPIStore.m @@ -22,6 +22,7 @@ #import +#import "MAPIStoreTypes.h" #import "NSObject+MAPIStore.h" #import "NSString+MAPIStore.h" @@ -29,6 +30,7 @@ #undef DEBUG #include +#include #include #include #include @@ -172,6 +174,38 @@ static void _fillFlatUIDWithGUID (struct FlatUID_r *flatUID, const struct GUID * return xid; } +- (struct SizedXid *) asSizedXidArrayInMemCtx: (void *) memCtx + with: (uint32_t *) length +{ + struct Binary_r bin; + struct SizedXid *sizedXIDArray; + + bin.cb = [self length]; + bin.lpb = (uint8_t *)[self bytes]; + + sizedXIDArray = get_SizedXidArray(memCtx, &bin, length); + if (!sizedXIDArray) + { + NSLog (@"Impossible to parse SizedXID array"); + return NULL; + } + + return sizedXIDArray; +} + +- (NSComparisonResult) compare: (NSData *) otherGlobCnt +{ + uint64_t globCnt = 0, oGlobCnt = 0; + + if ([self length] > 0) + globCnt = *(uint64_t *) [self bytes]; + + if ([otherGlobCnt length] > 0) + oGlobCnt = *(uint64_t *) [otherGlobCnt bytes]; + + return MAPICNCompare (globCnt, oGlobCnt, NULL); +} + + (id) dataWithChangeKeyGUID: (NSString *) guidString andCnt: (NSData *) globCnt; { From 27b9b7bfa7a97635e0250a406eb4004078e3d571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Mon, 10 Aug 2015 18:38:39 +0200 Subject: [PATCH 04/13] oc-mail: Increase Change Number after modifying seen flag It is required when you are using SynchronizeImportReadStateChanges ROP to update the MetaTagCnsetRead meta property. See [MS-OXCFXICS] Section 3.2.5.9.4.6 This could lead to sync issues. --- OpenChange/MAPIStoreMailMessage.m | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/OpenChange/MAPIStoreMailMessage.m b/OpenChange/MAPIStoreMailMessage.m index 37d4fc420..08c12772a 100644 --- a/OpenChange/MAPIStoreMailMessage.m +++ b/OpenChange/MAPIStoreMailMessage.m @@ -1647,13 +1647,27 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) - (int) setReadFlag: (uint8_t) flag { + BOOL modified = NO; + BOOL alreadyRead = NO; NSString *imapFlag = @"\\Seen"; + alreadyRead = [[[sogoObject fetchCoreInfos] objectForKey: @"flags"] + containsObject: @"seen"]; + /* TODO: notifications should probably be emitted from here */ if (flag & CLEAR_READ_FLAG) - [sogoObject removeFlags: imapFlag]; + { + [sogoObject removeFlags: imapFlag]; + modified = alreadyRead; + } else - [sogoObject addFlags: imapFlag]; + { + [sogoObject addFlags: imapFlag]; + modified = !alreadyRead; + } + + if (modified) + [(MAPIStoreMailFolder *)[self container] synchroniseCache]; return MAPISTORE_SUCCESS; } From ae3ac0a09a1ac8df5d2df477b9d5a8168f742069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Amor=20Garc=C3=ADa?= Date: Wed, 19 Aug 2015 18:56:15 +0200 Subject: [PATCH 05/13] oc-mail: Support attachments with filename extended parameter The attachments which used a extended parameter for their filename ('filename*=') where silently dropped. This was because MAPIStore was only looking for no-extended filename parameter. The solution is using the 'filename' from the SOGOExtension of the NSDictionary interface. --- OpenChange/MAPIStoreMailAttachment.m | 14 +------------- OpenChange/MAPIStoreMailMessage.m | 4 +--- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/OpenChange/MAPIStoreMailAttachment.m b/OpenChange/MAPIStoreMailAttachment.m index 388ad924d..da7763095 100644 --- a/OpenChange/MAPIStoreMailAttachment.m +++ b/OpenChange/MAPIStoreMailAttachment.m @@ -117,19 +117,7 @@ - (NSString *) _fileName { - NSString *fileName; - NSDictionary *parameters; - - fileName = [[bodyInfo objectForKey: @"parameterList"] - objectForKey: @"name"]; - if (!fileName) - { - parameters = [[bodyInfo objectForKey: @"disposition"] - objectForKey: @"parameterList"]; - fileName = [parameters objectForKey: @"filename"]; - } - - return fileName; + return [bodyInfo filename]; } - (int) getPidTagAttachLongFilename: (void **) data diff --git a/OpenChange/MAPIStoreMailMessage.m b/OpenChange/MAPIStoreMailMessage.m index 37d4fc420..af9e38790 100644 --- a/OpenChange/MAPIStoreMailMessage.m +++ b/OpenChange/MAPIStoreMailMessage.m @@ -1569,9 +1569,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) NSDictionary *parameters; NSUInteger count, max; - parameters = [[bodyInfo objectForKey: @"disposition"] - objectForKey: @"parameterList"]; - if ([[parameters objectForKey: @"filename"] length] > 0) + if ([[bodyInfo filename] length] > 0) { if ([keyPrefix length] == 0) keyPrefix = @"0"; From 3a60b6e38edbff5edc0d2836745ef1da702f4237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Mon, 24 Aug 2015 12:30:50 +0200 Subject: [PATCH 06/13] oc-mail: Implement ChangeNumber >= restriction And warn when other operator than > or >= is used. This allows OpenChange to query for change numbers starting from a given point. --- OpenChange/MAPIStoreMailMessageTable.m | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/OpenChange/MAPIStoreMailMessageTable.m b/OpenChange/MAPIStoreMailMessageTable.m index c13605829..c71097d63 100644 --- a/OpenChange/MAPIStoreMailMessageTable.m +++ b/OpenChange/MAPIStoreMailMessageTable.m @@ -161,15 +161,30 @@ static Class MAPIStoreMailMessageK, NSDataK, NSStringK; //[self logWithFormat: @"change number from oxcfxics: %.16lx", [value unsignedLongLongValue]]; //[self logWithFormat: @" modseq: %.16lx", [modseq unsignedLongLongValue]]; if (modseq) - modseq = [NSNumber numberWithUnsignedLongLong: - [modseq unsignedLongLongValue] + 1]; + { + if (res->relop == RELOP_GT) + modseq = [NSNumber numberWithUnsignedLongLong: + [modseq unsignedLongLongValue] + 1]; + + } else modseq = [NSNumber numberWithUnsignedLongLong: 0]; - *qualifier = [[EOKeyValueQualifier alloc] initWithKey: @"MODSEQ" - operatorSelector: EOQualifierOperatorGreaterThanOrEqualTo - value: modseq]; - [*qualifier autorelease]; - rc = MAPIRestrictionStateNeedsEval; + + if (res->relop == RELOP_GT || res->relop == RELOP_GE) + { + *qualifier = [[EOKeyValueQualifier alloc] initWithKey: @"MODSEQ" + operatorSelector: EOQualifierOperatorGreaterThanOrEqualTo + value: modseq]; + [*qualifier autorelease]; + rc = MAPIRestrictionStateNeedsEval; + } + else + { + /* Ignore other operations as IMAP only support MODSEQ >= X */ + [self warnWithFormat: @"Ignoring %@ as only supported operators are > and >=", + [self operatorFromRestrictionOperator: res->relop]]; + rc = MAPIRestrictionStateAlwaysTrue; + } } break; From 47859b76d62af1462dc9a8a9e67ed68780432798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Mon, 24 Aug 2015 23:42:38 +0200 Subject: [PATCH 07/13] oc-mail: Return right change key after saving a draft mail After saving a draft mail (this is done automatically by Outlook) a GetProps call is done checking the PidTagChangeKey has been updated properly. Without this patch, it returned MAPI_E_NOT_FOUND. With this patch, we addressed that problem and we have updated the Predecessor Change List metadata for the draft mail with the change key provided by the client to avoid conflicting messages whenever it is possible. --- OpenChange/MAPIStoreMailFolder.h | 2 ++ OpenChange/MAPIStoreMailFolder.m | 38 +++++++++++++++++++++++ OpenChange/MAPIStoreMailVolatileMessage.m | 36 ++++++++++++++------- OpenChange/MAPIStoreMessage.m | 3 +- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/OpenChange/MAPIStoreMailFolder.h b/OpenChange/MAPIStoreMailFolder.h index c56e74862..717868efb 100644 --- a/OpenChange/MAPIStoreMailFolder.h +++ b/OpenChange/MAPIStoreMailFolder.h @@ -52,6 +52,8 @@ - (NSString *) changeNumberForMessageUID: (NSString *) messageUid; - (void) setChangeKey: (NSData *) changeKey forMessageWithKey: (NSString *) messageKey; +- (BOOL) updatePredecessorChangeListWith: (NSData *) changeKey + forMessageWithKey: (NSString *) messageKey; - (NSData *) changeKeyForMessageWithKey: (NSString *) messageKey; - (NSData *) predecessorChangeListForMessageWithKey: (NSString *) messageKey; diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 963fb7463..06a7d039d 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -999,6 +999,44 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) [versionsMessage save]; } +- (BOOL) updatePredecessorChangeListWith: (NSData *) changeKey + forMessageWithKey: (NSString *) messageKey +{ + /* Update predecessor change list property given the change key. It + returns if the change key has been added to the list or not */ + BOOL added = NO; + NSData *globCnt, *oldGlobCnt; + NSDictionary *messageEntry; + NSMutableDictionary *changeList; + NSString *guid; + struct XID *xid; + + xid = [changeKey asXIDInMemCtx: NULL]; + guid = [NSString stringWithGUID: &xid->NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid->LocalId.data length: xid->LocalId.length]; + talloc_free (xid); + + messageEntry = [self _messageEntryFromMessageKey: messageKey]; + if (messageEntry) + { + changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; + if (changeList) + { + oldGlobCnt = [changeList objectForKey: guid]; + if (!oldGlobCnt || ([globCnt compare: oldGlobCnt] == NSOrderedDescending)) + { + [changeList setObject: globCnt forKey: guid]; + [versionsMessage save]; + added = YES; + } + } + else + [self errorWithFormat: @"Missing predecessor change list to update"]; + } + + return added; +} + - (NSData *) changeKeyForMessageWithKey: (NSString *) messageKey { NSDictionary *messages, *changeKeyDict; diff --git a/OpenChange/MAPIStoreMailVolatileMessage.m b/OpenChange/MAPIStoreMailVolatileMessage.m index f2581a54a..8a7ba2c58 100644 --- a/OpenChange/MAPIStoreMailVolatileMessage.m +++ b/OpenChange/MAPIStoreMailVolatileMessage.m @@ -34,6 +34,7 @@ #import #import #import +#import #import #import #import @@ -285,7 +286,7 @@ static NSString *recTypes[] = { @"orig", @"to", @"cc", @"bcc" }; version = [properties objectForKey: @"version"]; return (version - ? exchange_globcnt ([version unsignedLongLongValue]) + ? [version unsignedLongLongValue] : ULLONG_MAX); } @@ -1110,7 +1111,8 @@ MakeMessageBody (NSDictionary *mailProperties, NSDictionary *attachmentParts, NS - (void) save: (TALLOC_CTX *) memCtx { - NSString *folderName, *flag, *newIdString, *messageKey; + BOOL updatedMetadata; + NSString *folderName, *flag, *newIdString, *messageKey, *changeNumber; NSData *changeKey, *messageData; NGImap4Connection *connection; NGImap4Client *client; @@ -1146,21 +1148,33 @@ MakeMessageBody (NSDictionary *mailProperties, NSDictionary *attachmentParts, NS [sogoObject setNameInContainer: messageKey]; [mapping registerURL: [self url] withID: mid]; - /* synchronise the cache and update the change key with the one provided - by the client. Before doing this, lets issue a unselect/select combo - because of timing issues with Dovecot in obtaining the latest modseq. - Sometimes, Dovecot doesn't return the newly appended UID if we do - a "UID SORT (DATE) UTF-8 (MODSEQ XYZ) (NOT DELETED)" command (where - XYZ is the HIGHESTMODSEQ+1) immediately after IMAP APPEND */ + /* synchronise the cache and update the predecessor change list + with the change key provided by the client. Before doing + this, lets issue a unselect/select combo because of timing + issues with Dovecot in obtaining the latest modseq. + Sometimes, Dovecot doesn't return the newly appended UID if + we do a "UID SORT (DATE) UTF-8 (MODSEQ XYZ) (NOT DELETED)" + command (where XYZ is the HIGHESTMODSEQ+1) immediately after + IMAP APPEND */ [client unselect]; [client select: folderName]; [(MAPIStoreMailFolder *) container synchroniseCache]; changeKey = [properties objectForKey: MAPIPropertyKey (PR_CHANGE_KEY)]; if (changeKey) - [(MAPIStoreMailFolder *) container - setChangeKey: changeKey - forMessageWithKey: messageKey]; + { + updatedMetadata = [(MAPIStoreMailFolder *) container updatePredecessorChangeListWith: changeKey + forMessageWithKey: messageKey]; + if (!updatedMetadata) + [self warnWithFormat: @"Predecessor change list not updated with client data"]; + } + + /* Update version property (PR_CHANGE_KEY indeed) as it is + requested once it is saved */ + changeNumber = [(MAPIStoreMailFolder *) container changeNumberForMessageUID: newIdString]; + if (changeNumber) + [properties setObject: [NSNumber numberWithUnsignedLongLong: [changeNumber unsignedLongLongValue] >> 16] + forKey: @"version"]; } } diff --git a/OpenChange/MAPIStoreMessage.m b/OpenChange/MAPIStoreMessage.m index b842b2d6c..a5febc41c 100644 --- a/OpenChange/MAPIStoreMessage.m +++ b/OpenChange/MAPIStoreMessage.m @@ -549,11 +549,12 @@ rtf2html (NSData *compressedRTF) } [self save: memCtx]; - /* We make sure that any change-related properties are removes from the + /* We make sure that any change-related properties are removed from the properties dictionary, to make sure that related methods will be invoked the next time they are requested. */ [properties removeObjectForKey: MAPIPropertyKey (PidTagChangeKey)]; [properties removeObjectForKey: MAPIPropertyKey (PidTagChangeNumber)]; + [properties removeObjectForKey: MAPIPropertyKey (PidTagPredecessorChangeList)]; if ([container isKindOfClass: MAPIStoreFolderK]) { From 361d5ea3a605c94344ceea1a2e34e91c416db837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Amor=20Garc=C3=ADa?= Date: Sun, 30 Aug 2015 14:13:05 +0200 Subject: [PATCH 08/13] Added method [SOGoTest stringFromDiffBetween:and:] --- Tests/Unit/SOGoTest.h | 2 ++ Tests/Unit/SOGoTest.m | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/Tests/Unit/SOGoTest.h b/Tests/Unit/SOGoTest.h index 927cc27a9..ce555ea1a 100644 --- a/Tests/Unit/SOGoTest.h +++ b/Tests/Unit/SOGoTest.h @@ -52,6 +52,8 @@ - (BOOL) run; +- (NSString*) stringFromDiffBetween: (NSString*) str1 + and: (NSString*) str2; @end #define test(c) { \ diff --git a/Tests/Unit/SOGoTest.m b/Tests/Unit/SOGoTest.m index 1d548543e..ddee98f30 100644 --- a/Tests/Unit/SOGoTest.m +++ b/Tests/Unit/SOGoTest.m @@ -185,4 +185,79 @@ static NSString *SOGoTestAssertException = @"SOGoTestAssertException"; return YES; } +/* Helper function for diffForString:andString */ +NSString *_stringForCharacterAtIndex(NSUInteger index, NSString *str, NSUInteger length) +{ + NSString *chrStr; + unichar chr; + if (index < length) + { + chr = [str characterAtIndex: index]; + if (isprint(chr)) + { + chrStr = [NSString stringWithFormat: @"%c", chr]; + } + else + { + if (chr == 10) + chrStr = @"[NL]"; + else if (chr == 0) + chrStr = @"[\0]"; + else + chrStr = [NSString stringWithFormat: @"[NP: %u]", chr]; + } + } + else + { + chrStr = @"[none]"; + } + + return chrStr; +} + +/* + Returns a string with a very verbose diff of the two strings. + In case the strings are equal it returns an empty string. + Example output for the strings 'flower' and 'flotera': + +0 |f| +1 |l| +2 |o| +3 |w|t|<-- +4 |e| +5 |r| +6 |[none]|a|<-- + +*/ +- (NSString*) stringFromDiffBetween: (NSString*) str1 + and: (NSString*) str2 +{ + BOOL differencesFound = NO; + NSString *finalSTR = @""; + NSUInteger i, length1, length2; + NSString *sc1, *sc2; + + length1 = [str1 length]; + length2 = [str2 length]; + for (i = 0; i < length1 || i < length2; i++) + { + sc1 = _stringForCharacterAtIndex(i, str1, length1); + sc2 = _stringForCharacterAtIndex(i, str2, length2); + + if ([sc1 isEqualToString: sc2]) + finalSTR = [finalSTR stringByAppendingFormat: @"%u |%@|\n", i, sc1]; + else + { + finalSTR = [finalSTR stringByAppendingFormat: @"%u |%@|%@|<--\n", i, sc1, sc2]; + differencesFound = YES; + } + } + + if (!differencesFound) + return @""; + + return finalSTR; +} + + @end From cbc3e3e97dc9a56ac867894d9b7b131fb4de137b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Amor=20Garc=C3=ADa?= Date: Tue, 25 Aug 2015 16:41:38 +0200 Subject: [PATCH 09/13] Test for [NGMimeMessageGenerator generateDataForHeaderField:value:] The NGMimeMessageGenerator can be found in sope --- Tests/Unit/GNUmakefile | 2 + Tests/Unit/SOGoTest.h | 2 + Tests/Unit/TestNGMimeMessageGenerator.m | 134 ++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 Tests/Unit/TestNGMimeMessageGenerator.m diff --git a/Tests/Unit/GNUmakefile b/Tests/Unit/GNUmakefile index bcfd13850..70f3ff242 100644 --- a/Tests/Unit/GNUmakefile +++ b/Tests/Unit/GNUmakefile @@ -22,6 +22,8 @@ $(TEST_TOOL)_OBJC_FILES += \ TestSBJsonParser.m \ \ TestNGMimeAddressHeaderFieldGenerator.m \ + TestNGMimeMessageGenerator.m \ + \ TestNSData+Crypto.m \ TestNSString+Crypto.m \ TestNSString+URLEscaping.m \ diff --git a/Tests/Unit/SOGoTest.h b/Tests/Unit/SOGoTest.h index ce555ea1a..ed49be707 100644 --- a/Tests/Unit/SOGoTest.h +++ b/Tests/Unit/SOGoTest.h @@ -25,6 +25,8 @@ #import #import +#import +#import #import @class NSArray; diff --git a/Tests/Unit/TestNGMimeMessageGenerator.m b/Tests/Unit/TestNGMimeMessageGenerator.m new file mode 100644 index 000000000..51b4e4745 --- /dev/null +++ b/Tests/Unit/TestNGMimeMessageGenerator.m @@ -0,0 +1,134 @@ +/* TestNGMimeMessageGenerator.m - this file is part of SOGo + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#import "SOGoTest.h" +#import + +@interface TestNGMimeMessageGenerator : SOGoTest +@end + +@implementation TestNGMimeMessageGenerator + +- (void) test_generateDataForHeaderField_value +{ + NGMimeMessageGenerator *generator; + NSArray *cases = [NSArray arrayWithObjects: + [NSArray arrayWithObjects: @"Message-ID", @"", @"", nil], + [NSArray arrayWithObjects: @"Content-Type", + @"text/plain; charset=utf-8; format=flowed", + @"text/plain; charset=utf-8; format=flowed", + nil], + [NSArray arrayWithObjects: @"X-FullHeaderOneHebrewOneLatin", + @"עs", + @"=?utf-8?q?=D7=A2s?=", + nil], + [NSArray arrayWithObjects: @"X-FullHeaderOneLatineOneHebrew", + @"sע", + @"=?utf-8?q?s=D7=A2?=", + nil], + [NSArray arrayWithObjects: @"X-FullHeaderOneCharacterHebrew", + @"ע", + @"=?utf-8?q?=D7=A2?=", + nil], + [NSArray arrayWithObjects: @"X-FullHeaderOneCharacterRussian", + @"Б", + @"=?utf-8?q?=D0=91?=", + nil], + + [NSArray arrayWithObjects: @"X-FullHeaderParameter", + @"parameter=ע", + @"parameter==?utf-8?q?=D7=A2?=", + nil], + [NSArray arrayWithObjects: @"X-MixedHeaderParameters", + @"plain; parameter=ע; parameter-plain; parameter2=ea", + @"plain;\n parameter==?utf-8?q?=D7=A2?=;\n parameter-plain; parameter2=ea", + nil], + [NSArray arrayWithObjects: @"X-MixedHeaderAndNoParameter", + @"plain; parameter=ע; parameter-plain; ע", + @"plain;\n parameter==?utf-8?q?=D7=A2?=; parameter-plain;\n =?utf-8?q?=D7=A2?=", + nil], + + [NSArray arrayWithObjects: @"X-MixedHeaderAndTwoParameter", + @"plain; parameter=ע; parameter-plain; z=ע", + @"plain;\n parameter==?utf-8?q?=D7=A2?=; parameter-plain;\n z==?utf-8?q?=D7=A2?=", + nil], + [NSArray arrayWithObjects: @"X-MixedHeaderExtrablanks", + @"plain; parameter=ע; parameter 2spaces; parameter2=ea", + @"plain;\n \\ parameter==?utf-8?q?=D7=A2?=;\n parameter 2spaces; parameter2=ea", + nil], + [NSArray arrayWithObjects: @"X-Encoded-Unbalanced-Paramter-Quote", + @"text/plain; name=\"ע", + @"text/plain;\n name==?utf-8?q?=22=D7=A2?=", + nil], + [NSArray arrayWithObjects: @"content-type", + @"text/plain; name=\"АБВГДЕЁЖЗИЙ, КЛМНОПРСТУФ y ЦЧШЩЪЫЬЭЮЯ.txt\"", + @"text/plain;\n name=\"=?utf-8?q?=D0=90=D0=91=D0=92=D0=93=D0=94=D0=95=D0=81=D0=96=D0=97=D0=98=D0=99=2C_=D0=9A=D0=9B=D0=9C=D0=9D=D0=9E=D0=9F=D0=A0=D0=A1=D0=A2=D0=A3=D0=A4_y_=D0=A6=D0=A7=D0=A8=D0=A9=D0=AA=D0=AB=D0=AC=D0=AD=D0=AE=D0=AF=2Etxt?=\"", + nil], + [NSArray arrayWithObjects: @"content-disposition", + @"attachment; filename=\"АБВГДЕЁЖЗИЙ, КЛМНОПРСТУФ y ЦЧШЩЪЫЬЭЮЯ.txt\"", + @"attachment;\n filename=\"=?utf-8?q?=D0=90=D0=91=D0=92=D0=93=D0=94=D0=95=D0=81=D0=96=D0=97=D0=98=D0=99=2C_=D0=9A=D0=9B=D0=9C=D0=9D=D0=9E=D0=9F=D0=A0=D0=A1=D0=A2=D0=A3=D0=A4_y_=D0=A6=D0=A7=D0=A8=D0=A9=D0=AA=D0=AB=D0=AC=D0=AD=D0=AE=D0=AF=2Etxt?=\"", + nil], + [NSArray arrayWithObjects: @"content-length", @"2912", @"2912", nil], + [NSArray arrayWithObjects: @"content-transfer-encoding", @"quoted-printable", @"quoted-printable", nil], + nil + ]; + NSEnumerator *enumerator; + NSArray *testCase; + + [NGMimeMessageGenerator initialize]; + generator = [[NGMimeMessageGenerator alloc] init]; + [generator autorelease]; + + enumerator = [cases objectEnumerator]; + while ((testCase = [enumerator nextObject]) != nil) + { + NSData *result; + NSMutableData *resultWithNulByte; + NSString *header = [testCase objectAtIndex: 0]; + NSData *headerData = [testCase objectAtIndex: 1]; + NSString *expected = [testCase objectAtIndex: 2]; + result = [generator generateDataForHeaderField: header + value: headerData]; + if (result == nil) + result = [@"[nil]" dataUsingEncoding: NSUTF8StringEncoding]; + + resultWithNulByte = [result mutableCopy]; + [resultWithNulByte appendBytes: "\0" length: 1]; + NSString *resultString = [NSString stringWithCString:[resultWithNulByte bytes]]; + + + BOOL testResult = [resultString isEqualToString: expected]; + + NSString *diff = [self stringFromDiffBetween: [NSString stringWithString: resultString] + and: [NSString stringWithString: expected]]; + NSString *testErrorMsg = [NSString + stringWithFormat: @">> For %@ header received:\n%@[END]\n>> instead of:\n%@[END]\n>> for:\n%@\n>> diff:\n%@\n>> lengthReceived: %u lengthExpected: %u", + header, + resultString, + expected, + headerData, + diff, + [resultString length], + [expected length] + ]; + + testWithMessage(testResult, testErrorMsg); + } +} + +@end From b702e40ebbc78302fa7f1e29c02e5bc5af9ecb83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Thu, 3 Sep 2015 16:31:40 +0200 Subject: [PATCH 10/13] oc: load versionsMessage on setting up versionsMessage object could have outdated version in a root folder in the following case: * Download latest contents using FXBuffer * versionsMessage is updated by synchroniseCache * OpenMessage from last FXBuffer * Setup versions message as root folder * Get Predecessor Change List from that message We could just reload if needed the versions message if something is missing but I don't know if that situation fixes more than this one. --- OpenChange/MAPIStoreGCSFolder.m | 1 + OpenChange/MAPIStoreMailFolder.m | 1 + 2 files changed, 2 insertions(+) diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index 55e4f52e1..8781ef22b 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -75,6 +75,7 @@ static Class NSNumberK; [SOGoMAPIDBMessage objectWithName: @"versions.plist" inContainer: dbFolder]); [versionsMessage setObjectType: MAPIInternalCacheObject]; + [versionsMessage reloadIfNeeded]; } - (void) dealloc diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 06a7d039d..9373bb0e5 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -111,6 +111,7 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; [SOGoMAPIDBMessage objectWithName: @"versions.plist" inContainer: dbFolder]); [versionsMessage setObjectType: MAPIInternalCacheObject]; + [versionsMessage reloadIfNeeded]; } - (BOOL) ensureFolderExists From f19074334cabf8b78b2818e0c0a3547e9a1b9b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 4 Sep 2015 16:15:05 +0200 Subject: [PATCH 11/13] oc: Use enum mapistore_error as returned value for setReadFlag --- OpenChange/MAPIStoreCalendarMessage.m | 2 +- OpenChange/MAPIStoreMailMessage.m | 2 +- OpenChange/MAPIStoreMessage.h | 2 +- OpenChange/MAPIStoreMessage.m | 2 +- OpenChange/MAPIStoreSOGo.m | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/OpenChange/MAPIStoreCalendarMessage.m b/OpenChange/MAPIStoreCalendarMessage.m index 79a11e58b..7f025e400 100644 --- a/OpenChange/MAPIStoreCalendarMessage.m +++ b/OpenChange/MAPIStoreCalendarMessage.m @@ -671,7 +671,7 @@ static Class NSArrayK, MAPIStoreAppointmentWrapperK; return newAttachment; } -- (int) setReadFlag: (uint8_t) flag +- (enum mapistore_error) setReadFlag: (uint8_t) flag { return MAPISTORE_SUCCESS; } diff --git a/OpenChange/MAPIStoreMailMessage.m b/OpenChange/MAPIStoreMailMessage.m index af9e38790..3b482d6f8 100644 --- a/OpenChange/MAPIStoreMailMessage.m +++ b/OpenChange/MAPIStoreMailMessage.m @@ -1643,7 +1643,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) return attachment; } -- (int) setReadFlag: (uint8_t) flag +- (enum mapistore_error) setReadFlag: (uint8_t) flag { NSString *imapFlag = @"\\Seen"; diff --git a/OpenChange/MAPIStoreMessage.h b/OpenChange/MAPIStoreMessage.h index 6fa42218c..f914e543a 100644 --- a/OpenChange/MAPIStoreMessage.h +++ b/OpenChange/MAPIStoreMessage.h @@ -63,7 +63,7 @@ withAID: (uint32_t) aid; - (int) getAttachmentTable: (MAPIStoreAttachmentTable **) tablePtr andRowCount: (uint32_t *) countPtr; -- (int) setReadFlag: (uint8_t) flag; +- (enum mapistore_error) setReadFlag: (uint8_t) flag; - (enum mapistore_error) saveMessage: (TALLOC_CTX *) memCtx; - (NSArray *) activeContainerMessageTables; diff --git a/OpenChange/MAPIStoreMessage.m b/OpenChange/MAPIStoreMessage.m index a5febc41c..e6bda4578 100644 --- a/OpenChange/MAPIStoreMessage.m +++ b/OpenChange/MAPIStoreMessage.m @@ -919,7 +919,7 @@ rtf2html (NSData *compressedRTF) return [self getNo: data inMemCtx: memCtx];; } -- (int) setReadFlag: (uint8_t) flag +- (enum mapistore_error) setReadFlag: (uint8_t) flag { // [self subclassResponsibility: _cmd]; diff --git a/OpenChange/MAPIStoreSOGo.m b/OpenChange/MAPIStoreSOGo.m index 0923c44b5..9ccec826c 100644 --- a/OpenChange/MAPIStoreSOGo.m +++ b/OpenChange/MAPIStoreSOGo.m @@ -1120,7 +1120,7 @@ sogo_message_set_read_flag (void *message_object, uint8_t flag) struct MAPIStoreTallocWrapper *wrapper; NSAutoreleasePool *pool; MAPIStoreMessage *message; - int rc; + enum mapistore_error rc; if (message_object) { From 6280e4ded620658c80b56de91a3733084ff08d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 4 Sep 2015 16:16:00 +0200 Subject: [PATCH 12/13] oc: Implement setReadFlag for MAPIStoreDBMessage This is an utility for testing as I don't see any added value for real scenario but according to [MS-OXCMSG] all messages can have PidTagMessageFlags. --- OpenChange/MAPIStoreDBMessage.m | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index cdfe00cd8..7c208b2f8 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -313,4 +313,36 @@ return [sogoObject lastModified]; } +- (enum mapistore_error) setReadFlag: (uint8_t) flag +{ + /* Modify PidTagMessageFlags from SetMessageReadFlag and + SyncImportReadStateChanges ROPs */ + NSNumber *flags; + uint32_t newFlag; + + flags = [properties objectForKey: MAPIPropertyKey (PR_MESSAGE_FLAGS)]; + if (flags) + { + newFlag = [flags unsignedLongValue]; + if (flag & SUPPRESS_RECEIPT) + newFlag |= MSGFLAG_READ; + if (flag & CLEAR_RN_PENDING) + newFlag &= ~MSGFLAG_RN_PENDING; + if (flag & CLEAR_READ_FLAG) + newFlag &= ~MSGFLAG_READ; + if (flag & CLEAR_NRN_PENDING) + newFlag &= ~MSGFLAG_NRN_PENDING; + } + else + { + newFlag = MSGFLAG_READ; + if (flag & CLEAR_READ_FLAG) + newFlag = 0x0; + } + [properties setObject: [NSNumber numberWithUnsignedLong: newFlag] + forKey: MAPIPropertyKey (PR_MESSAGE_FLAGS)]; + + return MAPISTORE_SUCCESS; +} + @end From 1fdb44b0a7c2b8f4d9011633b7888be23b47af51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Amor=20Garc=C3=ADa?= Date: Tue, 8 Sep 2015 15:37:33 +0200 Subject: [PATCH 13/13] Fix some compilation warnings The warnings were introduced in the fix for attachment with used the filename extended parameter. Also removed trailing whitespaces. --- OpenChange/MAPIStoreMailAttachment.m | 5 ++-- OpenChange/MAPIStoreMailMessage.m | 40 ++++++++++++++-------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/OpenChange/MAPIStoreMailAttachment.m b/OpenChange/MAPIStoreMailAttachment.m index da7763095..575808f39 100644 --- a/OpenChange/MAPIStoreMailAttachment.m +++ b/OpenChange/MAPIStoreMailAttachment.m @@ -30,6 +30,7 @@ #import #import #import +#import #import "MAPIStoreTypes.h" #import "MAPIStoreMailMessage.h" @@ -108,7 +109,7 @@ static char recordBytes[] = {0xd9, 0xd8, 0x11, 0xa3, 0xe2, 0x90, 0x18, 0x41, 0x9e, 0x04, 0x58, 0x46, 0x9d, 0x6d, 0x1b, 0x68}; - + *data = [[NSData dataWithBytes: recordBytes length: 16] asBinaryInMemCtx: memCtx]; @@ -166,7 +167,7 @@ - (int) getPidTagAttachContentId: (void **) data inMemCtx: (TALLOC_CTX *) memCtx -{ +{ *data = [[bodyInfo objectForKey: @"bodyId"] asUnicodeInMemCtx: memCtx]; diff --git a/OpenChange/MAPIStoreMailMessage.m b/OpenChange/MAPIStoreMailMessage.m index bbf7d83b5..7ae2bd1d9 100644 --- a/OpenChange/MAPIStoreMailMessage.m +++ b/OpenChange/MAPIStoreMailMessage.m @@ -40,6 +40,7 @@ #import #import #import +#import #import "Codepages.h" #import "NSData+MAPIStore.h" @@ -196,7 +197,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) count1 = [keys indexOfObject: data1]; data2 = [entry2 objectForKey: @"mimeType"]; count2 = [keys indexOfObject: data2]; - + if (count1 == count2) { data1 = [entry1 objectForKey: @"key"]; @@ -529,7 +530,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) else stringValue = @""; *data = [stringValue asUnicodeInMemCtx: memCtx]; - + return MAPISTORE_SUCCESS; } @@ -624,7 +625,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) NSDictionary *coreInfos; NSArray *flags; unsigned int v = 0; - + coreInfos = [sogoObject fetchCoreInfos]; flags = [coreInfos objectForKey: @"flags"]; @@ -636,7 +637,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) if ([[self attachmentKeys] count] > 0) v |= MSGFLAG_HASATTACH; - + *data = MAPILongValue (memCtx, v); return MAPISTORE_SUCCESS; @@ -656,7 +657,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) v = 2; else v = 0; - + *data = MAPILongValue (memCtx, v); return MAPISTORE_SUCCESS; @@ -668,15 +669,15 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) NSDictionary *coreInfos; NSArray *flags; unsigned int v; - + coreInfos = [sogoObject fetchCoreInfos]; - + flags = [coreInfos objectForKey: @"flags"]; if ([flags containsObject: @"flagged"]) v = 6; else v = 0; - + *data = MAPILongValue (memCtx, v); return MAPISTORE_SUCCESS; @@ -755,7 +756,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) if ([ngAddress isKindOfClass: [NGMailAddress class]]) { cn = [ngAddress displayName]; - + // If we don't have a displayName, we use the email address instead. This // avoid bug #2119 - where Outlook won't display anything in the "From" field, // nor in the recipient field if we reply to the email. @@ -809,7 +810,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) entryId = MAPIStoreExternalEntryId (cn, email); *data = [entryId asBinaryInMemCtx: memCtx]; - + rc = MAPISTORE_SUCCESS; } else @@ -966,15 +967,15 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) { uint32_t v; NSString *s; - + s = [[sogoObject mailHeaders] objectForKey: @"x-priority"]; v = 0x1; - + if ([s hasPrefix: @"1"]) v = 0x2; else if ([s hasPrefix: @"2"]) v = 0x2; else if ([s hasPrefix: @"4"]) v = 0x0; else if ([s hasPrefix: @"5"]) v = 0x0; - + *data = MAPILongValue (memCtx, v); return MAPISTORE_SUCCESS; @@ -1163,14 +1164,14 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) if (!headerSetup) [self _fetchHeaderData]; - + if ([headerMimeType isEqualToString: @"text/plain"]) format = EDITOR_FORMAT_PLAINTEXT; else if ([headerMimeType isEqualToString: @"text/html"]) format = EDITOR_FORMAT_HTML; else format = 0; /* EDITOR_FORMAT_DONTKNOW */ - + *data = MAPILongValue (memCtx, format); return MAPISTORE_SUCCESS; @@ -1517,7 +1518,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) p = 0; recipient->data = talloc_array (msgData, void *, msgData->columns->cValues); memset (recipient->data, 0, msgData->columns->cValues * sizeof (void *)); - + // PR_OBJECT_TYPE = MAPI_MAILUSER (see MAPI_OBJTYPE) recipient->data[p] = MAPILongValue (msgData, MAPI_MAILUSER); p++; @@ -1525,7 +1526,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) // PR_DISPLAY_TYPE = DT_MAILUSER (see MS-NSPI) recipient->data[p] = MAPILongValue (msgData, 0); p++; - + // PR_7BIT_DISPLAY_NAME_UNICODE recipient->data[p] = [cn asUnicodeInMemCtx: msgData]; p++; @@ -1533,7 +1534,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) // PR_SMTP_ADDRESS_UNICODE recipient->data[p] = [email asUnicodeInMemCtx: msgData]; p++; - + // PR_SEND_INTERNET_ENCODING = 0x00060000 (plain text, see OXCMAIL) recipient->data[p] = MAPILongValue (msgData, 0x00060000); p++; @@ -1566,7 +1567,6 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) withPrefix: (NSString *) keyPrefix { NSArray *parts; - NSDictionary *parameters; NSUInteger count, max; if ([[bodyInfo filename] length] > 0) @@ -1706,7 +1706,7 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) return nil; } -- (void) save: (TALLOC_CTX *) memCtx +- (void) save: (TALLOC_CTX *) memCtx { NSNumber *value;