From 321672e2c3b5ed8468c5f530a7240a4f72945f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Mon, 20 Jul 2015 11:17:00 +0200 Subject: [PATCH] oc: Update predecessor change list on saving There were cases where only the change key was updated (GCS) or others were the change key was updated with wrong info. This changeset has as goal to update the predecessor change list and, change key if required, on saving taking into account the latest information given by the client in high level ROPs such as ImportMessageMove or SetProperties, and merge it with information provided by the server backend (IMAP server, SOGo DB) using `synchroniseCache`. For more details about `PidTagChangeKey` and `PidTagPredecessorChangeList` property values check [MS-OXCFXICS] Section 2.2.1.2 --- OpenChange/MAPIStoreDBMessage.m | 104 +++++++++++++++++++++++++++++++ OpenChange/MAPIStoreFolder.m | 11 ++-- OpenChange/MAPIStoreGCSFolder.h | 3 +- OpenChange/MAPIStoreGCSFolder.m | 98 ++++++++++++++++++++++++----- OpenChange/MAPIStoreGCSMessage.m | 9 ++- OpenChange/MAPIStoreMailFolder.m | 63 ++++++++++++++++--- OpenChange/NSData+MAPIStore.h | 2 + OpenChange/NSData+MAPIStore.m | 34 ++++++++++ 8 files changed, 292 insertions(+), 32 deletions(-) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index 077f2c708..cdfe00cd8 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -22,6 +22,7 @@ #import #import +#import #import #import #import @@ -34,6 +35,7 @@ #import "MAPIStoreDBFolder.h" #import "MAPIStoreDBMessage.h" #import "MAPIStoreTypes.h" +#import "NSData+MAPIStore.h" #import "NSObject+MAPIStore.h" #import "NSString+MAPIStore.h" @@ -104,6 +106,105 @@ return objectVersion; } +- (void) _updatePredecessorChangeList +{ + BOOL updated; + enum mapistore_error rc; + NSData *currentChangeList, *changeKey; + NSMutableArray *changeKeys; + NSMutableData *newChangeList; + NSUInteger count, len; + struct SizedXid *changes; + struct SPropValue property; + struct SRow aRow; + struct XID *currentChangeKey; + TALLOC_CTX *localMemCtx; + uint32_t nChanges; + + localMemCtx = talloc_new (NULL); + if (!localMemCtx) + { + [self errorWithFormat: @"No more memory"]; + return; + } + + changeKey = [self getReplicaKeyFromGlobCnt: [self objectVersion]]; + + currentChangeList = [properties objectForKey: MAPIPropertyKey (PidTagPredecessorChangeList)]; + if (!currentChangeList) + { + /* Create a new PredecessorChangeList */ + len = [changeKey length]; + newChangeList = [NSMutableData dataWithCapacity: len + 1]; + [newChangeList appendUInt8: len]; + [newChangeList appendData: changeKey]; + } + else + { + /* Update current predecessor change list with new change key */ + changes = [currentChangeList asSizedXidArrayInMemCtx: localMemCtx + with: &nChanges]; + + updated = NO; + currentChangeKey = [changeKey asXIDInMemCtx: localMemCtx]; + for (count = 0; count < nChanges && !updated; count++) + { + if (GUID_equal(&changes[count].XID.NameSpaceGuid, ¤tChangeKey->NameSpaceGuid)) + { + NSData *globCnt, *oldGlobCnt; + oldGlobCnt = [NSData dataWithBytes: changes[count].XID.LocalId.data length: changes[count].XID.LocalId.length]; + globCnt = [NSData dataWithBytes: currentChangeKey->LocalId.data length: currentChangeKey->LocalId.length]; + if ([globCnt compare: oldGlobCnt] == NSOrderedDescending) + { + if ([globCnt length] != [oldGlobCnt length]) + { + [self errorWithFormat: @"Cannot compare globcnt with different length: %@ and %@", globCnt, oldGlobCnt]; + abort(); + } + memcpy (changes[count].XID.LocalId.data, currentChangeKey->LocalId.data, currentChangeKey->LocalId.length); + updated = YES; + } + } + } + + /* Serialise it */ + changeKeys = [NSMutableArray array]; + + if (!updated) + [changeKeys addObject: changeKey]; + + for (count = 0; count < nChanges; count++) + { + changeKey = [NSData dataWithXID: &changes[count].XID]; + [changeKeys addObject: changeKey]; + } + + [changeKeys sortUsingFunction: MAPIChangeKeyGUIDCompare context: localMemCtx]; + + newChangeList = [NSMutableData data]; + len = [changeKeys count]; + for (count = 0; count < len; count++) + { + changeKey = [changeKeys objectAtIndex: count]; + [newChangeList appendUInt8: [changeKey length]]; + [newChangeList appendData: changeKey]; + } + } + + if ([newChangeList length] > 0) + { + property.ulPropTag = PidTagPredecessorChangeList; + property.value.bin = *[newChangeList asBinaryInMemCtx: localMemCtx]; + aRow.cValues = 1; + aRow.lpProps = &property; + rc = [self addPropertiesFromRow: &aRow]; + if (rc != MAPISTORE_SUCCESS) + [self errorWithFormat: @"Impossible to add a new predecessor change list: %d", rc]; + } + + talloc_free (localMemCtx); +} + // // FIXME: how this can happen? // @@ -166,6 +267,9 @@ [properties setObject: [NSNumber numberWithUnsignedLongLong: newVersion] forKey: @"version"]; + /* Update PredecessorChangeList accordingly */ + [self _updatePredecessorChangeList]; + [self logWithFormat: @"%d props in dict", [properties count]]; [sogoObject save]; diff --git a/OpenChange/MAPIStoreFolder.m b/OpenChange/MAPIStoreFolder.m index ac3b19bf3..735c04cd6 100644 --- a/OpenChange/MAPIStoreFolder.m +++ b/OpenChange/MAPIStoreFolder.m @@ -670,15 +670,18 @@ Class NSExceptionK, MAPIStoreFAIMessageK, MAPIStoreMessageTableK, MAPIStoreFAIMe [sourceMsg copyToMessage: destMsg inMemCtx: memCtx]; - if (targetChangeKey) + if (targetPredecessorChangeList) { - property.ulPropTag = PidTagChangeKey; - property.value.bin = *targetChangeKey; + property.ulPropTag = PidTagPredecessorChangeList; + property.value.bin = *targetPredecessorChangeList; aRow.cValues = 1; aRow.lpProps = &property; rc = [destMsg addPropertiesFromRow: &aRow]; if (rc != MAPISTORE_SUCCESS) - goto end; + { + [self errorWithFormat: @"Cannot add PredecessorChangeList on move"]; + goto end; + } } [destMsg save: memCtx]; if (!wantCopy) diff --git a/OpenChange/MAPIStoreGCSFolder.h b/OpenChange/MAPIStoreGCSFolder.h index d4e3660f1..ecf9cee18 100644 --- a/OpenChange/MAPIStoreGCSFolder.h +++ b/OpenChange/MAPIStoreGCSFolder.h @@ -42,7 +42,8 @@ /* synchronisation */ - (BOOL) synchroniseCache; - (void) updateVersionsForMessageWithKey: (NSString *) messageKey - withChangeKey: (NSData *) newChangeKey; + withChangeKey: (NSData *) oldChangeKey + andPredecessorChangeList: (NSData *) pcl; - (NSNumber *) lastModifiedFromMessageChangeNumber: (NSString *) changeNumber; - (NSString *) changeNumberForMessageWithKey: (NSString *) messageKey; - (NSData *) changeKeyForMessageWithKey: (NSString *) messageKey; diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index 73bb647f5..55e4f52e1 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -261,7 +261,6 @@ static Class NSNumberK; */ - (void) _setChangeKey: (NSData *) changeKey forMessageEntry: (NSMutableDictionary *) messageEntry - inChangeListOnly: (BOOL) inChangeListOnly { struct XID *xid; NSString *guid; @@ -274,15 +273,11 @@ static Class NSNumberK; globCnt = [NSData dataWithBytes: xid->LocalId.data length: xid->LocalId.length]; talloc_free (xid); - if (!inChangeListOnly) - { - /* 1. set change key association */ - changeKeyDict = [NSDictionary dictionaryWithObjectsAndKeys: - guid, @"GUID", - globCnt, @"LocalId", - nil]; - [messageEntry setObject: changeKeyDict forKey: @"ChangeKey"]; - } + /* 1. set change key association */ + changeKeyDict = [NSDictionary dictionaryWithObjectsAndKeys: guid, @"GUID", + globCnt, @"LocalId", + nil]; + [messageEntry setObject: changeKeyDict forKey: @"ChangeKey"]; /* 2. append/update predecessor change list */ changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; @@ -296,6 +291,77 @@ static Class NSNumberK; [changeList setObject: globCnt forKey: guid]; } +- (void) _updatePredecessorChangeList: (NSData *) predecessorChangeList + forMessageEntry: (NSMutableDictionary *) messageEntry + withOldChangeKey: (NSData *) oldChangeKey +{ + NSData *globCnt, *oldGlobCnt; + NSDictionary *changeKeyDict; + NSString *guid; + NSMutableDictionary *changeList; + struct SizedXid *sizedXIDList; + struct XID xid, *givenChangeKey; + TALLOC_CTX *localMemCtx; + uint32_t i, length; + + localMemCtx = talloc_new (NULL); + if (!localMemCtx) + { + [self errorWithFormat: @"No more memory"]; + return; + } + + if (predecessorChangeList) + { + sizedXIDList = [predecessorChangeList asSizedXidArrayInMemCtx: localMemCtx with: &length]; + + changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; + if (!changeList) + { + changeList = [NSMutableDictionary new]; + [messageEntry setObject: changeList + forKey: @"PredecessorChangeList"]; + [changeList release]; + } + + if (sizedXIDList) { + for (i = 0; i < length; i++) + { + xid = sizedXIDList[i].XID; + guid = [NSString stringWithGUID: &xid.NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid.LocalId.data length: xid.LocalId.length]; + oldGlobCnt = [changeList objectForKey: guid]; + if (!oldGlobCnt || ([globCnt compare: oldGlobCnt] == NSOrderedDescending)) + [changeList setObject: globCnt forKey: guid]; + } + } + } + + if (oldChangeKey) + { + givenChangeKey = [oldChangeKey asXIDInMemCtx: localMemCtx]; + if (givenChangeKey) { + guid = [NSString stringWithGUID: &givenChangeKey->NameSpaceGuid]; + globCnt = [NSData dataWithBytes: givenChangeKey->LocalId.data length: givenChangeKey->LocalId.length]; + + changeKeyDict = [messageEntry objectForKey: @"ChangeKey"]; + if (!changeKeyDict || + ([guid isEqualToString: [changeKeyDict objectForKey: @"GUID"]] + && ([globCnt compare: [changeKeyDict objectForKey: @"LocalId"]] == NSOrderedDescending))) + { + /* The given change key is greater than current one stored in + metadata or it does not exist */ + [messageEntry setObject: [NSDictionary dictionaryWithObjectsAndKeys: guid, @"GUID", + globCnt, @"LocalId", + nil] + forKey: @"ChangeKey"]; + } + } + } + + talloc_free (localMemCtx); +} + - (EOQualifier *) componentQualifier { if (!componentQualifier) @@ -465,8 +531,7 @@ static Class NSNumberK; // A GLOBCNT structure is a 6-byte global namespace counter, // we strip the first 2 bytes. The first two bytes is the ReplicaId changeKey = [self getReplicaKeyFromGlobCnt: newChangeNum >> 16]; - [self _setChangeKey: changeKey forMessageEntry: messageEntry - inChangeListOnly: NO]; + [self _setChangeKey: changeKey forMessageEntry: messageEntry]; } now = [NSCalendarDate date]; @@ -483,12 +548,13 @@ static Class NSNumberK; } - (void) updateVersionsForMessageWithKey: (NSString *) messageKey - withChangeKey: (NSData *) newChangeKey + withChangeKey: (NSData *) oldChangeKey + andPredecessorChangeList: (NSData *) pcl { NSMutableDictionary *messages, *messageEntry; [self synchroniseCache]; - if (newChangeKey) + if (oldChangeKey || pcl) { messages = [[versionsMessage properties] objectForKey: @"Messages"]; messageEntry = [messages objectForKey: messageKey]; @@ -496,8 +562,8 @@ static Class NSNumberK; [NSException raise: @"MAPIStoreIOException" format: @"no version record found for message '%@'", messageKey]; - [self _setChangeKey: newChangeKey forMessageEntry: messageEntry - inChangeListOnly: YES]; + [self _updatePredecessorChangeList: pcl forMessageEntry: messageEntry + withOldChangeKey: oldChangeKey]; [versionsMessage save]; } } diff --git a/OpenChange/MAPIStoreGCSMessage.m b/OpenChange/MAPIStoreGCSMessage.m index 7ab05e5f1..747bb783a 100644 --- a/OpenChange/MAPIStoreGCSMessage.m +++ b/OpenChange/MAPIStoreGCSMessage.m @@ -209,13 +209,16 @@ - (void) updateVersions { - NSData *newChangeKey; + /* Update ChangeKey and PredecessorChangeList on message's save */ + NSData *newChangeKey, *predecessorChangeList; newChangeKey = [properties objectForKey: MAPIPropertyKey (PR_CHANGE_KEY)]; + predecessorChangeList = [properties objectForKey: MAPIPropertyKey (PR_PREDECESSOR_CHANGE_LIST)]; [(MAPIStoreGCSFolder *) container - updateVersionsForMessageWithKey: [self nameInContainer] - withChangeKey: newChangeKey]; + updateVersionsForMessageWithKey: [self nameInContainer] + withChangeKey: newChangeKey + andPredecessorChangeList: predecessorChangeList]; } @end diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 9026dcbf8..963fb7463 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -68,6 +68,8 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; +#include + #undef DEBUG #include #include @@ -516,6 +518,44 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) return [modseq1 compare: modseq2]; } +- (void) _updatePredecessorChangeListWith: (NSData *) predecessorChangeList + forMessageEntry: (NSMutableDictionary *) messageEntry +{ + NSData *globCnt, *oldGlobCnt; + NSMutableDictionary *changeList; + NSString *guid; + struct SizedXid *sizedXIDList; + struct XID xid; + uint32_t i, length; + + sizedXIDList = [predecessorChangeList asSizedXidArrayInMemCtx: NULL with: &length]; + + changeList = [messageEntry objectForKey: @"PredecessorChangeList"]; + if (!changeList) + { + changeList = [NSMutableDictionary new]; + [messageEntry setObject: changeList + forKey: @"PredecessorChangeList"]; + [changeList release]; + } + + if (sizedXIDList) { + for (i = 0; i < length; i++) + { + xid = sizedXIDList[i].XID; + guid = [NSString stringWithGUID: &xid.NameSpaceGuid]; + globCnt = [NSData dataWithBytes: xid.LocalId.data length: xid.LocalId.length]; + oldGlobCnt = [changeList objectForKey: guid]; + if (!oldGlobCnt || ([globCnt compare: oldGlobCnt] == NSOrderedDescending)) + [changeList setObject: globCnt forKey: guid]; + } + + talloc_free (sizedXIDList); + } + + [versionsMessage save]; +} + - (void) _setChangeKey: (NSData *) changeKey forMessageEntry: (NSMutableDictionary *) messageEntry { @@ -924,8 +964,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) return changeNumber; } -- (void) setChangeKey: (NSData *) changeKey - forMessageWithKey: (NSString *) messageKey +- (NSMutableDictionary *) _messageEntryFromMessageKey: (NSString *) messageKey { NSMutableDictionary *messages, *messageEntry; NSString *messageUid; @@ -936,7 +975,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) messageEntry = [messages objectForKey: messageUid]; if (!messageEntry) { - [self warnWithFormat: @"attempting to synchronise to set the change key for " + [self warnWithFormat: @"attempting to synchronise to get the message entry for " @"this message %@", messageKey]; synced = [self synchroniseCacheForUID: messageUid]; if (synced) @@ -947,7 +986,15 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) abort (); } } - [self _setChangeKey: changeKey forMessageEntry: messageEntry]; + + return messageEntry; +} + +- (void) setChangeKey: (NSData *) changeKey + forMessageWithKey: (NSString *) messageKey +{ + [self _setChangeKey: changeKey + forMessageEntry: [self _messageEntryFromMessageKey: messageKey]]; [versionsMessage save]; } @@ -1232,7 +1279,7 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) NSDictionary *result; NSUInteger count; NSArray *a; - NSData *changeKey; + NSData *changeList; if (![sourceFolder isKindOfClass: [MAPIStoreMailFolder class]]) return [super moveCopyMessagesWithMIDs: srcMids andCount: midCount @@ -1327,11 +1374,11 @@ _parseCOPYUID (NSString *line, NSArray **destUIDsP) [self synchroniseCache]; for (count = 0; count < midCount; count++) { - changeKey = [NSData dataWithBinary: targetChangeKeys[count]]; + changeList = [NSData dataWithBinary: targetPredecessorChangeLists[count]]; messageKey = [NSString stringWithFormat: @"%@.eml", [destUIDs objectAtIndex: count]]; - [self setChangeKey: changeKey - forMessageWithKey: messageKey]; + [self _updatePredecessorChangeListWith: changeList + forMessageEntry: [self _messageEntryFromMessageKey: messageKey]]; } } diff --git a/OpenChange/NSData+MAPIStore.h b/OpenChange/NSData+MAPIStore.h index 5715b9128..13daa8aef 100644 --- a/OpenChange/NSData+MAPIStore.h +++ b/OpenChange/NSData+MAPIStore.h @@ -41,6 +41,8 @@ + (id) dataWithXID: (const struct XID *) xid; - (struct XID *) asXIDInMemCtx: (void *) memCtx; +- (struct SizedXid *) asSizedXidArrayInMemCtx: (void *) memCtx + with: (uint32_t *) length; + (id) dataWithChangeKeyGUID: (NSString *) guidString andCnt: (NSData *) globCnt; diff --git a/OpenChange/NSData+MAPIStore.m b/OpenChange/NSData+MAPIStore.m index 38ef1ccf4..5a2afeb1d 100644 --- a/OpenChange/NSData+MAPIStore.m +++ b/OpenChange/NSData+MAPIStore.m @@ -22,6 +22,7 @@ #import +#import "MAPIStoreTypes.h" #import "NSObject+MAPIStore.h" #import "NSString+MAPIStore.h" @@ -29,6 +30,7 @@ #undef DEBUG #include +#include #include #include #include @@ -172,6 +174,38 @@ static void _fillFlatUIDWithGUID (struct FlatUID_r *flatUID, const struct GUID * return xid; } +- (struct SizedXid *) asSizedXidArrayInMemCtx: (void *) memCtx + with: (uint32_t *) length +{ + struct Binary_r bin; + struct SizedXid *sizedXIDArray; + + bin.cb = [self length]; + bin.lpb = (uint8_t *)[self bytes]; + + sizedXIDArray = get_SizedXidArray(memCtx, &bin, length); + if (!sizedXIDArray) + { + NSLog (@"Impossible to parse SizedXID array"); + return NULL; + } + + return sizedXIDArray; +} + +- (NSComparisonResult) compare: (NSData *) otherGlobCnt +{ + uint64_t globCnt = 0, oGlobCnt = 0; + + if ([self length] > 0) + globCnt = *(uint64_t *) [self bytes]; + + if ([otherGlobCnt length] > 0) + oGlobCnt = *(uint64_t *) [otherGlobCnt bytes]; + + return MAPICNCompare (globCnt, oGlobCnt, NULL); +} + + (id) dataWithChangeKeyGUID: (NSString *) guidString andCnt: (NSData *) globCnt; {