From 10cabca57ef437b906e67c7245521cce1147725a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Thu, 10 Dec 2015 16:36:22 +0100 Subject: [PATCH 1/7] oc: Use ReplicaID from connection info Instead of hardcoding it to 0x0001. --- OpenChange/MAPIStoreSOGoObject.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/OpenChange/MAPIStoreSOGoObject.m b/OpenChange/MAPIStoreSOGoObject.m index 2d7b6ad99..7f1cc4b6a 100644 --- a/OpenChange/MAPIStoreSOGoObject.m +++ b/OpenChange/MAPIStoreSOGoObject.m @@ -197,6 +197,7 @@ static Class MAPIStoreFolderK; inMemCtx: (TALLOC_CTX *) memCtx { int rc; + struct mapistore_connection_info *connInfo; uint64_t obVersion; obVersion = [self objectVersion]; @@ -204,8 +205,9 @@ static Class MAPIStoreFolderK; rc = MAPISTORE_ERR_NOT_FOUND; else { + connInfo = [[self context] connectionInfo]; *data = MAPILongLongValue (memCtx, ((obVersion << 16) - | 0x0001)); + | connInfo->repl_id)); rc = MAPISTORE_SUCCESS; } From 98ed9c3b17f22c38a81e1140029cddf5fe2d88fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:22:26 +0100 Subject: [PATCH 2/7] oc-mail: Manage multiple CN restriction By ignoring <, =< and = operators and simplifying the following request: MODSEQ >= x || MODSEQ >= y || MODSEQ >= z --> MODSEQ >= min(x, y, z) This hack will reduce the number of current retrieved UID keys from the IMAP server. Current status is to retrieve everything when the multiple CN restriction is sent as the required restriction is too complex and it is not defined by the IMAP spec. The proper implementation for: CN > x_1 & CN < x_2 | CN > y_1 & CN < y_2 | CN > z_1 It will be something like this: set(MODSEQ >= x_1 + 1) - set(MODSEQ >= x_2) U set(MODSEQ >= y_1 + 1) - set(MODSEQ >= y_2) U set(MODSEQ >= z_1) Assuming x_1 <= x_2 <= y_1 <= y_2 <= z_1. --- OpenChange/MAPIStoreMailFolder.m | 67 +++++++++++++++++++++++++- OpenChange/MAPIStoreMailMessageTable.m | 4 +- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 6934e7a60..02e25acbd 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -74,6 +74,7 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; #include #include #include +#include #include #include @@ -257,6 +258,67 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; return MAPISTORE_SUCCESS; } +- (EOQualifier *) simplifyQualifier: (EOQualifier *) qualifier +{ + /* Hack: Reduce the number of MODSEQ constraints to a single one as + we assume the difference among MODSEQs will be small enough to + return a small number of UIDs. + + This is the only case we do simplify: + MODSEQ >= x | MODSEQ >= y | MODSEQ >= z => MODSEQ >= min(x,y,z) + */ + if (qualifier && [qualifier isKindOfClass: [EOOrQualifier class]]) + { + EOQualifier *simplifiedQualifier; + NSArray *quals; + NSNumber *minModseq; + NSUInteger i, count; + + quals = [(EOOrQualifier *)qualifier qualifiers]; + count = [quals count]; + if (count < 2) + return qualifier; + + minModseq = [NSNumber numberWithUnsignedLongLong: ULLONG_MAX]; + + for (i = 0; i < count; i++) + { + EOQualifier *subQualifier; + + subQualifier = [quals objectAtIndex: i]; + if ([subQualifier isKindOfClass: [EOAndQualifier class]] + && [[(EOAndQualifier *)subQualifier qualifiers] count] == 1) + subQualifier = [[(EOAndQualifier *)subQualifier qualifiers] objectAtIndex: 0]; + + if ([subQualifier isKindOfClass: [EOKeyValueQualifier class]] + && [[(EOKeyValueQualifier *)subQualifier key] isEqualToString: @"MODSEQ"]) + { + NSNumber *value; + + value = (NSNumber *)[(EOKeyValueQualifier *)subQualifier value]; + if ([minModseq compare: value] == NSOrderedDescending + && [value unsignedLongLongValue] > 0) + minModseq = (NSNumber *)[(EOKeyValueQualifier *)subQualifier value]; + + } + else + return qualifier; + } + + if ([minModseq unsignedLongLongValue] > 0 && [minModseq unsignedLongLongValue] < ULLONG_MAX) + { + simplifiedQualifier = [[EOKeyValueQualifier alloc] + initWithKey: @"MODSEQ" + operatorSelector: EOQualifierOperatorGreaterThanOrEqualTo + value: minModseq]; + [simplifiedQualifier autorelease]; + return simplifiedQualifier; + } + } + + return qualifier; +} + - (EOQualifier *) nonDeletedQualifier { static EOQualifier *nonDeletedQualifier = nil; @@ -281,7 +343,7 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; andSortOrderings: (NSArray *) sortOrderings { NSArray *uidKeys; - EOQualifier *fetchQualifier; + EOQualifier *fetchQualifier, *simplifiedQualifier; if ([self ensureFolderExists]) { @@ -290,9 +352,10 @@ static Class SOGoMailFolderK, MAPIStoreMailFolderK, MAPIStoreOutboxFolderK; if (qualifier) { + simplifiedQualifier = [self simplifyQualifier: qualifier]; fetchQualifier = [[EOAndQualifier alloc] initWithQualifiers: - [self nonDeletedQualifier], qualifier, + [self nonDeletedQualifier], simplifiedQualifier, nil]; [fetchQualifier autorelease]; } diff --git a/OpenChange/MAPIStoreMailMessageTable.m b/OpenChange/MAPIStoreMailMessageTable.m index c71097d63..d315f6527 100644 --- a/OpenChange/MAPIStoreMailMessageTable.m +++ b/OpenChange/MAPIStoreMailMessageTable.m @@ -181,8 +181,8 @@ static Class MAPIStoreMailMessageK, NSDataK, NSStringK; else { /* Ignore other operations as IMAP only support MODSEQ >= X */ - [self warnWithFormat: @"Ignoring %@ as only supported operators are > and >=", - [self operatorFromRestrictionOperator: res->relop]]; + [self warnWithFormat: @"Ignoring '%@' as only supported operators are > and >=", + NSStringFromSelector ([self operatorFromRestrictionOperator: res->relop])]; rc = MAPIRestrictionStateAlwaysTrue; } } From 2fc21e48d81e0a90cccfd058d3a10adfa86e1cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:31:15 +0100 Subject: [PATCH 3/7] oc: Use proper variable name for last modified value --- OpenChange/MAPIStoreGCSFolder.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index 2dacdb4cf..336474675 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -681,12 +681,12 @@ static Class NSNumberK; - (NSNumber *) lastModifiedFromMessageChangeNumber: (NSString *) changeNumber { NSDictionary *mapping; - NSNumber *modseq; + NSNumber *lastModified; mapping = [[versionsMessage properties] objectForKey: @"VersionMapping"]; - modseq = [mapping objectForKey: changeNumber]; + lastModified = [mapping objectForKey: changeNumber]; - return modseq; + return lastModified; } - (NSString *) changeNumberForMessageWithKey: (NSString *) messageKey From 3dff73636c0c401e1c13fae41d9490e8c657b889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:33:41 +0100 Subject: [PATCH 4/7] oc: Use NSString index for version lookup It was using NSNumber in versions Dictionary for GCSMessages but it is stored as the NSString representation (0x390300000000001), so the lookup has always failed. --- OpenChange/MAPIStoreGCSMessageTable.m | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/OpenChange/MAPIStoreGCSMessageTable.m b/OpenChange/MAPIStoreGCSMessageTable.m index e6da41039..7a544c173 100644 --- a/OpenChange/MAPIStoreGCSMessageTable.m +++ b/OpenChange/MAPIStoreGCSMessageTable.m @@ -27,6 +27,7 @@ #import #import +#import #import #import @@ -38,7 +39,6 @@ #import "MAPIStoreTypes.h" #import "MAPIStoreGCSFolder.h" - #import "MAPIStoreGCSMessageTable.h" #undef DEBUG @@ -89,9 +89,12 @@ if (res->ulPropTag == PidTagChangeNumber) { + NSString *changeNumber; + value = NSObjectFromMAPISPropValue (&res->lpProp); + changeNumber = [NSString stringWithUnsignedLongLong: [(NSNumber *)value unsignedLongLongValue]]; lastModified = [(MAPIStoreGCSFolder *) - container lastModifiedFromMessageChangeNumber: value]; + container lastModifiedFromMessageChangeNumber: changeNumber]; //[self logWithFormat: @"change number from oxcfxics: %.16lx", [value unsignedLongLongValue]]; //[self logWithFormat: @" c_lastmodified: %@", lastModified]; if (lastModified) From 7cdf48335b7ae8b25463e7acd04159c42f9e3db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:35:55 +0100 Subject: [PATCH 5/7] oc: Support every operator for CN restriction No limitation is found in SQL queries. So use them all. --- OpenChange/MAPIStoreGCSMessageTable.m | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/OpenChange/MAPIStoreGCSMessageTable.m b/OpenChange/MAPIStoreGCSMessageTable.m index 7a544c173..0dac1fff9 100644 --- a/OpenChange/MAPIStoreGCSMessageTable.m +++ b/OpenChange/MAPIStoreGCSMessageTable.m @@ -99,14 +99,21 @@ //[self logWithFormat: @" c_lastmodified: %@", lastModified]; if (lastModified) { + SEL operator; + + operator = [self operatorFromRestrictionOperator: res->relop]; *qualifier = [[EOKeyValueQualifier alloc] initWithKey: @"c_lastmodified" - operatorSelector: EOQualifierOperatorGreaterThanOrEqualTo + operatorSelector: operator value: lastModified]; [*qualifier autorelease]; rc = MAPIRestrictionStateNeedsEval; } else - rc = MAPIRestrictionStateAlwaysTrue; + { + [self logWithFormat: @"No last modified found for: 0x%.16"PRIx64". Then no restriction applied", + [value unsignedLongLongValue]]; + rc = MAPIRestrictionStateAlwaysTrue; + } } else { From 5993fe97c396d868cf9ed46049cee41ba3c0e849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:37:02 +0100 Subject: [PATCH 6/7] oc-db: Implement description message to ease debugging Which dumps the properties NSMutableDictionary --- OpenChange/MAPIStoreDBMessage.m | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index bb5acbb57..836cfb4e1 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -109,6 +109,49 @@ return self; } +- (NSString *) description +{ + id key, value; + NSEnumerator *propEnumerator; + NSMutableString *description; + + description = [NSMutableString stringWithFormat: @"%@ %@. Properties: {", NSStringFromClass ([self class]), + [self url]]; + + propEnumerator = [properties keyEnumerator]; + while ((key = [propEnumerator nextObject])) + { + uint32_t proptag = 0; + if ([key isKindOfClass: [NSString class]] && [(NSString *)key intValue] > 0) + proptag = [(NSString *)key intValue]; + else if ([key isKindOfClass: [NSNumber class]]) + proptag = [key unsignedLongValue]; + + if (proptag > 0) + { + const char *propTagName = get_proptag_name ([key unsignedLongValue]); + NSString *propName; + + if (propTagName) + propName = [NSString stringWithCString: propTagName + encoding: NSUTF8StringEncoding]; + else + propName = [NSString stringWithFormat: @"0x%.4x", [key unsignedLongValue]]; + + [description appendFormat: @"'%@': ", propName]; + } + else + [description appendFormat: @"'%@': ", key]; + + value = [properties objectForKey: key]; + [description appendFormat: @"%@ (%@), ", value, NSStringFromClass ([value class])]; + } + + [description appendString: @"}\n"]; + + return description; +} + - (uint64_t) objectVersion { NSNumber *versionNbr; From a618386d5ecec5e13a5b37fa7583dd7f9a08fdbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 11 Dec 2015 11:38:21 +0100 Subject: [PATCH 7/7] oc-db: Store version number instead of CN To be able to search in this for every kind of operator. As we cannot do a migration, we have to add a new key to the property dictionary (@"version_number") which stores the version: version = exchange_globcnt(CN >> 16) Instead of the CN structure which is stored in @"version" key. This way we can do searches for CN to download only missing data from the given state of the client for this kind of messages. --- OpenChange/MAPIStoreDBMessage.m | 29 +++++++++++++++++++++------- OpenChange/MAPIStoreDBMessageTable.m | 9 +++++---- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index 836cfb4e1..98d357bf7 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -154,16 +154,25 @@ - (uint64_t) objectVersion { - NSNumber *versionNbr; + /* Return the global counter from CN structure. + See [MS-OXCFXICS] Section 2.2.2.1 */ + NSNumber *versionNbr, *cn; uint64_t objectVersion; [(SOGoMAPIDBMessage *) sogoObject reloadIfNeeded]; - versionNbr = [properties objectForKey: @"version"]; + versionNbr = [properties objectForKey: @"version_number"]; if (versionNbr) - objectVersion = (([versionNbr unsignedLongLongValue] >> 16) - & 0x0000ffffffffffffLL); + objectVersion = exchange_globcnt ([versionNbr unsignedLongLongValue]); else - objectVersion = ULLONG_MAX; + { + /* Old version which stored the CN structure not useful for searching */ + cn = [properties objectForKey: @"version"]; + if (cn) + objectVersion = (([cn unsignedLongLongValue] >> 16) + & 0x0000ffffffffffffLL); + else + objectVersion = ULLONG_MAX; + } return objectVersion; } @@ -326,13 +335,19 @@ [properties setObject: attachmentParts forKey: @"attachments"]; newVersion = [[self context] getNewChangeNumber]; + newVersion = exchange_globcnt ((newVersion >> 16) & 0x0000ffffffffffffLL); + [properties setObject: [NSNumber numberWithUnsignedLongLong: newVersion] - forKey: @"version"]; + forKey: @"version_number"]; + + /* Remove old version */ + [properties removeObjectForKey: @"version"]; /* Update PredecessorChangeList accordingly */ [self _updatePredecessorChangeList]; - [self logWithFormat: @"%d props in dict", [properties count]]; + // [self logWithFormat: @"Saving %@", [self description]]; + // [self logWithFormat: @"%d props in dict", [properties count]]; [sogoObject save]; } diff --git a/OpenChange/MAPIStoreDBMessageTable.m b/OpenChange/MAPIStoreDBMessageTable.m index de22e238e..a8c68abd1 100644 --- a/OpenChange/MAPIStoreDBMessageTable.m +++ b/OpenChange/MAPIStoreDBMessageTable.m @@ -61,14 +61,15 @@ static Class MAPIStoreDBMessageK = Nil; if ((uint32_t) res->ulPropTag == PidTagChangeNumber) { + SEL operator; + value = NSObjectFromMAPISPropValue (&res->lpProp); cVersion = exchange_globcnt (([value unsignedLongLongValue] >> 16) & 0x0000ffffffffffffLL); version = [NSNumber numberWithUnsignedLongLong: cVersion]; - //[self logWithFormat: @"change number from oxcfxics: %.16lx", [value unsignedLongLongValue]]; - [self logWithFormat: @" version: %.16lx", cVersion]; - *qualifier = [[EOKeyValueQualifier alloc] initWithKey: @"version" - operatorSelector: EOQualifierOperatorGreaterThanOrEqualTo + operator = [self operatorFromRestrictionOperator: res->relop]; + *qualifier = [[EOKeyValueQualifier alloc] initWithKey: @"version_number" + operatorSelector: operator value: version]; [*qualifier autorelease]; rc = MAPIRestrictionStateNeedsEval;