From 734aba5ddbb6094fe4e7429b6d3d0cefe7344985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= Date: Mon, 14 Sep 2015 15:53:07 +0200 Subject: [PATCH 01/14] oc: Fix asCSSIdentifier in openchange_user_cleanup The method lacked the check for the initial character, which adds an underscore at the beginning of the strings that start with a digit. --- Scripts/openchange_user_cleanup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Scripts/openchange_user_cleanup b/Scripts/openchange_user_cleanup index e67289480..93a91c8ea 100755 --- a/Scripts/openchange_user_cleanup +++ b/Scripts/openchange_user_cleanup @@ -268,6 +268,9 @@ def asCSSIdentifier(inputString): newChars = [] + if str.isdigit(inputString[0]): + newChars.append("_") + for c in inputString: if c in cssEscapingCharMap: newChars.append(cssEscapingCharMap[c]) From ba68bd8935227fd98c43f13ede349ff081c5a254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= Date: Mon, 14 Sep 2015 15:40:34 +0200 Subject: [PATCH 02/14] Make folderKey encoding consistent The folder names are encoded through the `asCSSIdentifier` and `stringByEncodingImap4FolderName` functions when we store them as folder keys. In addition, the prefix "folder" is added to the key. The order in which these operations were done when storing the folder keys (and reverted when retrieving them) wasn't consistent trough the code. This led to problems such as creating twice a folder with a digit at the beginning of its name. The folder name goes now through the following operations when being stored as a key (the retrieval reverts these in the reverse order): * `stringByEncodingImap4FolderName` * `asCSSIdentifier` * Add "folder" prefix --- OpenChange/MAPIStoreMailContext.m | 24 +++++++++++++----------- SoObjects/Mailer/SOGoMailAccount.m | 4 ++-- SoObjects/Mailer/SOGoMailFolder.m | 16 ++++++++-------- UI/MailerUI/UIxMailFolderActions.m | 2 +- UI/MailerUI/UIxMailMainFrame.m | 4 ++-- UI/WebServerResources/MailerUI.js | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/OpenChange/MAPIStoreMailContext.m b/OpenChange/MAPIStoreMailContext.m index 7ec79ad50..c1f39565f 100644 --- a/OpenChange/MAPIStoreMailContext.m +++ b/OpenChange/MAPIStoreMailContext.m @@ -160,23 +160,25 @@ MakeDisplayFolderName (NSString *folderName) for (count = 0; count < max; count++) { context = talloc_zero (memCtx, struct mapistore_contexts_list); - // secondaryFolders has the names (1) Imap4Encoded and (2) asCSSIdentifier - // e.g.: Probl&AOg-mes_SP_de_SP_synchronisation + // secondaryFolders has the names (1) Imap4Encoded ,(2) asCSSIdentifier and (3) "folder"-prefixed + // e.g.: folderProbl&AOg-mes_SP_de_SP_synchronisation currentName = [secondaryFolders objectAtIndex: count]; - // To get the real name we have to revert that (applying the decode functions) - // in reverse order + // To get the real name we have to revert that (applying the decode functions + // in reverse order) // e.g.: Problèmes de synchronisation - realName = [[currentName fromCSSIdentifier] - stringByDecodingImap4FolderName]; + realName = [[[currentName substringFromIndex: 6] + fromCSSIdentifier] + stringByDecodingImap4FolderName]; // And finally to represent that as URI we have to (1) asCSSIdentifier, - // (2) Imap4Encode and (3) AddPercentEscapes - // e.g.: Probl&AOg-mes_SP_de_SP_synchronisation + // (2) Imap4Encode (3) AddPercentEscapes and (4) add the "folder" prefix + // e.g.: folderProbl&AOg-mes_SP_de_SP_synchronisation // In the example there are no percent escapes added because is already ok stringData = [[[realName asCSSIdentifier] stringByEncodingImap4FolderName] stringByAddingPercentEscapesUsingEncoding: NSUTF8StringEncoding]; + stringData = [NSString stringWithFormat: @"folder%@", stringData]; context->url = [[NSString stringWithFormat: @"%@%@", urlBase, stringData] asUnicodeInMemCtx: context]; - context->name = [[realName substringFromIndex: 6] asUnicodeInMemCtx: context]; + context->name = [realName asUnicodeInMemCtx: context]; context->main_folder = false; context->role = MAPISTORE_MAIL_ROLE; context->tag = "tag"; @@ -200,7 +202,7 @@ MakeDisplayFolderName (NSString *folderName) andTDBIndexing: NULL]; accountFolder = [[userContext rootFolders] objectForKey: @"mail"]; folderName = [NSString stringWithFormat: @"folder%@", - [newFolderName asCSSIdentifier]]; + [[newFolderName stringByEncodingImap4FolderName] asCSSIdentifier]]; newFolder = [SOGoMailFolder objectWithName: folderName inContainer: accountFolder]; if ([newFolder create]) @@ -209,7 +211,7 @@ MakeDisplayFolderName (NSString *folderName) withString: @"%40"], [userName stringByReplacingOccurrencesOfString: @"@" withString: @"%40"], - [[folderName stringByEncodingImap4FolderName] stringByAddingPercentEscapesUsingEncoding: NSUTF8StringEncoding]]; + [folderName stringByAddingPercentEscapesUsingEncoding: NSUTF8StringEncoding]]; else mapistoreURI = nil; diff --git a/SoObjects/Mailer/SOGoMailAccount.m b/SoObjects/Mailer/SOGoMailAccount.m index 97057a1c4..f24af644d 100644 --- a/SoObjects/Mailer/SOGoMailAccount.m +++ b/SoObjects/Mailer/SOGoMailAccount.m @@ -206,8 +206,8 @@ static NSString *inboxFolderName = @"INBOX"; [folders removeObjectsInArray: nss]; } - return [[folders stringsWithFormat: @"folder%@"] - resultsOfSelector: @selector (asCSSIdentifier)]; + return [[folders resultsOfSelector: @selector (asCSSIdentifier)] + stringsWithFormat: @"folder%@"]; } - (NSArray *) toManyRelationshipKeys diff --git a/SoObjects/Mailer/SOGoMailFolder.m b/SoObjects/Mailer/SOGoMailFolder.m index 671a6caf6..884e838b6 100644 --- a/SoObjects/Mailer/SOGoMailFolder.m +++ b/SoObjects/Mailer/SOGoMailFolder.m @@ -197,9 +197,9 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) { NSArray *subfolders; - subfolders = [[self subfolders] stringsWithFormat: @"folder%@"]; + subfolders = [[self subfolders] resultsOfSelector: @selector (asCSSIdentifier)]; - return [subfolders resultsOfSelector: @selector (asCSSIdentifier)]; + return [subfolders stringsWithFormat: @"folder%@"]; } - (NSArray *) subfolders @@ -632,7 +632,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) inContext: (id) localContext { NSArray *folders; - NSString *currentFolderName, *currentAccountName; + NSString *currentFolderName, *currentAccountName, *destinationAccountName; NSMutableString *imapDestinationFolder; NGImap4Client *client; id result; @@ -640,24 +640,24 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) #warning this code will fail on implementation using something else than '/' as delimiter imapDestinationFolder = [NSMutableString string]; - folders = [[destinationFolder componentsSeparatedByString: @"/"] - resultsOfSelector: @selector (fromCSSIdentifier)]; + folders = [destinationFolder componentsSeparatedByString: @"/"]; max = [folders count]; if (max > 1) { currentAccountName = [[self mailAccountFolder] nameInContainer]; client = [[self imap4Connection] client]; [imap4 selectFolder: [self imap4URL]]; + destinationAccountName = [[folders objectAtIndex: 1] fromCSSIdentifier]; for (count = 2; count < max; count++) { - currentFolderName = [[folders objectAtIndex: count] substringFromIndex: 6]; + currentFolderName = [[[folders objectAtIndex: count] substringFromIndex: 6] fromCSSIdentifier]; [imapDestinationFolder appendFormat: @"/%@", currentFolderName]; } if (client) { - if ([[folders objectAtIndex: 1] isEqualToString: currentAccountName]) + if ([destinationAccountName isEqualToString: currentAccountName]) { // We make sure the destination IMAP folder exist, if not, we create it. result = [[client status: imapDestinationFolder @@ -686,7 +686,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) userFolder = [[context activeUser] homeFolderInContext: context]; accounts = [userFolder lookupName: @"Mail" inContext: context acquire: NO]; - account = [accounts lookupName: [folders objectAtIndex: 1] inContext: localContext acquire: NO]; + account = [accounts lookupName: destinationAccountName inContext: localContext acquire: NO]; if ([account isKindOfClass: [NSException class]]) { diff --git a/UI/MailerUI/UIxMailFolderActions.m b/UI/MailerUI/UIxMailFolderActions.m index 4515eac4f..80fd4fa20 100644 --- a/UI/MailerUI/UIxMailFolderActions.m +++ b/UI/MailerUI/UIxMailFolderActions.m @@ -106,7 +106,7 @@ keyForMsgUIDs = [NSString stringWithFormat:@"/%@/%@", currentAccount, currentMailbox]; newFolderName = [[context request] formValueForKey: @"name"]; - newKeyForMsgUIDs = [[NSString stringWithFormat:@"/%@/folder%@", currentAccount, newFolderName] asCSSIdentifier]; + newKeyForMsgUIDs = [NSString stringWithFormat:@"/%@/folder%@", [currentAccount asCSSIdentifier], [newFolderName asCSSIdentifier]]; error = [co renameTo: newFolderName]; if (error) { diff --git a/UI/MailerUI/UIxMailMainFrame.m b/UI/MailerUI/UIxMailMainFrame.m index 488449207..e6ee428e5 100644 --- a/UI/MailerUI/UIxMailMainFrame.m +++ b/UI/MailerUI/UIxMailMainFrame.m @@ -688,8 +688,8 @@ for (k = 0; k < [pathComponents count]; k++) { - component = [NSString stringWithFormat: @"folder%@", [pathComponents objectAtIndex: k]]; - [path appendString: [component asCSSIdentifier]]; + component = [[pathComponents objectAtIndex: k] asCSSIdentifier]; + [path appendString: [NSString stringWithFormat: @"folder%@", component]]; if (k < [pathComponents count] - 1) [path appendString: @"/"]; } diff --git a/UI/WebServerResources/MailerUI.js b/UI/WebServerResources/MailerUI.js index 9144db5b9..d42e94c9c 100644 --- a/UI/WebServerResources/MailerUI.js +++ b/UI/WebServerResources/MailerUI.js @@ -3029,7 +3029,7 @@ Mailbox.prototype = { var currentFolder = this; while (currentFolder.parentFolder) { - fullName = ("/folder" + currentFolder.name).asCSSIdentifier() + fullName; + fullName = "/folder" + currentFolder.name.asCSSIdentifier() + fullName; currentFolder = currentFolder.parentFolder; } From f3ac7b83344c9cfce9e21efd3707436e1c356b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 18 Sep 2015 21:40:01 +0200 Subject: [PATCH 03/14] oc: Return FAI available properties on instance method as well The operation RopFastTransferSourceCopyTo calls the available properties for a message using the instance method. It seems the preferred method by Outlook to synchronise a FAI message. OpenChange calls the message to get the available properties, so the instance method is called. As it is specialised to return the custom hack FAI properties, we have to call that class method instead of using generic one available at NSObject (MAPIStoreProperties) class. This avoids crashing the Outlook client after we synchronise the calendar folder after changing the timeframe view (eg from day view to month view). --- OpenChange/MAPIStoreDBMessage.m | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index 7c208b2f8..8b41dcaba 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -76,6 +76,13 @@ return MAPISTORE_SUCCESS; } +- (enum mapistore_error) getAvailableProperties: (struct SPropTagArray **) propertiesP + inMemCtx: (TALLOC_CTX *) memCtx +{ + return [[self class] getAvailableProperties: propertiesP + inMemCtx: memCtx]; +} + - (id) initWithSOGoObject: (id) newSOGoObject inContainer: (MAPIStoreObject *) newContainer { From deca6e383a8a117b3b318d89e5b9c038df968656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Fri, 18 Sep 2015 21:45:00 +0200 Subject: [PATCH 04/14] oc: Include 0x683d0040 as another unknown FAI message property And returning right type. We have sorted the FAI properties by number and set the available name as a comment to ease the tracking afterwards. --- OpenChange/MAPIStoreDBMessage.m | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/OpenChange/MAPIStoreDBMessage.m b/OpenChange/MAPIStoreDBMessage.m index 8b41dcaba..bb5acbb57 100644 --- a/OpenChange/MAPIStoreDBMessage.m +++ b/OpenChange/MAPIStoreDBMessage.m @@ -45,16 +45,28 @@ @implementation MAPIStoreDBMessage -+ (int) getAvailableProperties: (struct SPropTagArray **) propertiesP - inMemCtx: (TALLOC_CTX *) memCtx ++ (enum mapistore_error) getAvailableProperties: (struct SPropTagArray **) propertiesP + inMemCtx: (TALLOC_CTX *) memCtx { struct SPropTagArray *properties; NSUInteger count; - enum MAPITAGS faiProperties[] = { 0x68350102, 0x683c0102, 0x683e0102, - 0x683f0102, 0x68410003, 0x68420102, - 0x68450102, 0x68460003, - // PR_VD_NAME_W, PR_VD_FLAGS, PR_VD_VERSION, PR_VIEW_CLSID - 0x7006001F, 0x70030003, 0x70070003, 0x68330048 }; + + enum MAPITAGS faiProperties[] = { + 0x68330048, /* PR_VIEW_CLSID */ + 0x68350102, /* PR_VIEW_STATE */ + 0x683c0102, + 0x683d0040, + 0x683e0102, + 0x683f0102, /* PR_VIEW_VIEWTYPE_KEY */ + 0x68410003, + 0x68420102, + 0x68450102, + 0x68460003, + 0x7006001F, /* PR_VD_NAME_W */ + 0x70030003, /* PR_VD_FLAGS */ + 0x70070003 /* PR_VD_VERSION */ + }; + size_t faiSize = sizeof(faiProperties) / sizeof(enum MAPITAGS); properties = talloc_zero (memCtx, struct SPropTagArray); From c8acba8f5a8eb43c6b4a5ff2b648af82bbf8fd59 Mon Sep 17 00:00:00 2001 From: Ludovic Marcotte Date: Thu, 10 Sep 2015 20:07:42 -0400 Subject: [PATCH 05/14] (fix) safe-guard against not-yet-created calendar of local users --- SoObjects/Appointments/SOGoAppointmentFolder.m | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SoObjects/Appointments/SOGoAppointmentFolder.m b/SoObjects/Appointments/SOGoAppointmentFolder.m index f0d310bd7..3ff2b26ec 100644 --- a/SoObjects/Appointments/SOGoAppointmentFolder.m +++ b/SoObjects/Appointments/SOGoAppointmentFolder.m @@ -2980,7 +2980,10 @@ firstInstanceCalendarDateRange: (NGCalendarDateRange *) fir theUser = [SOGoUser userWithLogin: theUID]; aParent = [theUser calendarsFolderInContext: context]; - + + if ([aParent isKindOfClass: [NSException class]]) + return nil; + aFolders = [aParent subFolders]; e = [aFolders objectEnumerator]; while ((aFolder = [e nextObject])) From 0ca6e7c6d32b37fac3c4a692c5c3abd6cdf80b16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Garc=C3=ADa=20S=C3=A1ez?= Date: Thu, 24 Sep 2015 18:56:07 +0200 Subject: [PATCH 06/14] Fix freebusy for multidomain environments When returning contacts we have to supply also the domain field. Because in a multidomain environment UIDField is unique only in the domain so an user must be identified as uid@domain. So when creating http requests from client side, we have to use uid@domain instead of only uid so the SOGoUser created on server side when parsing the requests is created properly. --- SoObjects/Contacts/SOGoContactSourceFolder.m | 8 ++++++++ UI/WebServerResources/UIxAttendeesEditor.js | 19 ++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/SoObjects/Contacts/SOGoContactSourceFolder.m b/SoObjects/Contacts/SOGoContactSourceFolder.m index e385d9c09..5f9acea4b 100644 --- a/SoObjects/Contacts/SOGoContactSourceFolder.m +++ b/SoObjects/Contacts/SOGoContactSourceFolder.m @@ -48,6 +48,7 @@ #import #import #import +#import #import #import @@ -236,6 +237,13 @@ data = @""; [newRecord setObject: data forKey: @"c_cn"]; + if ([[SOGoSystemDefaults sharedSystemDefaults] enableDomainBasedUID]) + { + data = [oldRecord objectForKey: @"c_domain"]; + if (data) + [newRecord setObject: data forKey: @"c_domain"]; + } + data = [oldRecord objectForKey: @"mail"]; if (!data) data = @""; diff --git a/UI/WebServerResources/UIxAttendeesEditor.js b/UI/WebServerResources/UIxAttendeesEditor.js index 16d297425..c33f45451 100644 --- a/UI/WebServerResources/UIxAttendeesEditor.js +++ b/UI/WebServerResources/UIxAttendeesEditor.js @@ -221,9 +221,11 @@ function performSearchCallback(http) { node.address = completeEmail; // log("node.address: " + node.address); if (contact["c_uid"]) { - node.uid = (contact["isMSExchange"]? UserLogin + ":" : "") + contact["c_uid"]; - } - else { + var login = contact["c_uid"]; + if (contact["c_domain"]) + login += "@" + contact["c_domain"]; + node.uid = (contact["isMSExchange"]? UserLogin + ":" : "") + login; + } else node.uid = null; node.appendChild(new Element('div').addClassName('colorBox').addClassName('noFreeBusy')); } @@ -275,13 +277,16 @@ function performSearchCallback(http) { else { if (document.currentPopupMenu) hideMenu(document.currentPopupMenu); - + if (data.contacts.length == 1) { // Single result var contact = data.contacts[0]; - if (contact["c_uid"]) - input.uid = (contact["isMSExchange"]? UserLogin + ":" : "") + contact["c_uid"]; - else + if (contact["c_uid"]) { + var login = contact["c_uid"]; + if (contact["c_domain"]) + login += "@" + contact["c_domain"]; + input.uid = (contact["isMSExchange"]? UserLogin + ":" : "") + login; + } else input.uid = null; var isList = (contact["c_component"] && contact["c_component"] == "vlist"); From b32951384568dc705d16cde4be910a886cb58b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= Date: Fri, 25 Sep 2015 12:14:32 +0200 Subject: [PATCH 07/14] Store c_content in sogo_cache_folder as a BLOB This reduces considerably the query time for that column, which can be rather large. --- SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m | 6 +++--- SoObjects/SOGo/SOGoCacheGCSObject.m | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m b/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m index cef17775b..6adb55ef2 100644 --- a/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m +++ b/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m @@ -57,7 +57,7 @@ @" c_lastmodified INT4 NOT NULL," @" c_version INT4 NOT NULL DEFAULT 0," @" c_deleted SMALLINT NOT NULL DEFAULT 0," - @" c_content TEXT)"); + @" c_content BYTEA)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } @@ -77,7 +77,7 @@ @" c_lastmodified INT NOT NULL," @" c_version INT NOT NULL DEFAULT 0," @" c_deleted TINYINT NOT NULL DEFAULT 0," - @" c_content LONGTEXT)"); + @" c_content BLOB)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } @@ -97,7 +97,7 @@ @" c_lastmodified INT4 NOT NULL," @" c_version INT4 NOT NULL DEFAULT 0," @" c_deleted SMALLINT NOT NULL DEFAULT 0," - @" c_content CLOB)"); + @" c_content BLOB)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } diff --git a/SoObjects/SOGo/SOGoCacheGCSObject.m b/SoObjects/SOGo/SOGoCacheGCSObject.m index a2251173a..a4b32a74a 100644 --- a/SoObjects/SOGo/SOGoCacheGCSObject.m +++ b/SoObjects/SOGo/SOGoCacheGCSObject.m @@ -152,7 +152,7 @@ static EOAttribute *textColumn = nil; - (void) setupFromRecord: (NSDictionary *) record { NSInteger intValue; - NSString *propsValue; + NSData *content; NSDictionary *newValues; objectType = [[record objectForKey: @"c_type"] intValue]; @@ -166,10 +166,10 @@ static EOAttribute *textColumn = nil; dateWithTimeIntervalSince1970: (NSTimeInterval) intValue]); deleted = ([[record objectForKey: @"c_deleted"] intValue] > 0); version = [[record objectForKey: @"c_version"] intValue]; - propsValue = [record objectForKey: @"c_content"]; - if ([propsValue isNotNull]) + content = [record objectForKey: @"c_content"]; + if ([content isNotNull]) { - newValues = [[propsValue dataByDecodingBase64] BSONValue]; + newValues = [[content dataByDecodingBase64] BSONValue]; [properties addEntriesFromDictionary: newValues]; } else From c01b21e402af6d7f4bee720d68a89a5d22c397be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Amor=20Garc=C3=ADa?= Date: Sun, 27 Sep 2015 12:32:17 +0200 Subject: [PATCH 08/14] oc-mail: Fixed false positive in [MAPIStoreAttachment hasContentId] This method does not longer returns true if the content id was a empty string. In some case the old false positive triggered the removal of attachments when sending messages. --- OpenChange/MAPIStoreMailVolatileMessage.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/OpenChange/MAPIStoreMailVolatileMessage.m b/OpenChange/MAPIStoreMailVolatileMessage.m index 8a7ba2c58..d6154c8e8 100644 --- a/OpenChange/MAPIStoreMailVolatileMessage.m +++ b/OpenChange/MAPIStoreMailVolatileMessage.m @@ -167,9 +167,9 @@ static NSString *recTypes[] = { @"orig", @"to", @"cc", @"bcc" }; - (BOOL) hasContentId { - return ([properties - objectForKey: MAPIPropertyKey (PR_ATTACH_CONTENT_ID_UNICODE)] - != nil); + NSString *contentId = [properties + objectForKey: MAPIPropertyKey (PR_ATTACH_CONTENT_ID_UNICODE)]; + return contentId && [contentId length] > 0; } - (NGMimeBodyPart *) asMIMEBodyPart @@ -233,7 +233,7 @@ static NSString *recTypes[] = { @"orig", @"to", @"cc", @"bcc" }; [map addObject: contentDisposition forKey: @"content-disposition"]; contentId = [properties objectForKey: MAPIPropertyKey (PR_ATTACH_CONTENT_ID_UNICODE)]; - if (contentId) + if (contentId && [contentId length] > 0) [map setObject: [NSString stringWithFormat: @"<%@>", contentId] forKey: @"content-id"]; bodyPart = [NGMimeBodyPart bodyPartWithHeader: map]; From eebf878e896bd7b1395e4b2bdcb7a93d320c80ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Wed, 30 Sep 2015 11:35:12 +0200 Subject: [PATCH 09/14] Ignore recurrence-id vevents without dtstart and outside date range Happened in an imported vevent from Mozilla Thunderbird. The crash was: Sep 14 15:49:38 sogod [21063]: <0x6442DBF8[SOGoAppointmentFolder]:personal> missing 'c_startdate' in record? Sep 14 15:49:38 sogod [21063]: <0x6442DBF8[SOGoAppointmentFolder]:personal> missing 'c_enddate' in record? 2015-09-14 15:49:38.927 sogod[21063] NGCalendarDateRange.m:37 Assertion failed in NGCalendarDateRange(instance), method initWithStartDate:endDate:. startDate MUST NOT be nil! EXCEPTION: NAME:NSInternalInconsistencyException REASON:NGCalendarDateRange.m:37 Assertion failed in NGCalendarDateRange(instance), method initWithStartDate:endDate:. startDate MUST NOT be nil! INFO:(null) The relevant ICS file lines are the following ones: BEGIN:VEVENT UID:040000008200E00074C5B7101A82E00800000000901646A7234BCE01000000000000000010000000E9152C8FF1C27D488C91967FAAFCC2B0 RECURRENCE-ID:20140513T100000Z DURATION:PT1H CLASS:PUBLIC ATTENDEE;PARTSTAT=NEEDS-ACTION;ROLE=REQ-PARTICIPANT;RSVP=TRUE;CN=krsny >> Ann Thierry K:mailto:krsny@example.com END:VEVENT --- .../Appointments/SOGoAppointmentFolder.m | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/SoObjects/Appointments/SOGoAppointmentFolder.m b/SoObjects/Appointments/SOGoAppointmentFolder.m index 3ff2b26ec..a73b98e5f 100644 --- a/SoObjects/Appointments/SOGoAppointmentFolder.m +++ b/SoObjects/Appointments/SOGoAppointmentFolder.m @@ -1063,7 +1063,7 @@ firstInstanceCalendarDateRange: (NGCalendarDateRange *) fir { if ([dateRange containsDate: [component startDate]]) { - // We must pass nill to :container here in order to avoid re-entrancy issues. + // We must pass nil to :container here in order to avoid re-entrancy issues. newRecord = [self _fixupRecord: [component quickRecordFromContent: nil container: nil]]; [ma replaceObjectAtIndex: recordIndex withObject: newRecord]; } @@ -1080,15 +1080,20 @@ firstInstanceCalendarDateRange: (NGCalendarDateRange *) fir { // The recurrence id of the exception is outside the date range; // simply add the exception to the records array. - // We must pass nill to :container here in order to avoid re-entrancy issues. + // We must pass nil to :container here in order to avoid re-entrancy issues. newRecord = [self _fixupRecord: [component quickRecordFromContent: nil container: nil]]; - newRecordRange = [NGCalendarDateRange - calendarDateRangeWithStartDate: [newRecord objectForKey: @"startDate"] - endDate: [newRecord objectForKey: @"endDate"]]; - if ([dateRange doesIntersectWithDateRange: newRecordRange]) + if ([newRecord objectForKey: @"startDate"] && [newRecord objectForKey: @"endDate"]) { + newRecordRange = [NGCalendarDateRange + calendarDateRangeWithStartDate: [newRecord objectForKey: @"startDate"] + endDate: [newRecord objectForKey: @"endDate"]]; + if ([dateRange doesIntersectWithDateRange: newRecordRange]) [ma addObject: newRecord]; - else + else + newRecord = nil; + } else { + [self warnWithFormat: @"Recurrence %@ without dtstart or dtend. Ignoring", recurrenceId]; newRecord = nil; + } } if (newRecord) From c55eab640411a08d118130913313c91b87ca938e Mon Sep 17 00:00:00 2001 From: Xavy Bahillo Date: Thu, 1 Oct 2015 18:16:59 +0200 Subject: [PATCH 10/14] Changed Spanish string for view all (events/tasks) As current string was referring only to events --- UI/Scheduler/SpanishSpain.lproj/Localizable.strings | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UI/Scheduler/SpanishSpain.lproj/Localizable.strings b/UI/Scheduler/SpanishSpain.lproj/Localizable.strings index 99397e321..aed9f2908 100644 --- a/UI/Scheduler/SpanishSpain.lproj/Localizable.strings +++ b/UI/Scheduler/SpanishSpain.lproj/Localizable.strings @@ -231,7 +231,7 @@ /* Searching */ -"view_all" = "Todos los eventos"; +"view_all" = "Todos"; "view_today" = "Hoy"; "view_next7" = "Próximos 7 días"; "view_next14" = "Próximos 14 días"; From 498cec767b7433d9de21708bff601618db15fde8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= Date: Fri, 16 Oct 2015 12:08:50 +0200 Subject: [PATCH 11/14] Revert "Store c_content in sogo_cache_folder as a BLOB" This reverts commit b32951384568dc705d16cde4be910a886cb58b49. It was causing issues and we'll need to do more testing on this optimisation. --- SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m | 6 +++--- SoObjects/SOGo/SOGoCacheGCSObject.m | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m b/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m index 6adb55ef2..cef17775b 100644 --- a/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m +++ b/SoObjects/SOGo/GCSSpecialQueries+SOGoCacheObject.m @@ -57,7 +57,7 @@ @" c_lastmodified INT4 NOT NULL," @" c_version INT4 NOT NULL DEFAULT 0," @" c_deleted SMALLINT NOT NULL DEFAULT 0," - @" c_content BYTEA)"); + @" c_content TEXT)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } @@ -77,7 +77,7 @@ @" c_lastmodified INT NOT NULL," @" c_version INT NOT NULL DEFAULT 0," @" c_deleted TINYINT NOT NULL DEFAULT 0," - @" c_content BLOB)"); + @" c_content LONGTEXT)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } @@ -97,7 +97,7 @@ @" c_lastmodified INT4 NOT NULL," @" c_version INT4 NOT NULL DEFAULT 0," @" c_deleted SMALLINT NOT NULL DEFAULT 0," - @" c_content BLOB)"); + @" c_content CLOB)"); return [NSString stringWithFormat: sqlFolderFormat, tableName]; } diff --git a/SoObjects/SOGo/SOGoCacheGCSObject.m b/SoObjects/SOGo/SOGoCacheGCSObject.m index a4b32a74a..a2251173a 100644 --- a/SoObjects/SOGo/SOGoCacheGCSObject.m +++ b/SoObjects/SOGo/SOGoCacheGCSObject.m @@ -152,7 +152,7 @@ static EOAttribute *textColumn = nil; - (void) setupFromRecord: (NSDictionary *) record { NSInteger intValue; - NSData *content; + NSString *propsValue; NSDictionary *newValues; objectType = [[record objectForKey: @"c_type"] intValue]; @@ -166,10 +166,10 @@ static EOAttribute *textColumn = nil; dateWithTimeIntervalSince1970: (NSTimeInterval) intValue]); deleted = ([[record objectForKey: @"c_deleted"] intValue] > 0); version = [[record objectForKey: @"c_version"] intValue]; - content = [record objectForKey: @"c_content"]; - if ([content isNotNull]) + propsValue = [record objectForKey: @"c_content"]; + if ([propsValue isNotNull]) { - newValues = [[content dataByDecodingBase64] BSONValue]; + newValues = [[propsValue dataByDecodingBase64] BSONValue]; [properties addEntriesFromDictionary: newValues]; } else From d998786ea61d39c3ad3bfa27a9587bc1c12ad279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Tue, 20 Oct 2015 09:56:11 +0200 Subject: [PATCH 12/14] oc-mail: Always increase the CN when setting the message read flag And perform the real IMAP operation on save method as described by [MS-OXCFXICS] and [MS-OXCMSG] Section 2.2.3.3, these operations must be committed when SaveChangesMessage is called. As it is expected by Outlook to increase the change number when performing the `SetMessageReadFlag` ROP, if it is not done, the client tries indefinitely to store that. --- OpenChange/MAPIStoreMailMessage.m | 50 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/OpenChange/MAPIStoreMailMessage.m b/OpenChange/MAPIStoreMailMessage.m index 7ae2bd1d9..55ad92bd0 100644 --- a/OpenChange/MAPIStoreMailMessage.m +++ b/OpenChange/MAPIStoreMailMessage.m @@ -1645,27 +1645,11 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) - (enum mapistore_error) setReadFlag: (uint8_t) flag { - BOOL modified = NO; - BOOL alreadyRead = NO; - NSString *imapFlag = @"\\Seen"; - - alreadyRead = [[[sogoObject fetchCoreInfos] objectForKey: @"flags"] - containsObject: @"seen"]; - /* TODO: notifications should probably be emitted from here */ if (flag & CLEAR_READ_FLAG) - { - [sogoObject removeFlags: imapFlag]; - modified = alreadyRead; - } + [properties setObject: [NSNumber numberWithBool: NO] forKey: @"read_flag_set"]; else - { - [sogoObject addFlags: imapFlag]; - modified = !alreadyRead; - } - - if (modified) - [(MAPIStoreMailFolder *)[self container] synchroniseCache]; + [properties setObject: [NSNumber numberWithBool: YES] forKey: @"read_flag_set"]; return MAPISTORE_SUCCESS; } @@ -1708,7 +1692,10 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) - (void) save: (TALLOC_CTX *) memCtx { + BOOL modified = NO; + BOOL seen, storedSeenFlag; NSNumber *value; + NSString *imapFlag = @"\\Seen"; value = [properties objectForKey: MAPIPropertyKey (PR_FLAG_STATUS)]; if (value) @@ -1718,8 +1705,35 @@ _compareBodyKeysByPriority (id entry1, id entry2, void *data) [sogoObject addFlags: @"\\Flagged"]; else /* 0: unflagged, 1: follow up complete */ [sogoObject removeFlags: @"\\Flagged"]; + + modified = YES; } + /* Manage seen flag on save */ + value = [properties objectForKey: @"read_flag_set"]; + if (value) + { + seen = [value boolValue]; + storedSeenFlag = [[[sogoObject fetchCoreInfos] objectForKey: @"flags"] containsObject: @"seen"]; + /* We modify the flags anyway to generate a new change number */ + if (seen) + { + if (storedSeenFlag) + [sogoObject removeFlags: imapFlag]; + [sogoObject addFlags: imapFlag]; + } + else + { + if (!storedSeenFlag) + [sogoObject addFlags: imapFlag]; + [sogoObject removeFlags: imapFlag]; + } + modified = YES; + } + + if (modified) + [(MAPIStoreMailFolder *)[self container] synchroniseCache]; + if (mailIsSharingObject) [[self _sharingObject] saveWithMessage: self andSOGoObject: sogoObject]; From 8800cdf64115c013ce141640f8c1c481f77dcd11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= Date: Fri, 16 Oct 2015 11:11:50 +0200 Subject: [PATCH 13/14] oc-calendar: force cache synchronisation for a message before aborting Sometimes we're trying to get the `objectVersion` of a calendar message, but this message's entry is not in the cache. The method `synchroniseCache` won't work in this case, so we try to force the synchronisation of that particular message in order to get the change number before aborting. --- OpenChange/MAPIStoreGCSFolder.h | 1 + OpenChange/MAPIStoreGCSFolder.m | 109 +++++++++++++++++++++++++++++++ OpenChange/MAPIStoreGCSMessage.m | 33 +++++++--- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/OpenChange/MAPIStoreGCSFolder.h b/OpenChange/MAPIStoreGCSFolder.h index ecf9cee18..de2b9692b 100644 --- a/OpenChange/MAPIStoreGCSFolder.h +++ b/OpenChange/MAPIStoreGCSFolder.h @@ -41,6 +41,7 @@ /* synchronisation */ - (BOOL) synchroniseCache; +- (BOOL) synchroniseCacheFor: (NSString *) nameInContainer; - (void) updateVersionsForMessageWithKey: (NSString *) messageKey withChangeKey: (NSData *) oldChangeKey andPredecessorChangeList: (NSData *) pcl; diff --git a/OpenChange/MAPIStoreGCSFolder.m b/OpenChange/MAPIStoreGCSFolder.m index 8781ef22b..2dacdb4cf 100644 --- a/OpenChange/MAPIStoreGCSFolder.m +++ b/OpenChange/MAPIStoreGCSFolder.m @@ -548,6 +548,115 @@ static Class NSNumberK; return rc; } +- (BOOL) synchroniseCacheFor: (NSString *) nameInContainer +{ + /* Try to synchronise old messages in versions.plist cache using an + specific c_name. It returns a boolean indicating if the + synchronisation was carried out succesfully. + + It should be used as last resort, keeping synchroniseCache as the + main sync entry point. */ + + uint64_t changeNumber; + NSString *changeNumberStr; + NSData *changeKey; + NSNumber *cLastModified, *cDeleted, *cVersion; + EOFetchSpecification *fs; + EOQualifier *searchQualifier, *fetchQualifier; + NSArray *fetchResults; + NSDictionary *result; + NSMutableDictionary *currentProperties, *messages, *mapping, *messageEntry; + GCSFolder *ocsFolder; + static NSArray *fields; + + [versionsMessage reloadIfNeeded]; + currentProperties = [versionsMessage properties]; + + messages = [currentProperties objectForKey: @"Messages"]; + if (!messages) + { + messages = [NSMutableDictionary new]; + [currentProperties setObject: messages forKey: @"Messages"]; + [messages release]; + } + + messageEntry = [messages objectForKey: nameInContainer]; + if (!messageEntry) + { + /* Fetch the message by its name */ + if (!fields) + fields = [[NSArray alloc] + initWithObjects: @"c_name", @"c_version", @"c_lastmodified", + @"c_deleted", nil]; + + searchQualifier = [[EOKeyValueQualifier alloc] initWithKey: @"c_name" + operatorSelector: EOQualifierOperatorEqual + value: nameInContainer]; + fetchQualifier = [[EOAndQualifier alloc] + initWithQualifiers: searchQualifier, + [self contentComponentQualifier], + nil]; + [fetchQualifier autorelease]; + [searchQualifier release]; + + ocsFolder = [sogoObject ocsFolder]; + fs = [EOFetchSpecification + fetchSpecificationWithEntityName: [ocsFolder folderName] + qualifier: fetchQualifier + sortOrderings: nil]; + fetchResults = [ocsFolder fetchFields: fields + fetchSpecification: fs + ignoreDeleted: NO]; + + if ([fetchResults count] == 1) + { + result = [fetchResults objectAtIndex: 0]; + cLastModified = [result objectForKey: @"c_lastmodified"]; + cDeleted = [result objectForKey: @"c_deleted"]; + if ([cDeleted isKindOfClass: NSNumberK] && [cDeleted intValue]) + cVersion = [NSNumber numberWithInt: -1]; + else + cVersion = [result objectForKey: @"c_version"]; + + changeNumber = [[self context] getNewChangeNumber]; + changeNumberStr = [NSString stringWithUnsignedLongLong: changeNumber]; + + /* Create new message entry in Messages dict */ + messageEntry = [NSMutableDictionary new]; + [messages setObject: messageEntry forKey: nameInContainer]; + [messageEntry release]; + + /* Store cLastModified, cVersion and the change number */ + [messageEntry setObject: cLastModified forKey: @"c_lastmodified"]; + [messageEntry setObject: cVersion forKey: @"c_version"]; + [messageEntry setObject: changeNumberStr forKey: @"version"]; + + /* Store the change key */ + changeKey = [self getReplicaKeyFromGlobCnt: changeNumber >> 16]; + [self _setChangeKey: changeKey forMessageEntry: messageEntry]; + + /* Store the changeNumber -> cLastModified mapping */ + mapping = [currentProperties objectForKey: @"VersionMapping"]; + if (!mapping) + { + mapping = [NSMutableDictionary new]; + [currentProperties setObject: mapping forKey: @"VersionMapping"]; + [mapping release]; + } + [mapping setObject: cLastModified forKey: changeNumberStr]; + + /* Save the message */ + [versionsMessage save]; + return YES; + } + else + return NO; + } + + /* If message entry exists, then synchroniseCache did its job */ + return YES; +} + - (void) updateVersionsForMessageWithKey: (NSString *) messageKey withChangeKey: (NSData *) oldChangeKey andPredecessorChangeList: (NSData *) pcl diff --git a/OpenChange/MAPIStoreGCSMessage.m b/OpenChange/MAPIStoreGCSMessage.m index 747bb783a..2f7afeada 100644 --- a/OpenChange/MAPIStoreGCSMessage.m +++ b/OpenChange/MAPIStoreGCSMessage.m @@ -180,7 +180,8 @@ { uint64_t version = ULLONG_MAX; NSString *changeNumber; - + BOOL synced; + if (!isNew) { changeNumber = [(MAPIStoreGCSFolder *) container @@ -189,16 +190,28 @@ { [self warnWithFormat: @"attempting to get change number" @" by synchronising folder..."]; - [(MAPIStoreGCSFolder *) container synchroniseCache]; - changeNumber = [(MAPIStoreGCSFolder *) container - changeNumberForMessageWithKey: [self nameInContainer]]; - - if (changeNumber) - [self logWithFormat: @"got one"]; - else + synced = [(MAPIStoreGCSFolder *) container synchroniseCache]; + if (synced) { - [self errorWithFormat: @"still nothing. We crash!"]; - abort(); + changeNumber = [(MAPIStoreGCSFolder *) container + changeNumberForMessageWithKey: [self nameInContainer]]; + } + if (!changeNumber) + { + [self warnWithFormat: @"attempting to get change number" + @" by synchronising this specific message..."]; + synced = [(MAPIStoreGCSFolder *) container + synchroniseCacheFor: [self nameInContainer]]; + if (synced) + { + changeNumber = [(MAPIStoreGCSFolder *) container + changeNumberForMessageWithKey: [self nameInContainer]]; + } + if (!changeNumber) + { + [self errorWithFormat: @"still nothing. We crash!"]; + abort(); + } } } version = [changeNumber unsignedLongLongValue] >> 16; From 6f5bb3882cf5f332d97950dcb7785c6233af31cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrique=20J=2E=20Hern=C3=A1ndez=20Blasco?= Date: Wed, 21 Oct 2015 10:53:08 +0200 Subject: [PATCH 14/14] oc-mail: Sync expunged messages on first cache sync There is a use case where this has caused crashes: A message was hard-deleted using an IMAP client, this is the first message you deleted in that folder and you have cleared offline items in the client so a full sync is asked by upper layer. In that situation, the SyncLastDeleteChangeNumber version property is not set and return 0 in [getDeletedKeysFromChangeNumber:andCN:inTableType] making OpenChange to crash while it is asking for deleted fmids since a given change number. This is a regression from 18d7070c4a44a8437. --- OpenChange/MAPIStoreMailFolder.m | 53 +++++++++++++++----------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/OpenChange/MAPIStoreMailFolder.m b/OpenChange/MAPIStoreMailFolder.m index 9373bb0e5..7c8e4f8ac 100644 --- a/OpenChange/MAPIStoreMailFolder.m +++ b/OpenChange/MAPIStoreMailFolder.m @@ -594,8 +594,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) { BOOL rc = YES; uint64_t newChangeNum; - NSNumber *ti, *modseq, *initialLastModseq, *lastModseq, - *nextModseq; + NSNumber *ti, *modseq, *lastModseq, *nextModseq; NSString *changeNumber, *uid, *messageKey; uint64_t lastModseqNbr; EOQualifier *searchQualifier; @@ -634,7 +633,6 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) } lastModseq = [currentProperties objectForKey: @"SyncLastModseq"]; - initialLastModseq = lastModseq; if (lastModseq) { lastModseqNbr = [lastModseq unsignedLongLongValue]; @@ -718,40 +716,37 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data) } /* 2. we synchronise expunged UIDs */ - if (initialLastModseq) + fetchResults = [(SOGoMailFolder *) sogoObject + fetchUIDsOfVanishedItems: lastModseqNbr]; + + max = [fetchResults count]; + + changeNumber = nil; + for (count = 0; count < max; count++) { - fetchResults = [(SOGoMailFolder *) sogoObject - fetchUIDsOfVanishedItems: lastModseqNbr]; - - max = [fetchResults count]; - - changeNumber = nil; - for (count = 0; count < max; count++) + uid = [[fetchResults objectAtIndex: count] stringValue]; + if ([messages objectForKey: uid]) { - uid = [[fetchResults objectAtIndex: count] stringValue]; - if ([messages objectForKey: uid]) + if (!changeNumber) { - if (!changeNumber) - { - newChangeNum = [[self context] getNewChangeNumber]; - changeNumber = [NSString stringWithUnsignedLongLong: newChangeNum]; - } - [messages removeObjectForKey: uid]; - [self logWithFormat: @"Removed message entry for UID %@", uid]; - } - else - { - [self logWithFormat:@"Message entry not found for UID %@", uid]; + newChangeNum = [[self context] getNewChangeNumber]; + changeNumber = [NSString stringWithUnsignedLongLong: newChangeNum]; } + [messages removeObjectForKey: uid]; + [self logWithFormat: @"Removed message entry for UID %@", uid]; } - if (changeNumber) + else { - [currentProperties setObject: changeNumber - forKey: @"SyncLastDeleteChangeNumber"]; - [mapping setObject: lastModseq forKey: changeNumber]; - foundChange = YES; + [self logWithFormat:@"Message entry not found for UID %@", uid]; } } + if (changeNumber) + { + [currentProperties setObject: changeNumber + forKey: @"SyncLastDeleteChangeNumber"]; + [mapping setObject: lastModseq forKey: changeNumber]; + foundChange = YES; + } if (foundChange) {