RTFHandler: support charsets in plain text

Allow to switch charset in RTF plain text parts.

OpenChange/RTFHandler.m

@@ -32,7 +32,7 @@
 
 #define DEFAULT_CHARSET 1
 #define FONTNAME_LEN_MAX 100
-#define UTF8_FIRST_BYTE_LAST_CODEPOINT 0x7F
+
 //
 // Charset definitions. See http://msdn.microsoft.com/en-us/goglobal/bb964654 for all details.
 //
@@ -1215,7 +1215,6 @@ inline static void parseUl(RTFHandler *self, BOOL hasArg, int arg, RTFFormatting
   unsigned char c;
   NSData *d;
   NSString *s;
-  unichar uch;
 
   stack = [[RTFStack alloc] init];
   fontTable = nil;
@@ -1223,7 +1222,7 @@ inline static void parseUl(RTFHandler *self, BOOL hasArg, int arg, RTFFormatting
   defaultCharset = ansicpg1252;
   formattingOptions = nil;
 
-  _html = [[NSMutableData alloc] init];
+  _html = [[[NSMutableData alloc] init] autorelease];
   [_html appendBytes: "<html><meta charset='utf-8'><body>"  length: 34];
   
 
@@ -1254,7 +1253,6 @@ inline static void parseUl(RTFHandler *self, BOOL hasArg, int arg, RTFFormatting
               unsigned short index;
 
               const unsigned short * active_charset;
-
               if (formattingOptions && formattingOptions->charset)
                 active_charset = formattingOptions->charset;
               else
@@ -1439,18 +1437,15 @@ inline static void parseUl(RTFHandler *self, BOOL hasArg, int arg, RTFFormatting
           // We avoid appending NULL bytes or endlines
           if (c && (c != '\n')) 
             {
-              if (c <= UTF8_FIRST_BYTE_LAST_CODEPOINT) 
-                {
-                  // in this case utf8 and ascii encoding are the same
-                  [_html appendBytes: &c  length: 1];
-                }
-              else 
-                {
-                  uch = c;
-                  s = [NSString stringWithCharacters: &uch length: 1];
-                  d = [s dataUsingEncoding: NSUTF8StringEncoding];
-                  [_html appendData: d];
-                }
+              const unsigned short * active_charset;
+              if (formattingOptions && formattingOptions->charset)
+                active_charset = formattingOptions->charset;
+              else
+                active_charset = defaultCharset;
+              
+              s = [NSString stringWithCharacters: &(active_charset[c])  length: 1];
+              d = [s dataUsingEncoding: NSUTF8StringEncoding];
+              [_html appendData: d];
             }
           ADVANCE;
         }
@@ -1459,7 +1454,7 @@ inline static void parseUl(RTFHandler *self, BOOL hasArg, int arg, RTFFormatting
   [_html appendBytes: "</body></html>"  length: 14];
   
   [stack release];
-  return [_html autorelease];
+  return _html;
 }
 
 /* This method is for ease of testing and should not be used in normal operations */

Tests/Unit/Fixtures/cyr_event_ru_editor.rtf

@@ -0,0 +1 @@
+{\rtf1\adeflang1025\ansi\ansicpg1251\uc1\adeff0\deff0\stshfdbch0\stshfloch37\stshfhich37\stshfbi37\deflang1049\deflangfe1049\themelang1033\themelangfe0\themelangcs0{\fonttbl{\f0\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f0\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f37\fbidi \fswiss\fcharset0\fprq2{\*\panose 020f0502020204030204}Calibri;}{\f31500\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f31501\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f31502\fbidi \froman\fcharset0\fprq2{\*\panose 02040503050406030204}Cambria;}{\f31503\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f31504\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f31505\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f31506\fbidi \fswiss\fcharset0\fprq2{\*\panose 020f0502020204030204}Calibri;}{\f31507\fbidi \froman\fcharset0\fprq2{\*\panose 02020603050405020304}Times New Roman;}{\f179\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f180\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f182\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f183\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f184\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f185\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f186\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f187\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f179\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f180\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f182\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f183\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f184\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f185\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f186\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f187\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f549\fbidi \fswiss\fcharset238\fprq2 Calibri CE;}{\f550\fbidi \fswiss\fcharset204\fprq2 Calibri Cyr;}{\f552\fbidi \fswiss\fcharset161\fprq2 Calibri Greek;}{\f553\fbidi \fswiss\fcharset162\fprq2 Calibri Tur;}{\f556\fbidi \fswiss\fcharset186\fprq2 Calibri Baltic;}{\f557\fbidi \fswiss\fcharset163\fprq2 Calibri (Vietnamese);}{\f31508\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f31509\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f31511\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f31512\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f31513\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f31514\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f31515\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f31516\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f31518\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f31519\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f31521\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f31522\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f31523\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f31524\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f31525\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f31526\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f31528\fbidi \froman\fcharset238\fprq2 Cambria CE;}{\f31529\fbidi \froman\fcharset204\fprq2 Cambria Cyr;}{\f31531\fbidi \froman\fcharset161\fprq2 Cambria Greek;}{\f31532\fbidi \froman\fcharset162\fprq2 Cambria Tur;}{\f31535\fbidi \froman\fcharset186\fprq2 Cambria Baltic;}{\f31536\fbidi \froman\fcharset163\fprq2 Cambria (Vietnamese);}{\f31538\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f31539\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f31541\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f31542\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f31543\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f31544\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f31545\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f31546\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f31548\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f31549\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f31551\fbidi \froman\fcharset161\fprq2 Times New Roman Greek;}{\f31552\fbidi \froman\fcharset162\fprq2 Times New Roman Tur;}{\f31553\fbidi \froman\fcharset177\fprq2 Times New Roman (Hebrew);}{\f31554\fbidi \froman\fcharset178\fprq2 Times New Roman (Arabic);}{\f31555\fbidi \froman\fcharset186\fprq2 Times New Roman Baltic;}{\f31556\fbidi \froman\fcharset163\fprq2 Times New Roman (Vietnamese);}{\f31558\fbidi \froman\fcharset238\fprq2 Times New Roman CE;}{\f31559\fbidi \froman\fcharset204\fprq2 Times New Roman Cyr;}{\f31561\fbidi \froman\fcharset161\fpLine truncated

Tests/Unit/TestRTFHandler.m

@@ -44,26 +44,18 @@
     return nil;
 
   RTFHandler *handler = [[RTFHandler alloc] initWithData: rtf];
-  NSMutableData *data2 = [handler parse];
-  if (data2 == nil)
+  NSMutableData *data = [handler parse];
+  if (data == nil)
     {
       NSString *error = [NSString stringWithFormat: @"Couldn't parse RTF data:\n %s",
                          (char *)[rtf bytes]];           
       testWithMessage(NO, error);
     }
 
-  html = [[NSString alloc] initWithData: data2 encoding: NSUTF8StringEncoding];
+  html = [[NSString alloc] initWithData: data encoding: NSUTF8StringEncoding];
   if (html == nil) 
     {
-      html = [[NSString alloc] initWithData: data2 encoding: NSASCIIStringEncoding];
-    }
-  if (html == nil) 
-    {
-      html = [[NSString alloc] initWithData: data2 encoding: NSISOLatin1StringEncoding];
-    }
-  if (html == nil) 
-    {
-      NSString *error = [NSString stringWithFormat: @"Couldn't convert parsed data"];
+      NSString *error = [NSString stringWithFormat: @"Couldn't convert parsed data to UTF8 string"];
       testWithMessage(NO, error);
     }
   return html;
@@ -305,4 +297,13 @@
                  againstExpectedHTML: expected];  
 }
 
+- (void) test_cyr_event_ru_editor
+{
+  NSString *file =@"cyr_event_ru_editor.rtf";
+  NSString *expected=@"<html><meta charset='utf-8'><body><font face=\"Calibri\"><font face=\"Calibri Cyr\"><font color=\"#000000\">йчсмй</font></font><font color=\"#000000\"><br></font></font></body></html>";
+
+  [self checkHTMLConversionOfRTFFile: file
+                 againstExpectedHTML: expected];  
+}
+
 @end