diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index fb99dc01e..3621a78d9 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -778,6 +778,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -786,92 +813,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -1259,6 +1233,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 25f52895b..12467cbec 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -28706,6 +28706,33 @@ class serializer break; } } + + // A byte below 0x60 is a complete code point and needs no + // UTF-8 decoding. The scanner stops only at a control + // character, a quote, or a backslash in that range, so look up + // its escape sequence and write all 8 bytes of the table entry + // straight into the write buffer; only the length stored in the + // last byte of the entry is kept. + if (data[i] < 0x60) + { + // emit any bytes still pending in string_buffer first to + // preserve output order + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + } + const auto& escape = ascii_escapes()[data[i]]; + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size())) + { + flush(); + } + std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size()); + write_buffer_pos += static_cast(escape[7]); + bytes_after_last_accept = 0; + undumped_chars = 0; + continue; + } } const auto byte = static_cast(s[i]); @@ -28714,92 +28741,39 @@ class serializer { case UTF8_ACCEPT: // decode found a new code point { - switch (codepoint) + // the code point is at least 0x60 (smaller ones are handled + // above) and is not printable ASCII (which the scanner + // copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII + // character + if (EnsureAscii) { - case 0x08: // backspace + JSON_ASSERT(codepoint >= 0x7F); + if (error_handler == error_handler_t::keep) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; + // this character was buffered as raw bytes below + // in case it turned out to be part of an + // ill-formed sequence (which is kept as is); now + // that it decoded to a well-formed code point, + // undo that and \u-escape it like any other + // character instead + bytes = bytes_after_last_accept; } - - case 0x09: // horizontal tab + if (codepoint <= 0xFFFF) { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; + write_u_escape(bytes, static_cast(codepoint)); } - - case 0x0A: // newline + else { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // EnsureAscii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F))) - { - if (EnsureAscii && error_handler == error_handler_t::keep) - { - // this character was buffered as raw bytes - // below in case it turned out to be part of - // an ill-formed sequence (which is kept as - // is); now that it decoded to a well-formed - // code point, undo that and \u-escape it - // like any other character instead - bytes = bytes_after_last_accept; - } - if (codepoint <= 0xFFFF) - { - write_u_escape(bytes, static_cast(codepoint)); - } - else - { - write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); - write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; + write_u_escape(bytes, static_cast(0xD7C0u + (codepoint >> 10u))); + write_u_escape(bytes, static_cast(0xDC00u + (codepoint & 0x3FFu))); } } + else + { + // copy byte to buffer (all previous bytes of the + // sequence have been copied in the default case below) + string_buffer[bytes++] = s[i]; + } // write buffer and reset index; there must be 13 bytes // left, as this is the maximal number of bytes to be @@ -29187,6 +29161,121 @@ class serializer } } + /*! + @brief the JSON form of each character below 0x60 + + Entry c holds the escape sequence of character c - the short form for + \\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters - + or c itself if it needs no escaping, padded with zeros to 7 bytes. The last + byte is the length of the sequence. The 8 bytes of an entry are copied at + once, so writing an escape needs neither a branch nor a byte-wise loop. + */ + static const std::array, 0x60>& ascii_escapes() noexcept + { + static const std::array, 0x60> escapes = + { + { + {{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00 + {{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01 + {{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02 + {{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03 + {{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04 + {{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05 + {{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06 + {{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07 + {{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08 + {{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09 + {{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A + {{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B + {{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C + {{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D + {{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E + {{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F + {{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10 + {{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11 + {{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12 + {{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13 + {{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14 + {{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15 + {{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16 + {{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17 + {{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18 + {{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19 + {{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A + {{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B + {{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C + {{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D + {{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E + {{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F + {{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20 + {{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21 + {{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22 + {{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23 + {{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24 + {{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25 + {{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26 + {{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27 + {{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28 + {{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29 + {{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A + {{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B + {{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C + {{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D + {{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E + {{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F + {{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30 + {{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31 + {{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32 + {{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33 + {{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34 + {{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35 + {{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36 + {{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37 + {{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38 + {{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39 + {{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A + {{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B + {{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C + {{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D + {{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E + {{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F + {{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40 + {{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41 + {{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42 + {{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43 + {{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44 + {{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45 + {{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46 + {{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47 + {{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48 + {{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49 + {{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A + {{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B + {{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C + {{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D + {{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E + {{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F + {{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50 + {{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51 + {{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52 + {{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53 + {{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54 + {{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55 + {{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56 + {{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57 + {{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58 + {{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59 + {{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A + {{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B + {{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C + {{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D + {{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E + {{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F + } + }; + return escapes; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index c56c4f5c1..4913c30a6 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -20,8 +20,11 @@ using nlohmann::json; #include +#include #include #include +#include +#include #include "test_utils.hpp" @@ -923,66 +926,68 @@ TEST_CASE("serialization boundary values for the string buffer") // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes // < 13`), 13 being one more than the most a single code point can ever // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12 - // bytes). Every write into string_buffer that this check protects happens - // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so - // `bytes` only ever takes even values at the point the check runs - the - // tightest values actually reachable are therefore 498 (512 - 498 == 14, - // one simple escape away from the threshold) and 500 (512 - 500 == 12, - // where the flush fires immediately and resets bytes to 0). + // bytes). Characters below 0x60 (control characters, '"' and '\\') are + // escaped straight into the write buffer and never reach string_buffer, so + // under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a + // character below U+10000) or 12 (a surrogate pair). With 2-byte characters + // such as U+00E4, `bytes` takes multiples of 6 at the point the check runs; + // the tightest values reachable are therefore 498 (512 - 498 == 14, after + // 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the + // flush fires immediately and resets bytes to 0). - SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold") + SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold") { for (const int count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(count) - const json j = std::string(static_cast(count), '\n'); + std::string umlauts; std::string expected = "\""; for (int i = 0; i < count; ++i) { - expected += "\\n"; + umlauts += "\xC3\xA4"; + expected += "\\u00e4"; } expected += '"'; - CHECK(j.dump() == expected); + CHECK(json(umlauts).dump(-1, ' ', true) == expected); } } - SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair") + SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair") { // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that // write happens in a single step with no intermediate flush check, so // it is the write most exposed by an off-by-one in the "< 13" guard. - // A prefix of 249 newlines leaves exactly 14 bytes of headroom + // A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom // (512 - 498), the smallest margin the guard ever actually allows - // into a new code point; 250 newlines instead trigger the guard's own + // into a new code point; 84 characters instead trigger the guard's own // flush first, so the emoji starts from a freshly emptied (512-byte) - // buffer, and 251 repeats that with one more escape already past the + // buffer, and 85 repeats that with one more escape already past the // reset. Together they cover the margin the guard allows landing on, // one step before, and one step after - all must still produce the // identical, correct escapes. for (const int prefix_count : { - 249, 250, 251 + 83, 84, 85 }) { CAPTURE(prefix_count) - const std::string prefix(static_cast(prefix_count), '\n'); - const std::string emoji = "\xF0\x9F\x98\x80"; - const json j = prefix + emoji; - + std::string prefix; std::string expected_prefix; for (int i = 0; i < prefix_count; ++i) { - expected_prefix += "\\n"; + prefix += "\xC3\xA4"; + expected_prefix += "\\u00e4"; } + const std::string emoji = "\xF0\x9F\x98\x80"; + const json j = prefix + emoji; - // newline escaping does not depend on ensure_ascii: only the - // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair) + // without ensure_ascii, the whole string is copied unchanged std::string expected_raw = "\""; - expected_raw += expected_prefix; + expected_raw += prefix; expected_raw += emoji; expected_raw += '"'; std::string expected_ascii = "\""; @@ -1027,3 +1032,524 @@ TEST_CASE("serialization boundary values for the string buffer") } } } + +namespace +{ +// appends the escaped form of the ASCII character c to out; written +// independently of the table in serializer::ascii_escapes() +void append_escaped_ascii(std::string& out, const char c, const bool ensure_ascii) +{ + const auto byte = static_cast(c); + switch (byte) + { + case '\b': + out += "\\b"; + break; + case '\t': + out += "\\t"; + break; + case '\n': + out += "\\n"; + break; + case '\f': + out += "\\f"; + break; + case '\r': + out += "\\r"; + break; + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + default: + if (byte < 0x20 || (ensure_ascii && byte == 0x7F)) + { + // byte < 0x80, so the first two hex digits are 0 + const char* const hex = "0123456789abcdef"; + out += "\\u00"; + out += hex[byte >> 4U]; + out += hex[byte & 0x0FU]; + } + else + { + out += c; + } + break; + } +} + +// reference escaper for ASCII strings +std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii) +{ + std::string result = "\""; + for (const char c : s) + { + append_escaped_ascii(result, c, ensure_ascii); + } + result += '"'; + return result; +} +} // namespace + +TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer") +{ + SECTION("every ASCII character at every position") + { + // lengths up to 17 cover the 8-byte strides of the bulk scanner on + // both sides of the character, so the scanner stops at it from every + // offset + for (std::size_t length = 1; length <= 17; ++length) + { + for (std::size_t pos = 0; pos < length; ++pos) + { + for (unsigned int c = 0; c < 0x80; ++c) + { + std::string s(length, 'a'); + s[pos] = static_cast(c); + CAPTURE(length) + CAPTURE(pos) + CAPTURE(c) + const json j = s; + CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false)); + CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + } + } + } + + SECTION("runs of escaped characters") + { + std::string s; + for (unsigned int c = 0; c < 0x20; ++c) + { + s += static_cast(c); + } + s += R"("\"\)"; + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true)); + } + + SECTION("escapes straddle the end of the write buffer") + { + // the 8-byte store of a table entry needs room for all 8 bytes, even + // when the escape itself is shorter; a prefix of k plain characters + // shifts the 6-byte escapes across every offset of the 1024-byte + // write buffer + for (std::size_t k = 0; k < 8; ++k) + { + CAPTURE(k) + const std::string s = std::string(k, 'a') + std::string(400, '\x01'); + CHECK(json(s).dump() == reference_escape_ascii(s, false)); + CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]"); + } + } + + SECTION("pending \\u escapes of non-ASCII characters are written first") + { + // under ensure_ascii, U+00E4 is \u-escaped into the string buffer; + // a following escape from the table must not overtake it + const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\"; + CHECK(j.dump(-1, ' ', true) == "\"\\u00e4\\n\\u00e4\\\"\\ud83d\\ude00\\\\\""); + CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\""); + } + + SECTION("escapes after invalid UTF-8 for every error handler") + { + const json j = "a\xFF\n\xC3\""; + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == "\"a\\n\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"a\\ufffd\\n\\ufffd\\\"\""); + CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\""); + } +} + +namespace +{ +// expected result of dumping a string: the output, or (with +// error_handler_t::strict) the message of the type_error.316 exception +struct expected_dump +{ + bool throws = false; + std::string text{}; // NOLINT(readability-redundant-member-init) +}; + +std::string upper_hex_byte(const unsigned char byte) +{ + const char* const hex = "0123456789ABCDEF"; + return std::string(1, hex[byte >> 4U]) + hex[byte & 0x0FU]; +} + +void append_u_escape(std::string& out, const std::uint32_t code_unit) +{ + const char* const hex = "0123456789abcdef"; + out += "\\u"; + for (int shift = 12; shift >= 0; shift -= 4) + { + out += hex[(code_unit >> static_cast(shift)) & 0x0FU]; + } +} + +void append_replacement_character(std::string& out, const bool ensure_ascii) +{ + out += ensure_ascii ? "\\ufffd" : "\xEF\xBF\xBD"; +} + +// reference serializer for arbitrary byte strings, written from the documented +// behavior of dump() and independently of serializer::dump_escaped(): UTF-8 is +// decoded strictly (well-formed sequences as in Table 3-7 of the Unicode +// Standard), and each ill-formed sequence is handled as the error handler says +expected_dump reference_dump_string(const std::string& s, const bool ensure_ascii, const json::error_handler_t error_handler) +{ + expected_dump result; + std::string& out = result.text; + out = "\""; + + // handle the ill-formed bytes s[begin, end), of which s[index] is the byte + // that could not be decoded; returns false if an exception is expected + const auto ill_formed = [&](const std::size_t begin, const std::size_t end, const std::size_t index) + { + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] invalid UTF-8 byte at index " + std::to_string(index) + ": 0x" + upper_hex_byte(static_cast(s[index])); + return false; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + // kept as is, even with ensure_ascii + out.append(s, begin, end - begin); + break; + } + return true; + }; + + std::size_t i = 0; + while (i < s.size()) + { + const auto lead = static_cast(s[i]); + if (lead < 0x80) + { + append_escaped_ascii(out, s[i], ensure_ascii); + ++i; + continue; + } + + // number of bytes of the sequence and the range of its second byte + std::size_t length = 0; + std::uint32_t code_point = 0; + unsigned char lower = 0x80; + unsigned char upper = 0xBF; + if (lead >= 0xC2 && lead <= 0xDF) + { + length = 2; + code_point = lead & 0x1FU; + } + else if (lead >= 0xE0 && lead <= 0xEF) + { + length = 3; + code_point = lead & 0x0FU; + lower = (lead == 0xE0) ? 0xA0 : 0x80; // no overlong encodings + upper = (lead == 0xED) ? 0x9F : 0xBF; // no surrogates + } + else if (lead >= 0xF0 && lead <= 0xF4) + { + length = 4; + code_point = lead & 0x07U; + lower = (lead == 0xF0) ? 0x90 : 0x80; // no overlong encodings + upper = (lead == 0xF4) ? 0x8F : 0xBF; // nothing above U+10FFFF + } + else + { + // a byte that cannot start a sequence is consumed + if (!ill_formed(i, i + 1, i)) + { + return result; + } + ++i; + continue; + } + + std::size_t j = i + 1; + for (; j < i + length; ++j) + { + if (j == s.size()) + { + // the string ends in the middle of the sequence + switch (error_handler) + { + case json::error_handler_t::strict: + result.throws = true; + out = "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x" + upper_hex_byte(static_cast(s.back())); + return result; + case json::error_handler_t::ignore: + break; + case json::error_handler_t::replace: + append_replacement_character(out, ensure_ascii); + break; + case json::error_handler_t::keep: + default: + out.append(s, i, s.size() - i); + break; + } + out += '"'; + return result; + } + + const auto byte = static_cast(s[j]); + if (byte < lower || byte > upper) + { + break; + } + code_point = (code_point << 6U) | (byte & 0x3FU); + lower = 0x80; + upper = 0xBF; + } + + if (j < i + length) + { + // the sequence was abandoned at s[j]; that byte is read again as + // the start of a new character + if (!ill_formed(i, j, j)) + { + return result; + } + i = j; + continue; + } + + if (!ensure_ascii) + { + out.append(s, i, length); + } + else if (code_point <= 0xFFFF) + { + append_u_escape(out, code_point); + } + else + { + append_u_escape(out, 0xD800U + ((code_point - 0x10000U) >> 10U)); + append_u_escape(out, 0xDC00U + ((code_point - 0x10000U) & 0x3FFU)); + } + i = j; + } + + out += '"'; + return result; +} + +std::string hex_dump(const std::string& s) +{ + std::string result; + for (const char c : s) + { + result += upper_hex_byte(static_cast(c)); + result += ' '; + } + return result; +} + +// compares dump() of s with the reference for every error handler, with +// ensure_ascii on and off; CHECKs only on a mismatch to keep the sweep fast; +// error_handler_t::strict can be skipped where the position of the string +// buffer does not matter, as throwing is slow on some platforms +void check_dump_against_reference(const std::string& s, const bool with_strict = true) +{ + const json j = s; + const std::array error_handlers = + { + { + json::error_handler_t::strict, json::error_handler_t::ignore, + json::error_handler_t::replace, json::error_handler_t::keep + } + }; + for (const auto error_handler : error_handlers) + { + if (!with_strict && error_handler == json::error_handler_t::strict) + { + continue; + } + for (const bool ensure_ascii : + { + false, true + }) + { + const expected_dump expected = reference_dump_string(s, ensure_ascii, error_handler); + expected_dump actual; + int id = 0; + try + { + actual.text = j.dump(-1, ' ', ensure_ascii, error_handler); + } + catch (const json::exception& e) + { + actual.throws = true; + actual.text = e.what(); + id = e.id; + } + if (actual.throws != expected.throws || actual.text != expected.text || (actual.throws && id != 316)) + { + const std::string input = hex_dump(s); + CAPTURE(input) + CAPTURE(ensure_ascii) + CAPTURE(static_cast(error_handler)) + CHECK(actual.throws == expected.throws); + CHECK(actual.text == expected.text); + CHECK((!actual.throws || id == 316)); + } + } + } +} +} // namespace + +TEST_CASE("invalid UTF-8 at every position is handled like the reference") +{ + // ill-formed byte sequences; those that end in a truncated sequence are + // also cut short by the filler that follows, or by the end of the string + const std::vector patterns = + { + // stray continuation bytes + "\x80", "\xBF", + // bytes that cannot start a sequence + "\xC0", "\xC1", "\xF5", "\xF6", "\xF7", "\xF8", "\xF9", "\xFA", "\xFB", "\xFC", "\xFD", "\xFE", "\xFF", + // overlong encodings + "\xC0\xAF", "\xE0\x80\x80", "\xF0\x80\x80\x80", + // UTF-16 surrogates + "\xED\xA0\x80", "\xED\xBF\xBF", + // a code point above U+10FFFF + "\xF4\x90\x80\x80", + // sequences cut short by ASCII, a control character, a quote, a + // backslash, DEL, or the lead byte of a well-formed character + "\xC3" "a", "\xE2\x82" "\n", "\xE2\x82\"", "\xF0\x9F\x98\\", "\xC3\x7F", "\xC3\xC3\xA4", "\xE2\xF0\x9F\x98\x80", + // truncated sequences + "\xC3", "\xE2\x82", "\xF0\x9F\x98" + }; + + // well-formed characters of 1, 2, 3, and 4 bytes + const std::vector fillers = {"a", "\xC3\xA4", "\xE2\x82\xAC", "\xF0\x9F\x98\x80"}; + + const auto make_string = [](const std::string & filler, const std::size_t before, const std::string & pattern, const std::size_t after) + { + std::string s; + for (std::size_t k = 0; k < before; ++k) + { + s += filler; + } + s += pattern; + for (std::size_t k = 0; k < after; ++k) + { + s += filler; + } + return s; + }; + + SECTION("the reference") + { + // the reference itself, against hand-written results + const std::string s = "a\xFF\n\xC3\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82"; + CHECK(reference_dump_string(s, false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF"); + CHECK(reference_dump_string(s.substr(2), false, json::error_handler_t::strict).text == "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0x22"); + CHECK(reference_dump_string("\xF0\x9F\x98\x80\xE2\x82", true, json::error_handler_t::strict).text == "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x82"); + CHECK(reference_dump_string(s, false, json::error_handler_t::ignore).text == "\"a\\n\\\"\xF0\x9F\x98\x80\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::ignore).text == "\"a\\n\\\"\\ud83d\\ude00\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::replace).text == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\xEF\xBF\xBD\xEF\xBF\xBD\xF0\x9F\x98\x80\xEF\xBF\xBD\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::replace).text == "\"a\\ufffd\\n\\ufffd\\\"\\ufffd\\ufffd\\ud83d\\ude00\\ufffd\""); + CHECK(reference_dump_string(s, false, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\xF0\x9F\x98\x80\xE2\x82") + "\""); + CHECK(reference_dump_string(s, true, json::error_handler_t::keep).text == "\"" + std::string("a\xFF\\n\xC3\\\"\xE0\x80\\ud83d\\ude00\xE2\x82") + "\""); + check_dump_against_reference(s); + } + + SECTION("short strings") + { + // strings of up to 17 bytes put the pattern at every offset of an + // 8-byte stride of the bulk scanners, with whole strides on both + // sides; lengths around 24, 32, and 40 bytes cross the 16- and 32-byte + // strides; error_handler_t::strict is checked for up to 17 bytes of + // ASCII and 8 bytes of other characters only, as throwing is slow on + // some platforms + for (const auto& filler : fillers) + { + for (std::size_t length = 0; length * filler.size() <= 40; ++length) + { + const std::size_t bytes = length * filler.size(); + if (filler.size() == 1 && bytes > 17 && (bytes + 1) % 8 > 2) + { + continue; + } + for (std::size_t pos = 0; pos <= length; ++pos) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, length - pos), bytes <= (filler.size() == 1 ? 17 : 8)); + } + } + } + } + } + + SECTION("strings crossing the end of the string buffer") + { + // characters that are not copied in bulk go through a 512-byte string + // buffer that is written out once fewer than 13 bytes are left; with + // ensure_ascii, each 2-byte character takes 6 bytes of it and each + // 4-byte character 12, so the pattern meets the end of the buffer after + // about 84 or 42 characters (3-byte characters behave like 2-byte + // ones); plain ASCII, copied in bulk, is checked around 512 bytes; + // positions further away are sampled; error_handler_t::strict throws + // at the first ill-formed byte, regardless of the string buffer, and is + // covered above + for (const auto& filler : fillers) + { + if (filler.size() == 3) + { + continue; + } + const std::size_t escaped_size = filler.size() == 1 ? 1 : filler.size() * 3; + const std::size_t middle = 506 / escaped_size; + const std::size_t window = filler.size() == 1 ? 8 : 16; + for (std::size_t pos = 0; pos <= middle + window; pos += (pos + window >= middle) ? 1 : 29) + { + for (const std::size_t after : + { + std::size_t{0}, std::size_t{12} + }) + { + for (const auto& pattern : patterns) + { + check_dump_against_reference(make_string(filler, pos, pattern, after), false); + } + } + } + } + } + + SECTION("repeated invalid sequences at every offset of the string buffer") + { + // ill-formed sequences that are kept or replaced fill the string + // buffer without a bulk copy in between; a prefix of 0 to 12 bytes + // 0xFF (kept as 1 byte, replaced by 3 or 6) shifts them across every + // offset of the end of the buffer + for (const auto& pattern : patterns) + { + for (std::size_t shift = 0; shift <= 12; ++shift) + { + std::string s(shift, '\xFF'); + while (s.size() < 600) + { + s += pattern; + } + check_dump_against_reference(s, shift == 0); + check_dump_against_reference(s + "\xF0\x9F\x98\x80", false); + } + } + } +}