diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index d1c78346e..8c16bc5f1 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1027,6 +1027,27 @@ class serializer const auto byte = static_cast(s[i]); + if (EnsureAscii && state == UTF8_ACCEPT && byte >= 0x80) + { + // a run of well-formed multi-byte characters is escaped + // straight into the write buffer; an ill-formed sequence + // ends the run and is left to the byte-at-a-time path + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + bytes_after_last_accept = 0; + } + const std::size_t escaped = write_u_escapes(reinterpret_cast(s.data()) + i, s.size() - i); + if (escaped != 0) + { + bytes_after_last_accept = 0; + undumped_chars = 0; + i += escaped - 1; + continue; + } + } + switch (decode(state, codepoint, byte)) { case UTF8_ACCEPT: // decode found a new code point @@ -1603,6 +1624,75 @@ class serializer return escapes; } + /*! + @brief escape the well-formed multi-byte characters at the start of @a data + + Writes the "\\uXXXX" escape of each character (two of them for a character + above U+FFFF) straight into the write buffer, which is why the caller must + have written out what is pending in @ref string_buffer. Stops at the first + byte that is below 0x80 or starts an ill-formed or truncated sequence, which + the caller handles. + + @return the number of bytes of @a data that have been escaped + */ + JSON_HEDLEY_NEVER_INLINE std::size_t write_u_escapes(const unsigned char* data, std::size_t n) + { + constexpr const char* nibble_to_hex = "0123456789abcdef"; + std::size_t i = 0; + while (i < n && data[i] >= 0x80) + { + const std::size_t length = validate_one_utf8(data + i, n - i); + if (length == 0) + { + break; + } + + const auto b0 = static_cast(data[i]); + const auto b1 = static_cast(data[i + 1]); + std::uint32_t codepoint = 0; + switch (length) + { + case 2: + codepoint = ((b0 & 0x1Fu) << 6u) | (b1 & 0x3Fu); + break; + case 3: + codepoint = ((b0 & 0x0Fu) << 12u) | ((b1 & 0x3Fu) << 6u) | (static_cast(data[i + 2]) & 0x3Fu); + break; + default: + codepoint = ((b0 & 0x07u) << 18u) | ((b1 & 0x3Fu) << 12u) | ((static_cast(data[i + 2]) & 0x3Fu) << 6u) | (static_cast(data[i + 3]) & 0x3Fu); + break; + } + + // at most two escapes of six bytes each + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < 12)) + { + flush(); + } + char* d = write_buffer.data() + write_buffer_pos; + if (codepoint > 0xFFFFu) + { + d = put_u_escape(d, static_cast(0xD7C0u + (codepoint >> 10u)), nibble_to_hex); + codepoint = 0xDC00u + (codepoint & 0x3FFu); + } + d = put_u_escape(d, static_cast(codepoint), nibble_to_hex); + write_buffer_pos = static_cast(d - write_buffer.data()); + i += length; + } + return i; + } + + static char* put_u_escape(char* d, const std::uint16_t codeunit, const char* nibble_to_hex) noexcept + { + const auto unit = static_cast(codeunit); + d[0] = '\\'; + d[1] = 'u'; + d[2] = nibble_to_hex[(unit >> 12u) & 0x0Fu]; + d[3] = nibble_to_hex[(unit >> 8u) & 0x0Fu]; + d[4] = nibble_to_hex[(unit >> 4u) & 0x0Fu]; + d[5] = nibble_to_hex[unit & 0x0Fu]; + return d + 6; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 346cfebd0..80690adc0 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -31171,6 +31171,27 @@ class serializer const auto byte = static_cast(s[i]); + if (EnsureAscii && state == UTF8_ACCEPT && byte >= 0x80) + { + // a run of well-formed multi-byte characters is escaped + // straight into the write buffer; an ill-formed sequence + // ends the run and is left to the byte-at-a-time path + if (bytes != 0) + { + put_buffer(string_buffer, bytes); + bytes = 0; + bytes_after_last_accept = 0; + } + const std::size_t escaped = write_u_escapes(reinterpret_cast(s.data()) + i, s.size() - i); + if (escaped != 0) + { + bytes_after_last_accept = 0; + undumped_chars = 0; + i += escaped - 1; + continue; + } + } + switch (decode(state, codepoint, byte)) { case UTF8_ACCEPT: // decode found a new code point @@ -31747,6 +31768,75 @@ class serializer return escapes; } + /*! + @brief escape the well-formed multi-byte characters at the start of @a data + + Writes the "\\uXXXX" escape of each character (two of them for a character + above U+FFFF) straight into the write buffer, which is why the caller must + have written out what is pending in @ref string_buffer. Stops at the first + byte that is below 0x80 or starts an ill-formed or truncated sequence, which + the caller handles. + + @return the number of bytes of @a data that have been escaped + */ + JSON_HEDLEY_NEVER_INLINE std::size_t write_u_escapes(const unsigned char* data, std::size_t n) + { + constexpr const char* nibble_to_hex = "0123456789abcdef"; + std::size_t i = 0; + while (i < n && data[i] >= 0x80) + { + const std::size_t length = validate_one_utf8(data + i, n - i); + if (length == 0) + { + break; + } + + const auto b0 = static_cast(data[i]); + const auto b1 = static_cast(data[i + 1]); + std::uint32_t codepoint = 0; + switch (length) + { + case 2: + codepoint = ((b0 & 0x1Fu) << 6u) | (b1 & 0x3Fu); + break; + case 3: + codepoint = ((b0 & 0x0Fu) << 12u) | ((b1 & 0x3Fu) << 6u) | (static_cast(data[i + 2]) & 0x3Fu); + break; + default: + codepoint = ((b0 & 0x07u) << 18u) | ((b1 & 0x3Fu) << 12u) | ((static_cast(data[i + 2]) & 0x3Fu) << 6u) | (static_cast(data[i + 3]) & 0x3Fu); + break; + } + + // at most two escapes of six bytes each + if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < 12)) + { + flush(); + } + char* d = write_buffer.data() + write_buffer_pos; + if (codepoint > 0xFFFFu) + { + d = put_u_escape(d, static_cast(0xD7C0u + (codepoint >> 10u)), nibble_to_hex); + codepoint = 0xDC00u + (codepoint & 0x3FFu); + } + d = put_u_escape(d, static_cast(codepoint), nibble_to_hex); + write_buffer_pos = static_cast(d - write_buffer.data()); + i += length; + } + return i; + } + + static char* put_u_escape(char* d, const std::uint16_t codeunit, const char* nibble_to_hex) noexcept + { + const auto unit = static_cast(codeunit); + d[0] = '\\'; + d[1] = 'u'; + d[2] = nibble_to_hex[(unit >> 12u) & 0x0Fu]; + d[3] = nibble_to_hex[(unit >> 8u) & 0x0Fu]; + d[4] = nibble_to_hex[(unit >> 4u) & 0x0Fu]; + d[5] = nibble_to_hex[unit & 0x0Fu]; + return d + 6; + } + /*! * @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer * diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 338eef291..6164e5fdb 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -1631,3 +1631,42 @@ TEST_CASE("short strings are written like the reference, as values and as keys") } } } + +TEST_CASE("runs of multi-byte characters are escaped like the reference under ensure_ascii") +{ + // a run of well-formed multi-byte characters is escaped straight into the + // write buffer; it has to flush that mid-run, stop at an ASCII character or + // an ill-formed sequence and leave those to the general path, and pick up + // behind what the general path has pending + const std::vector characters = + { + "\xc2\x80", "\xdf\xbf", "\xe0\xa0\x80", "\xe2\x82\xac", "\xed\x9f\xbf", "\xef\xbf\xbf", + "\xf0\x90\x80\x80", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf" + }; + const std::vector interruptions = + { + "", "a", "\n", "\"", "\x7f", "\xff", "\x80", "\xc3", "\xe2\x82", "\xf0\x9f\x98", "\xed\xa0\x80" + }; + + for (std::size_t shift = 0; shift <= 13; ++shift) + { + for (const auto& interruption : interruptions) + { + for (const std::size_t count : + { + std::size_t{1}, std::size_t{2}, std::size_t{85}, std::size_t{200} + }) + { + std::string s(shift, 'x'); + for (std::size_t i = 0; i < count; ++i) + { + s += characters[(i + shift) % characters.size()]; + } + s += interruption; + s += characters[shift % characters.size()]; + s += characters[(shift + 3) % characters.size()]; + check_dump_against_reference(s, shift < 2); + } + } + } +}