From 272d07b48bd7542e615bc734cc89120b573bba52 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 21 Oct 2026 22:23:30 +0200 Subject: [PATCH] Scan the strings of json_view's dump() with ensure_ascii 16 bytes at a time With ensure_ascii, a string of the source that has a byte >= 0x7F was written by a generic path: put() per piece and a validate-and-decode step for every character beyond ASCII. It now goes through a writer with the cursor in a local variable. The part of the string that has nothing to escape is found 16 bytes at a time (SSE2 or NEON; the existing 8-byte scan without vector support) and copied; strings that are ASCII throughout are still decided by the 8-byte check of dump() with ensure_ascii (a 16-byte scan for them was 2-5 % slower for short strings). The two-byte sequences (lead C2..DF) and the three-byte sequences whose continuation bytes may have any value (lead E1..EF except ED) are decoded with a few compares; E0, ED, four-byte sequences, and damaged input go through validate_one_utf8() as before, so damaged images still throw type_error.316. dump(-1, ' ', true) instructions per call, macOS arm64 (develop -> now); the compact dump without ensure_ascii and dumps of ASCII text are unchanged: twitter 5.42M -> 3.57M citm 3.28M -> 3.24M jeopardy(4k) 5.78M -> 5.65M CJK text, raw 66.5M -> 35.3M escaped \u 32.7M -> 17.0M Latin text with accents 11.5M -> 9.4M ASCII text 4.40M -> 4.40M Linux instructions (GCC 13 / Clang 18), dump(-1, ' ', true): twitter -9 % / -33 % CJK text -14 % / -47 % Latin text -8 % / -18 % citm -0.2 % / -0.4 % ASCII text -0.1 % / -0.3 % The compact dump without ensure_ascii is unchanged (canada 0.0 %). The unit test covers every character of the Basic Multilingual Plane, and a character that needs an escape at each position of strings of 0..40 bytes. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/serializer.hpp | 284 ++++++++++++++----- single_include/nlohmann/json_view.hpp | 285 +++++++++++++++----- tests/src/unit-json_view_dump.cpp | 94 +++++++ 3 files changed, 519 insertions(+), 144 deletions(-) diff --git a/include/nlohmann/detail/view/serializer.hpp b/include/nlohmann/detail/view/serializer.hpp index 6cc282dd4..061780cf5 100644 --- a/include/nlohmann/detail/view/serializer.hpp +++ b/include/nlohmann/detail/view/serializer.hpp @@ -23,6 +23,7 @@ #include #include #include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -189,6 +190,70 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept return true; } +#if NLOHMANN_VIEW_VECTOR +/// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a +/// backslash, a control character, DEL, or not ASCII; a signed compare with +/// 0x20 finds control characters and non-ASCII bytes at once), one bit per +/// byte (x86) or one nibble per byte (NEON) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + return static_cast(static_cast(_mm_movemask_epi8(special))); +#endif +} + +/// the width in bits of a byte in the mask of ascii_stop_mask() +constexpr unsigned ascii_stop_bits() noexcept +{ + return NLOHMANN_VIEW_NEON ? 4u : 1u; +} +#endif + +/// As find_ascii_copyable_run(), 16 bytes per step. end is the end of the +/// buffer that holds the n bytes at p (a short run is read as 16 bytes if +/// they are in the buffer). +inline std::size_t ascii_copyable_run(const unsigned char* p, std::size_t n, const unsigned char* end) noexcept +{ +#if NLOHMANN_VIEW_VECTOR + if (n >= 16) + { + std::size_t i = 0; + for (; i + 16 <= n; i += 16) + { + const std::uint64_t mask = ascii_stop_mask(p + i); + if (mask != 0) + { + return i + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()); + } + } + if (i == n) + { + return n; + } + // the last 16 bytes: those before i are known to be copyable + const std::uint64_t mask = ascii_stop_mask(p + n - 16); + return mask != 0 ? n - 16 + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()) : n; + } + if (end - p >= 16) + { + // (the bytes after the run are masked out) + const std::uint64_t mask = ascii_stop_mask(p) & ((std::uint64_t{1} << (n * ascii_stop_bits())) - 1u); + return mask != 0 ? static_cast(count_trailing_zeros(mask)) / ascii_stop_bits() : n; + } +#else + static_cast(end); +#endif + return find_ascii_copyable_run(p, n); +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -440,10 +505,151 @@ class view_serializer return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); } - /// a string with ensure_ascii (dump_text()), out of line - NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + /*! + @brief a string with ensure_ascii: the characters that are not printable ASCII as \\uXXXX + (a character beyond U+FFFF as a surrogate pair); strings of a document + are valid UTF-8, except in a damaged image loaded with image_check::bounds, + for which this throws what basic_json::dump() throws for the string + */ + NLOHMANN_VIEW_NOINLINE void write_ascii(const node& n) { - write_string(n); + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const std::size_t size = n.len; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for k bytes + const auto room = [&](std::size_t k) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(lim - w) < k)) + { + m_out.set_cursor(w); + m_out.reserve(k); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + static constexpr const char* hex = "0123456789abcdef"; + // \\uXXXX + const auto u_escape = [&](std::uint32_t u) + { + w[0] = '\\'; + w[1] = 'u'; + w[2] = hex[(u >> 12u) & 0xFu]; + w[3] = hex[(u >> 8u) & 0xFu]; + w[4] = hex[(u >> 4u) & 0xFu]; + w[5] = hex[u & 0xFu]; + w += 6; + }; + room(1); + *w++ = '"'; + std::size_t i = 0; + // (a string of the source starts with a run that has nothing to escape, 16 bytes per step) + const std::size_t prefix = (n.flags & node_flags::storage) == 0 ? ascii_copyable_run(s, size, reinterpret_cast(m_doc.src + m_doc.size)) : 0; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (prefix != 0) + { + room(prefix); + std::memcpy(w, s, prefix); + w += prefix; + i = prefix; + } + while (i < size) + { + // (a character that needs an escape sequence is not worth a call) + const std::size_t run = is_ascii_copyable(s[i]) ? find_ascii_copyable_run(s + i, size - i) : 0; + if (run != 0) + { + room(run); + std::memcpy(w, s + i, run); + w += run; + i += run; + continue; + } + room(12); + if (s[i] >= 0x80) + { + const unsigned char lead = s[i]; + // (the two-byte sequences of Latin text and the three-byte sequences of most other + // text, with continuation bytes of any value: the ranges that are restricted (E0, + // ED, and the sequences of four bytes) go through validate_one_utf8()) + if (lead >= 0xE1 && lead <= 0xEF && lead != 0xED && size - i >= 3 && (s[i + 1] & 0xC0u) == 0x80 && (s[i + 2] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x0Fu) << 12u) | ((s[i + 1] & 0x3Fu) << 6u) | (s[i + 2] & 0x3Fu)); + i += 3; + continue; + } + if (lead >= 0xC2 && lead <= 0xDF && size - i >= 2 && (s[i + 1] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x1Fu) << 6u) | (s[i + 1] & 0x3Fu)); + i += 2; + continue; + } + const std::size_t len = validate_one_utf8(s + i, size - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) + { + m_out.set_cursor(w); + invalid_utf8(s, size); + return; // LCOV_EXCL_LINE (invalid_utf8 throws) + } + std::uint32_t codepoint = s[i] & (0xFFu >> (len + 1)); + for (std::size_t k = 1; k < len; ++k) + { + codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); + } + i += len; + if (codepoint <= 0xFFFF) + { + u_escape(codepoint); + } + else + { + u_escape(0xD7C0u + (codepoint >> 10u)); + u_escape(0xDC00u + (codepoint & 0x3FFu)); + } + continue; + } + // a quote, a backslash, or a control character (or DEL) + const unsigned char c = s[i++]; + char escape = 0; + switch (c) + { + case 0x08: + escape = 'b'; + break; + case 0x09: + escape = 't'; + break; + case 0x0A: + escape = 'n'; + break; + case 0x0C: + escape = 'f'; + break; + case 0x0D: + escape = 'r'; + break; + case 0x22: + escape = '"'; + break; + case 0x5C: + escape = '\\'; + break; + default: + break; + } + if (escape != 0) + { + w[0] = '\\'; + w[1] = escape; + w += 2; + } + else + { + u_escape(c); + } + } + room(1); + *w++ = '"'; + m_out.set_cursor(w); } /// the copies of dump_text() that are not fixed-size moves (long @@ -471,8 +677,8 @@ class view_serializer the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a - string of the source is copied only if it has no byte >= 0x7F (see - is_ascii_without_delete()); any other string is written by write_string(). + string of the source is copied only if it has nothing to escape (see + copied()); any other string is written by write_ascii(). */ template void dump_text(const node* root) @@ -554,7 +760,7 @@ class view_serializer m_out.set_cursor(w); if (enabled(Ascii)) { - write_ascii_string(n); + write_ascii(n); } else { @@ -869,72 +1075,6 @@ class view_serializer m_out.put(s.data(), s.size()); } - void write_string(const node& n) - { - const char* const s = m_doc.str(n); - m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) - { - // a string of the source without escape sequences has nothing to - // escape (with ensure_ascii, if it is printable ASCII) - m_out.put(s, n.len); - } - else if (m_style.ensure_ascii) - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - else - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - m_out.put('"'); - } - - /// as serializer::dump_escaped(); strings of a document are valid UTF-8, - /// except in a damaged image loaded with image_check::bounds, for which - /// this throws what basic_json::dump() throws for the string - template - void write_escaped(const unsigned char* s, std::size_t n) - { - std::size_t i = 0; - while (i < n) - { - std::size_t run = 0; - if (!EnsureAscii) - { - run = string_bulk_run(s + i, n - i); - } - else if (is_ascii_copyable(s[i])) - { - run = find_ascii_copyable_run(s + i, n - i); - } - if (run != 0) - { - m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - i += run; - continue; - } - std::uint32_t codepoint = s[i]; - std::size_t len = 1; - if (codepoint >= 0x80) - { - len = validate_one_utf8(s + i, n - i); - if (NLOHMANN_VIEW_UNLIKELY(len == 0)) - { - invalid_utf8(s, n); - return; - } - codepoint &= 0xFFu >> (len + 1); - for (std::size_t k = 1; k < len; ++k) - { - codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); - } - } - write_codepoint(codepoint, s + i, len); - i += len; - } - } - /// throw what basic_json::dump() throws for a string that is not valid UTF-8 NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 21a5bb89c..e54d2981e 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -6089,6 +6089,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -6255,6 +6257,70 @@ inline bool is_ascii_without_delete(const char* s, std::size_t n) noexcept return true; } +#if NLOHMANN_VIEW_VECTOR +/// the bytes of the 16 at p that ensure_ascii cannot copy (a quote, a +/// backslash, a control character, DEL, or not ASCII; a signed compare with +/// 0x20 finds control characters and non-ASCII bytes at once), one bit per +/// byte (x86) or one nibble per byte (NEON) +NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t ascii_stop_mask(const unsigned char* p) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vorrq_u8(vceqq_u8(in, vdupq_n_u8(0x7F)), vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20)))); + return vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8(0x7F)), _mm_cmplt_epi8(in, _mm_set1_epi8(0x20)))); + return static_cast(static_cast(_mm_movemask_epi8(special))); +#endif +} + +/// the width in bits of a byte in the mask of ascii_stop_mask() +constexpr unsigned ascii_stop_bits() noexcept +{ + return NLOHMANN_VIEW_NEON ? 4u : 1u; +} +#endif + +/// As find_ascii_copyable_run(), 16 bytes per step. end is the end of the +/// buffer that holds the n bytes at p (a short run is read as 16 bytes if +/// they are in the buffer). +inline std::size_t ascii_copyable_run(const unsigned char* p, std::size_t n, const unsigned char* end) noexcept +{ +#if NLOHMANN_VIEW_VECTOR + if (n >= 16) + { + std::size_t i = 0; + for (; i + 16 <= n; i += 16) + { + const std::uint64_t mask = ascii_stop_mask(p + i); + if (mask != 0) + { + return i + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()); + } + } + if (i == n) + { + return n; + } + // the last 16 bytes: those before i are known to be copyable + const std::uint64_t mask = ascii_stop_mask(p + n - 16); + return mask != 0 ? n - 16 + (static_cast(count_trailing_zeros(mask)) / ascii_stop_bits()) : n; + } + if (end - p >= 16) + { + // (the bytes after the run are masked out) + const std::uint64_t mask = ascii_stop_mask(p) & ((std::uint64_t{1} << (n * ascii_stop_bits())) - 1u); + return mask != 0 ? static_cast(count_trailing_zeros(mask)) / ascii_stop_bits() : n; + } +#else + static_cast(end); +#endif + return find_ascii_copyable_run(p, n); +} + /// A stack that starts in a buffer of the caller (a local array) and moves to /// the heap (a vector of the caller) only when that is full, so that dumps of /// shallow documents need no allocation. The top is a pointer, as in @@ -6506,10 +6572,151 @@ class view_serializer return (n.flags & node_flags::storage) == 0 && is_ascii_without_delete(src + n.off, n.len); } - /// a string with ensure_ascii (dump_text()), out of line - NLOHMANN_VIEW_NOINLINE void write_ascii_string(const node& n) + /*! + @brief a string with ensure_ascii: the characters that are not printable ASCII as \\uXXXX + (a character beyond U+FFFF as a surrogate pair); strings of a document + are valid UTF-8, except in a damaged image loaded with image_check::bounds, + for which this throws what basic_json::dump() throws for the string + */ + NLOHMANN_VIEW_NOINLINE void write_ascii(const node& n) { - write_string(n); + const auto* const s = reinterpret_cast(m_doc.str(n)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const std::size_t size = n.len; + char* w = m_out.cursor(); + char* lim = m_out.limit(); + // room for k bytes + const auto room = [&](std::size_t k) + { + if (NLOHMANN_VIEW_UNLIKELY(static_cast(lim - w) < k)) + { + m_out.set_cursor(w); + m_out.reserve(k); + w = m_out.cursor(); + lim = m_out.limit(); + } + }; + static constexpr const char* hex = "0123456789abcdef"; + // \\uXXXX + const auto u_escape = [&](std::uint32_t u) + { + w[0] = '\\'; + w[1] = 'u'; + w[2] = hex[(u >> 12u) & 0xFu]; + w[3] = hex[(u >> 8u) & 0xFu]; + w[4] = hex[(u >> 4u) & 0xFu]; + w[5] = hex[u & 0xFu]; + w += 6; + }; + room(1); + *w++ = '"'; + std::size_t i = 0; + // (a string of the source starts with a run that has nothing to escape, 16 bytes per step) + const std::size_t prefix = (n.flags & node_flags::storage) == 0 ? ascii_copyable_run(s, size, reinterpret_cast(m_doc.src + m_doc.size)) : 0; // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + if (prefix != 0) + { + room(prefix); + std::memcpy(w, s, prefix); + w += prefix; + i = prefix; + } + while (i < size) + { + // (a character that needs an escape sequence is not worth a call) + const std::size_t run = is_ascii_copyable(s[i]) ? find_ascii_copyable_run(s + i, size - i) : 0; + if (run != 0) + { + room(run); + std::memcpy(w, s + i, run); + w += run; + i += run; + continue; + } + room(12); + if (s[i] >= 0x80) + { + const unsigned char lead = s[i]; + // (the two-byte sequences of Latin text and the three-byte sequences of most other + // text, with continuation bytes of any value: the ranges that are restricted (E0, + // ED, and the sequences of four bytes) go through validate_one_utf8()) + if (lead >= 0xE1 && lead <= 0xEF && lead != 0xED && size - i >= 3 && (s[i + 1] & 0xC0u) == 0x80 && (s[i + 2] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x0Fu) << 12u) | ((s[i + 1] & 0x3Fu) << 6u) | (s[i + 2] & 0x3Fu)); + i += 3; + continue; + } + if (lead >= 0xC2 && lead <= 0xDF && size - i >= 2 && (s[i + 1] & 0xC0u) == 0x80) + { + u_escape(((lead & 0x1Fu) << 6u) | (s[i + 1] & 0x3Fu)); + i += 2; + continue; + } + const std::size_t len = validate_one_utf8(s + i, size - i); + if (NLOHMANN_VIEW_UNLIKELY(len == 0)) + { + m_out.set_cursor(w); + invalid_utf8(s, size); + return; // LCOV_EXCL_LINE (invalid_utf8 throws) + } + std::uint32_t codepoint = s[i] & (0xFFu >> (len + 1)); + for (std::size_t k = 1; k < len; ++k) + { + codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); + } + i += len; + if (codepoint <= 0xFFFF) + { + u_escape(codepoint); + } + else + { + u_escape(0xD7C0u + (codepoint >> 10u)); + u_escape(0xDC00u + (codepoint & 0x3FFu)); + } + continue; + } + // a quote, a backslash, or a control character (or DEL) + const unsigned char c = s[i++]; + char escape = 0; + switch (c) + { + case 0x08: + escape = 'b'; + break; + case 0x09: + escape = 't'; + break; + case 0x0A: + escape = 'n'; + break; + case 0x0C: + escape = 'f'; + break; + case 0x0D: + escape = 'r'; + break; + case 0x22: + escape = '"'; + break; + case 0x5C: + escape = '\\'; + break; + default: + break; + } + if (escape != 0) + { + w[0] = '\\'; + w[1] = escape; + w += 2; + } + else + { + u_escape(c); + } + } + room(1); + *w++ = '"'; + m_out.set_cursor(w); } /// the copies of dump_text() that are not fixed-size moves (long @@ -6537,8 +6744,8 @@ class view_serializer the source copied by fixed-size moves of 32 bytes where the source has that many bytes left, instead of a library call per token. The buffer keeps 64 bytes of slack for the overshoot. With ensure_ascii (Ascii), a - string of the source is copied only if it has no byte >= 0x7F (see - is_ascii_without_delete()); any other string is written by write_string(). + string of the source is copied only if it has nothing to escape (see + copied()); any other string is written by write_ascii(). */ template void dump_text(const node* root) @@ -6620,7 +6827,7 @@ class view_serializer m_out.set_cursor(w); if (enabled(Ascii)) { - write_ascii_string(n); + write_ascii(n); } else { @@ -6935,72 +7142,6 @@ class view_serializer m_out.put(s.data(), s.size()); } - void write_string(const node& n) - { - const char* const s = m_doc.str(n); - m_out.put('"'); - if ((n.flags & node_flags::storage) == 0 && (!m_style.ensure_ascii || is_ascii_without_delete(s, n.len))) - { - // a string of the source without escape sequences has nothing to - // escape (with ensure_ascii, if it is printable ASCII) - m_out.put(s, n.len); - } - else if (m_style.ensure_ascii) - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - else - { - write_escaped(reinterpret_cast(s), n.len); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - m_out.put('"'); - } - - /// as serializer::dump_escaped(); strings of a document are valid UTF-8, - /// except in a damaged image loaded with image_check::bounds, for which - /// this throws what basic_json::dump() throws for the string - template - void write_escaped(const unsigned char* s, std::size_t n) - { - std::size_t i = 0; - while (i < n) - { - std::size_t run = 0; - if (!EnsureAscii) - { - run = string_bulk_run(s + i, n - i); - } - else if (is_ascii_copyable(s[i])) - { - run = find_ascii_copyable_run(s + i, n - i); - } - if (run != 0) - { - m_out.put(reinterpret_cast(s + i), run); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - i += run; - continue; - } - std::uint32_t codepoint = s[i]; - std::size_t len = 1; - if (codepoint >= 0x80) - { - len = validate_one_utf8(s + i, n - i); - if (NLOHMANN_VIEW_UNLIKELY(len == 0)) - { - invalid_utf8(s, n); - return; - } - codepoint &= 0xFFu >> (len + 1); - for (std::size_t k = 1; k < len; ++k) - { - codepoint = (codepoint << 6u) | (s[i + k] & 0x3Fu); - } - } - write_codepoint(codepoint, s + i, len); - i += len; - } - } - /// throw what basic_json::dump() throws for a string that is not valid UTF-8 NLOHMANN_VIEW_NOINLINE static void invalid_utf8(const unsigned char* s, std::size_t n) { diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index cd39139ea..664e4608c 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -186,6 +186,100 @@ TEST_CASE("json_view dump") CHECK(deep_document.root().dump() == deep); } + SECTION("ensure_ascii: a character that needs an escape at each position of a string") + { + // string lengths around the width of the vector steps, in a string value, a key, and a value near the end of the text + for (const char* const special : + { + "\xC3\xA9", "\xE4\xB8\xAD", "\xF0\x9F\x98\x80", "\x7F", "\\n", "\\u0001", "\\\"", "\\\\" + }) + { + for (std::size_t length = 0; length <= 40; ++length) + { + for (std::size_t position = 0; position <= length; ++position) + { + std::string content(length, 'a'); + content.insert(position, special); + const std::string texts[] = + { + "\"" + content + "\"", + "[\"" + content + "\"]", + "{\"" + content + "\":1}", + }; + for (const std::string& text : texts) + { + CAPTURE(text) + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + } + } + } + } + } + + SECTION("ensure_ascii: every character of the Basic Multilingual Plane and some beyond") + { + // (the two- and three-byte sequences, including the ranges that need a check: E0, ED, EF BF BF) + std::string text = "["; + std::size_t in_chunk = 0; + const auto append = [&](std::uint32_t cp) + { + if (cp < 0x80) + { + if (cp < 0x20 || cp == '"' || cp == '\\') + { + return; + } + text += static_cast(cp); + } + else if (cp < 0x800) + { + text += static_cast(0xC0 | (cp >> 6)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + text += static_cast(0xE0 | (cp >> 12)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + else + { + text += static_cast(0xF0 | (cp >> 18)); + text += static_cast(0x80 | ((cp >> 12) & 0x3F)); + text += static_cast(0x80 | ((cp >> 6) & 0x3F)); + text += static_cast(0x80 | (cp & 0x3F)); + } + }; + for (std::uint32_t cp = 0x20; cp < 0x10400; ++cp) + { + if (cp >= 0xD800 && cp < 0xE000) + { + continue; + } + if (in_chunk == 0) + { + text += text.size() > 1 ? ",\"" : "\""; + } + append(cp); + if (++in_chunk == 37) + { + text += "\""; + in_chunk = 0; + } + } + text += in_chunk != 0 ? "\"]" : "]"; + const ordered_json_document d = ordered_json_document::parse(text); + const ordered_json j = ordered_json::parse(text); + CHECK(d.root().dump(-1, ' ', true) == j.dump(-1, ' ', true)); + CHECK(d.root().dump(2, ' ', true) == j.dump(2, ' ', true)); + CHECK(d.root().dump() == j.dump()); + CHECK(d.root().dump(2) == j.dump(2)); + } + SECTION("deep nesting, indented") { // indentations of up to and beyond the width of the writer's fixed-size copy