From 5cfff60577e5c99a82d3c32c80ab1d7bfb52c7e7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 16 Oct 2026 22:45:12 +0200 Subject: [PATCH] Copy the text between escapes while scanning it A string that contains an escape is decoded by builder::slow_string(). After each escape it scanned the next run of plain bytes (8 or 16 byte-by-byte steps, then a vector loop), and then copied that run to the decode arena in a second pass. The run is now copied 16 bytes at a time while it is scanned (one vector load, a store of the same 16 bytes, and the mask of the bytes that end the run), and the write position lives in a local variable, not in the builder, since a store of a character may alias the builder's members and made the compiler reload it after every escape. A run that ends at a quote needs no scan, and non-ASCII text goes through the UTF-8 check as before; the portable code is unchanged. Parsing, Apple M1 instructions: jeopardy.json 304.0 M -> 293.8 M (-3.4 %), 3000 strings with escapes 1.72 M -> 1.54 M (-10 %), twitter -0.6 %. Linux x86-64 Ir, GCC 13 / Clang 18: escaped strings -7.6 % / -11.4 %, twitter 0.0 % / 0.0 %. Documents and error messages are identical on 900 k random strings of escapes, UTF-8 and control characters (also under ASan), and on mutants of the test corpus. Belongs to the PR "json_view: read-side speedups". Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/builder.hpp | 161 ++++++++++++++----- include/nlohmann/detail/view/simd.hpp | 28 ++++ single_include/nlohmann/json_view.hpp | 189 ++++++++++++++++++----- tests/src/unit-json_view_builder.cpp | 52 +++++++ 4 files changed, 354 insertions(+), 76 deletions(-) diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 400684a43..27ce3de37 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -275,11 +275,41 @@ class builder 0, 0, '\b', 0, 0, 0, '\f', 0, 0, 0, 0, 0, 0, 0, '\n', 0, 0, 0, '\r', 0, '\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + // The write position is kept in locals: the stores of characters into + // the arena may alias the members of the builder, which would then be + // reloaded after every one. It is stored back where the arena grows + // and at the end. + const unsigned char* const end = m_end; const std::size_t start = arena_used(); - arena_run(s, static_cast(p - s)); + char* w = m_arena_write; + const char* w_end = m_arena_end; +#define NLOHMANN_VIEW_ENSURE(n) \ + if (NLOHMANN_VIEW_UNLIKELY(static_cast(w_end - w) < (n))) \ + { \ + m_arena_write = w; \ + arena_grow(n); \ + w = m_arena_write; \ + w_end = m_arena_end; \ + } + { + const auto n = static_cast(p - s); + if (n <= 16 && end - s >= 16) + { + NLOHMANN_VIEW_ENSURE(16) + std::memcpy(w, s, 16); + } + else if (n != 0) + { + // (an empty run leaves an empty arena unallocated, and memcpy() + // must not be passed its null pointer even for zero bytes) + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, s, n); + } + w += n; + } for (;;) { - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -287,6 +317,7 @@ class builder if (c == '"') { ++p; + m_arena_write = w; return decoded{p, start, arena_used() - start}; } if (c != '\\') @@ -297,7 +328,7 @@ class builder return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); } ++p; - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -315,7 +346,7 @@ class builder { return failed(error_code::string_surrogate_low, p); } - if (m_end - p < 2 || p[0] != '\\' || p[1] != 'u') + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') { return failed(error_code::string_surrogate_high, p); } @@ -331,27 +362,108 @@ class builder } cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); } - arena_ensure(4); - m_arena_write = put_utf8(m_arena_write, cp); + NLOHMANN_VIEW_ENSURE(4) + w = put_utf8(w, cp); } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { - arena_ensure(1); - *m_arena_write++ = simple_escape[d]; + NLOHMANN_VIEW_ENSURE(1) + *w++ = simple_escape[d]; } else { --p; return failed(error_code::string_escape, p); } - if (p != m_end && *p == '\\') +#if NLOHMANN_VIEW_NEON + if (p != end && *p == '\\') { continue; // consecutive escapes ("\u00e4\u00f6"): no run in between } + // The first 16 bytes of the run are checked one by one, in predicted branches, and copied + // as one block: where the runs between escapes are short, the vector step below would put + // its longer dependency chain (load, compare, mask, count, add) on p for each of them, + // and be slower than these steps (the scanner of the keys works the same way) + { + if (end - p >= 16) + { + const std::uint8_t* const plain = string_plain(); + std::size_t steps = 16; +#define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { steps = (i); goto run_found; } + NLOHMANN_VIEW_REPEAT16(NLOHMANN_VIEW_STEP) +#undef NLOHMANN_VIEW_STEP +run_found: + NLOHMANN_VIEW_ENSURE(16) + std::memcpy(w, p, 16); // (only the first `steps` bytes are kept) + w += steps; + p += steps; + if (steps < 16) + { + if (*p >= 0x80) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + } + continue; + } + } + } +#else + if (p != end && (*p == '\\' || *p == '"')) + { + continue; // consecutive escapes ("\u00e4\u00f6"), or the end of the string: no run in between + } +#endif +#if NLOHMANN_VIEW_VECTOR + // the plain bytes up to the next quote, backslash, or control + // character are copied as they are scanned, 16 bytes at a time + for (;;) + { + if (NLOHMANN_VIEW_UNLIKELY(end - p < 16)) + { + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + break; + } + NLOHMANN_VIEW_ENSURE(16) + const unsigned k = copy_plain_run16(p, w); + w += k; + p += k; + if (k == 16) + { + continue; + } + if (NLOHMANN_VIEW_UNLIKELY(*p >= 0x80)) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + } + break; + } +#else const unsigned char* const r = p; - p = scan_string_run(p, m_end); - arena_run(r, static_cast(p - r)); + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; +#endif } +#undef NLOHMANN_VIEW_ENSURE } /// does the float token [s, p) overflow FloatType? (parse() rejects it) @@ -397,14 +509,6 @@ class builder return m_arena_write != nullptr ? static_cast(m_arena_write - m_doc.arena.data()) : 0; } - NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(std::size_t n) - { - if (NLOHMANN_VIEW_UNLIKELY(static_cast(m_arena_end - m_arena_write) < n)) - { - arena_grow(n); - } - } - NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); @@ -413,25 +517,6 @@ class builder m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } - /// append the run [r, r + n) to the arena; short runs as one fixed-size - /// 16-byte move when both sides have the room (no library call) - NLOHMANN_VIEW_ALWAYS_INLINE void arena_run(const unsigned char* r, std::size_t n) - { - if (n <= 16 && m_end - r >= 16) - { - arena_ensure(16); - std::memcpy(m_arena_write, r, 16); - } - else if (n != 0) - { - // (an empty run leaves an empty arena unallocated, and memcpy() - // must not be passed its null pointer even for zero bytes) - arena_ensure(n); - std::memcpy(m_arena_write, r, n); - } - m_arena_write += n; - } - /// UTF-8 encoding of cp at w (room for 4 bytes); returns the position after /// it. (A returned pointer stays in a register; writing through a /// reference to m_arena_write would store and reload it around every byte.) diff --git a/include/nlohmann/detail/view/simd.hpp b/include/nlohmann/detail/view/simd.hpp index b83cd5a3b..7a0248f63 100644 --- a/include/nlohmann/detail/view/simd.hpp +++ b/include/nlohmann/detail/view/simd.hpp @@ -116,6 +116,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } return p; } + +/*! +@brief copy 16 bytes from p to w, and find the first quote, backslash, +control character, or non-ASCII byte among them + +The bytes are copied whether or not they are all plain (the caller has room +for 16 bytes and keeps only the plain ones). + +@return the offset of that byte, or 16 if there is none +*/ +NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, char* w) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; +#endif +} #endif #if NLOHMANN_VIEW_SSSE3_DISPATCH diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 527062a72..7f7ec815f 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -756,6 +756,34 @@ NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* vector_plain_run(const unsigned } return p; } + +/*! +@brief copy 16 bytes from p to w, and find the first quote, backslash, +control character, or non-ASCII byte among them + +The bytes are copied whether or not they are all plain (the caller has room +for 16 bytes and keeps only the plain ones). + +@return the offset of that byte, or 16 if there is none +*/ +NLOHMANN_VIEW_ALWAYS_INLINE unsigned copy_plain_run16(const unsigned char* p, char* w) noexcept +{ +#if NLOHMANN_VIEW_NEON + const uint8x16_t in = vld1q_u8(p); + vst1q_u8(reinterpret_cast(w), in); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + const uint8x16_t special = vorrq_u8(vorrq_u8(vceqq_u8(in, vdupq_n_u8('"')), vceqq_u8(in, vdupq_n_u8('\\'))), + vcltq_s8(vreinterpretq_s8_u8(in), vdupq_n_s8(0x20))); + const std::uint64_t bits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(special), 4)), 0); + return bits != 0 ? static_cast(count_trailing_zeros(bits) >> 2u) : 16u; +#else + const __m128i in = _mm_loadu_si128(static_cast(static_cast(p))); // NOLINT(bugprone-casting-through-void) + _mm_storeu_si128(static_cast<__m128i*>(static_cast(w)), in); // NOLINT(bugprone-casting-through-void) + const __m128i special = _mm_or_si128(_mm_or_si128(_mm_cmpeq_epi8(in, _mm_set1_epi8('"')), _mm_cmpeq_epi8(in, _mm_set1_epi8('\\'))), + _mm_cmplt_epi8(in, _mm_set1_epi8(0x20))); + const auto bits = static_cast(static_cast(_mm_movemask_epi8(special))); + return bits != 0 ? static_cast(count_trailing_zeros(bits)) : 16u; +#endif +} #endif #if NLOHMANN_VIEW_SSSE3_DISPATCH @@ -1437,11 +1465,41 @@ class builder 0, 0, '\b', 0, 0, 0, '\f', 0, 0, 0, 0, 0, 0, 0, '\n', 0, 0, 0, '\r', 0, '\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; + // The write position is kept in locals: the stores of characters into + // the arena may alias the members of the builder, which would then be + // reloaded after every one. It is stored back where the arena grows + // and at the end. + const unsigned char* const end = m_end; const std::size_t start = arena_used(); - arena_run(s, static_cast(p - s)); + char* w = m_arena_write; + const char* w_end = m_arena_end; +#define NLOHMANN_VIEW_ENSURE(n) \ + if (NLOHMANN_VIEW_UNLIKELY(static_cast(w_end - w) < (n))) \ + { \ + m_arena_write = w; \ + arena_grow(n); \ + w = m_arena_write; \ + w_end = m_arena_end; \ + } + { + const auto n = static_cast(p - s); + if (n <= 16 && end - s >= 16) + { + NLOHMANN_VIEW_ENSURE(16) + std::memcpy(w, s, 16); + } + else if (n != 0) + { + // (an empty run leaves an empty arena unallocated, and memcpy() + // must not be passed its null pointer even for zero bytes) + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, s, n); + } + w += n; + } for (;;) { - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -1449,6 +1507,7 @@ class builder if (c == '"') { ++p; + m_arena_write = w; return decoded{p, start, arena_used() - start}; } if (c != '\\') @@ -1459,7 +1518,7 @@ class builder return failed(c < 0x20 ? error_code::string_control_character : error_code::string_utf8, p); } ++p; - if (p == m_end) + if (p == end) { return failed(error_code::string_missing_quote, p); } @@ -1477,7 +1536,7 @@ class builder { return failed(error_code::string_surrogate_low, p); } - if (m_end - p < 2 || p[0] != '\\' || p[1] != 'u') + if (end - p < 2 || p[0] != '\\' || p[1] != 'u') { return failed(error_code::string_surrogate_high, p); } @@ -1493,27 +1552,108 @@ class builder } cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); } - arena_ensure(4); - m_arena_write = put_utf8(m_arena_write, cp); + NLOHMANN_VIEW_ENSURE(4) + w = put_utf8(w, cp); } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { - arena_ensure(1); - *m_arena_write++ = simple_escape[d]; + NLOHMANN_VIEW_ENSURE(1) + *w++ = simple_escape[d]; } else { --p; return failed(error_code::string_escape, p); } - if (p != m_end && *p == '\\') +#if NLOHMANN_VIEW_NEON + if (p != end && *p == '\\') { continue; // consecutive escapes ("\u00e4\u00f6"): no run in between } + // The first 16 bytes of the run are checked one by one, in predicted branches, and copied + // as one block: where the runs between escapes are short, the vector step below would put + // its longer dependency chain (load, compare, mask, count, add) on p for each of them, + // and be slower than these steps (the scanner of the keys works the same way) + { + if (end - p >= 16) + { + const std::uint8_t* const plain = string_plain(); + std::size_t steps = 16; +#define NLOHMANN_VIEW_STEP(i) if (NLOHMANN_VIEW_LIKELY(plain[p[i]] != 0)) {} else { steps = (i); goto run_found; } + NLOHMANN_VIEW_REPEAT16(NLOHMANN_VIEW_STEP) +#undef NLOHMANN_VIEW_STEP +run_found: + NLOHMANN_VIEW_ENSURE(16) + std::memcpy(w, p, 16); // (only the first `steps` bytes are kept) + w += steps; + p += steps; + if (steps < 16) + { + if (*p >= 0x80) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + } + continue; + } + } + } +#else + if (p != end && (*p == '\\' || *p == '"')) + { + continue; // consecutive escapes ("\u00e4\u00f6"), or the end of the string: no run in between + } +#endif +#if NLOHMANN_VIEW_VECTOR + // the plain bytes up to the next quote, backslash, or control + // character are copied as they are scanned, 16 bytes at a time + for (;;) + { + if (NLOHMANN_VIEW_UNLIKELY(end - p < 16)) + { + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + break; + } + NLOHMANN_VIEW_ENSURE(16) + const unsigned k = copy_plain_run16(p, w); + w += k; + p += k; + if (k == 16) + { + continue; + } + if (NLOHMANN_VIEW_UNLIKELY(*p >= 0x80)) + { + // non-ASCII: the UTF-8 check, and the rest of the run + const unsigned char* const r = p; + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; + } + break; + } +#else const unsigned char* const r = p; - p = scan_string_run(p, m_end); - arena_run(r, static_cast(p - r)); + p = scan_string_run(p, end); + const auto n = static_cast(p - r); + NLOHMANN_VIEW_ENSURE(n) + std::memcpy(w, r, n); + w += n; +#endif } +#undef NLOHMANN_VIEW_ENSURE } /// does the float token [s, p) overflow FloatType? (parse() rejects it) @@ -1559,14 +1699,6 @@ class builder return m_arena_write != nullptr ? static_cast(m_arena_write - m_doc.arena.data()) : 0; } - NLOHMANN_VIEW_ALWAYS_INLINE void arena_ensure(std::size_t n) - { - if (NLOHMANN_VIEW_UNLIKELY(static_cast(m_arena_end - m_arena_write) < n)) - { - arena_grow(n); - } - } - NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n) { const std::size_t used = arena_used(); @@ -1575,25 +1707,6 @@ class builder m_arena_end = m_doc.arena.data() + m_doc.arena.size(); } - /// append the run [r, r + n) to the arena; short runs as one fixed-size - /// 16-byte move when both sides have the room (no library call) - NLOHMANN_VIEW_ALWAYS_INLINE void arena_run(const unsigned char* r, std::size_t n) - { - if (n <= 16 && m_end - r >= 16) - { - arena_ensure(16); - std::memcpy(m_arena_write, r, 16); - } - else if (n != 0) - { - // (an empty run leaves an empty arena unallocated, and memcpy() - // must not be passed its null pointer even for zero bytes) - arena_ensure(n); - std::memcpy(m_arena_write, r, n); - } - m_arena_write += n; - } - /// UTF-8 encoding of cp at w (room for 4 bytes); returns the position after /// it. (A returned pointer stays in a register; writing through a /// reference to m_arena_write would store and reload it around every byte.) diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index b43af739d..6935981fb 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -523,6 +523,58 @@ TEST_CASE("json_view builder: strings across vector blocks") } } +TEST_CASE("json_view builder: escapes between runs of text") +{ + // The text between escapes is copied 16 bytes at a time where there are + // that many bytes left, and with the UTF-8 check where there is + // non-ASCII text. Runs of every length from 0 to 40 between valid and + // invalid escapes, UTF-8 sequences, and control characters, in strings + // that are cut at all places near the end of the text. + const std::array valid = {{"\\n", "\\\"", "\\\\", "\\/", "\\t", "\\u00e4", "\\u20ac", "\\ud83d\\ude00", "\\u0000"}}; + const std::array invalid = {{"\\x", "\\ud800", "\\udc00", "\\u12G4", "\\u12", "\\ud83dx", "\\ud83d\\u0041", "\\", "\x01"}}; + const std::array utf8 = {{"\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xc3", "\xff", "\x80"}}; + std::mt19937 rng(2026); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t run = 0; run <= 40; ++run) + { + for (int i = 0; i < 60; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + const auto pieces = 1 + (rng() % 6); + for (unsigned k = 0; k < pieces; ++k) + { + content += valid[rng() % valid.size()]; + content += std::string(k == 0 ? run : rng() % 41, static_cast('b' + (rng() % 20))); + switch (rng() % 8) + { + case 0: + content += utf8[rng() % 3]; + break; + case 1: + content += utf8[rng() % utf8.size()]; + break; + case 2: + content += invalid[rng() % invalid.size()]; + break; + default: + break; + } + } + check_string(content); + check_string(content.substr(0, content.size() - (rng() % 4))); + } + } + // the text before the first escape, and runs and escapes at the end of the text + for (std::size_t before = 0; before <= 40; ++before) + { + for (const char* escape : valid) + { + check_string(std::string(before, 'a') + escape); + check_string(std::string(before, 'a') + escape + std::string(before % 19, 'z')); + check_string(std::string(before, 'a') + escape + escape + std::string(before % 19, 'z')); + } + } +} + TEST_CASE("json_view node integer bits") { using nlohmann::detail::view::integer_bits;