From 5bbc7a1098dacc485f9241157b766fb86cbfe9b6 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 16 Oct 2026 21:23:15 +0200 Subject: [PATCH] Write short plain strings and keys in one go in dump() Most strings in real documents are object keys and short values. For those, dump_escaped() scanned the bytes, called memcpy with a length known only at run time, and the surrounding quote, colon and space were three more bounds-checked writes. A string of up to 16 printable ASCII characters (no quote, backslash or DEL) is now read with two overlapping loads, checked with one SWAR test per word, and written together with its quotes and the colon with two overlapping stores. Anything else, and a write buffer without room, takes the general path unchanged. Little-endian targets only; others keep the general path. dump_scalar() held the code of every scalar type and saved and restored all callee-saved registers on every call. It only dispatches now, and the string, integer, float and binary writers are functions of their own, so each pays only for what it uses (the string path keeps the check for long strings in front of any attempt, which keeps documents of long strings neutral). Apple M1, Apple clang, retired instructions per dump(), on top of the printable-ASCII table of #5878: citm_catalog 7.54M -> 5.96M (-21 %) twitter 4.87M -> 4.40M (-10 %) jeopardy 603M -> 501M (-17 %) logs 8.65M -> 8.17M (-5 %) a 139-byte object 3497 -> 2997 (-14 %) signed_ints -4 %, canada -3 % Linux x86-64 (callgrind), the whole series against develop, gcc 13 / clang 18: citm_catalog -16 % / -19 % (compact), twitter -22 % / -24 %. Documents of long strings (paths, latin and CJK text) are within 1 %. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/macro_unscope.hpp | 1 + include/nlohmann/detail/output/serializer.hpp | 346 ++++++++++++----- single_include/nlohmann/json.hpp | 347 +++++++++++++----- tests/src/unit-serialization.cpp | 51 +++ 4 files changed, 569 insertions(+), 176 deletions(-) diff --git a/include/nlohmann/detail/macro_unscope.hpp b/include/nlohmann/detail/macro_unscope.hpp index d3cdb0d3f..ddfe086c8 100644 --- a/include/nlohmann/detail/macro_unscope.hpp +++ b/include/nlohmann/detail/macro_unscope.hpp @@ -33,6 +33,7 @@ #undef JSON_SCAN_SSSE3_TARGET #undef JSON_SCAN_VECTOR #undef JSON_SCAN_VECTOR_UTF8 +#undef JSON_DUMP_SHORT_STRINGS #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 7bb4edfc5..b97a6efb5 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -34,6 +34,14 @@ #include #include +// the short-string fast path of the serializer assembles words from little-endian +// loads, see serializer::put_short_quoted() +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define JSON_DUMP_SHORT_STRINGS 1 +#else + #define JSON_DUMP_SHORT_STRINGS 0 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -169,9 +177,7 @@ class serializer for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); + put_quoted(i->first, "\": "); dump_internal(i->second, new_indent, depth + 1); put_literal(",\n"); } @@ -180,9 +186,7 @@ class serializer JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); + put_quoted(i->first, "\": "); dump_internal(i->second, new_indent, depth + 1); put_char('\n'); @@ -197,9 +201,7 @@ class serializer auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); + put_quoted(i->first, "\":"); dump_internal(i->second, current_indent, depth + 1); put_char(','); } @@ -207,9 +209,7 @@ class serializer // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); + put_quoted(i->first, "\":"); dump_internal(i->second, current_indent, depth + 1); put_char('}'); @@ -353,16 +353,13 @@ class serializer put_indent(frame.child_indent); } - put_char('"'); - dump_escaped(frame.object_it->first); - if (pretty_print) { - put_literal("\": "); + put_quoted(frame.object_it->first, "\": "); } else { - put_literal("\":"); + put_quoted(frame.object_it->first, "\":"); } const BasicJsonType& element = frame.object_it->second; @@ -537,78 +534,13 @@ class serializer { case value_t::string: { - put_char('"'); - dump_escaped(*val.m_data.m_value.string); - put_char('"'); + dump_string_value(*val.m_data.m_value.string); return; } case value_t::binary: { - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for the bytes - const auto new_indent = next_indent(current_indent, indent_step); - - put_indent(new_indent); - - put_literal("\"bytes\": ["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_literal(", "); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\n"); - put_indent(new_indent); - - put_literal("\"subtype\": "); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - } - else - { - put_literal("null"); - } - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_literal("{\"bytes\":["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_char(','); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\"subtype\":"); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - put_char('}'); - } - else - { - put_literal("null}"); - } - } + dump_binary_value(current_indent, *val.m_data.m_value.binary); return; } @@ -627,19 +559,19 @@ class serializer case value_t::number_integer: { - dump_integer(val.m_data.m_value.number_integer); + dump_signed_value(val.m_data.m_value.number_integer); return; } case value_t::number_unsigned: { - dump_integer(val.m_data.m_value.number_unsigned); + dump_unsigned_value(val.m_data.m_value.number_unsigned); return; } case value_t::number_float: { - dump_float(val.m_data.m_value.number_float); + dump_float_value(val.m_data.m_value.number_float); return; } @@ -662,6 +594,107 @@ class serializer } } + /// @brief write the string @a s in quotes; kept out of line, so that the + /// other scalars do not pay for its register needs (and the other way round) + JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) + { +#if JSON_DUMP_SHORT_STRINGS + if (s.size() <= 16 && put_short_quoted(s, "\"")) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_char('"'); + } + + // The numbers, and the rest of what dump_scalar() writes, are left to + // functions of their own as well: a single function holding all of them + // saves and restores every register any of them needs, on every call. Now + // dump_scalar() only dispatches, and each writer pays for what it uses. + JSON_HEDLEY_NEVER_INLINE void dump_signed_value(const number_integer_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_unsigned_value(const number_unsigned_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_float_value(const number_float_t x) + { + dump_float(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_binary_value(const std::size_t current_indent, const typename BasicJsonType::binary_t& binary) + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); + i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(binary.back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); + i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(binary.back()); + } + + put_literal("],\"subtype\":"); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + } /*! @brief the indentation level to use for the children of the current value @@ -678,6 +711,133 @@ class serializer return new_indent; } + private: + /*! + @brief append the string @a s in quotes, followed by @a tail + + The quote, the string and @a tail (the closing quote, possibly with the + colon of an object key) are the bulk of what most documents consist of, and + most keys and many values are short and plain. Those are handled in one go + by put_short_quoted(); everything else takes the general path. + */ + template + void put_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { +#if JSON_DUMP_SHORT_STRINGS + if (put_short_quoted(s, tail)) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_literal(tail); + } + +#if JSON_DUMP_SHORT_STRINGS + /// the high bit of every byte of @a v that is not printable ASCII or is a + /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) + static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' + | ((v - 0x2020202020202020ull) & ~v) // < 0x20 + | v | (v + ones)) // >= 0x7F + & 0x8080808080808080ull; + } + + template + static WordType load_word(const char* p) noexcept + { + WordType w; + std::memcpy(&w, p, sizeof(w)); + return w; + } + + template + static void store_word(char* p, const WordType w) noexcept + { + std::memcpy(p, &w, sizeof(w)); + } + + /*! + @brief write `"`, @a s and @a tail if @a s is short and needs no escaping + + A string of up to 16 printable ASCII characters (no quote, backslash, or DEL) + is read with two overlapping loads and written with two overlapping stores, + instead of scanning it and calling memcpy with a length that is only known + at run time. Returns false, having changed nothing, if @a s does not + qualify or the write buffer lacks the room. + */ + template + bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + const std::size_t n = s.size(); + if (n > 16 || write_buffer_pos + n + N > write_buffer.size()) + { + return false; + } + + char* d = write_buffer.data() + write_buffer_pos + 1; + const char* const p = s.data(); + + if (n >= 8) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); + } + else if (n >= 4) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 4); + // the bytes above the string are filled with a plain character + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 4))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 4, b); + } + else if (n >= 2) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 2); + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 2))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 2, b); + } + else if (n == 1) + { + const std::uint64_t w = static_cast(static_cast(p[0])) | (0x4141414141414141ull << 8u); + if (swar_not_plain(w) != 0) + { + return false; + } + *d = p[0]; + } + + d[-1] = '"'; + std::memcpy(d + n, tail, N - 1); + write_buffer_pos += n + N; + return true; + } +#endif + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -690,14 +850,23 @@ class serializer @param[in] s the string to escape Complexity: Linear in the length of string @a s. + + @note kept out of line: with the short-string attempts in front of it gone + from here, compilers would otherwise inline it into @ref put_quoted + and then no longer into the loops that call that */ - void dump_escaped(const string_t& s) + JSON_HEDLEY_NEVER_INLINE void dump_escaped(const string_t& s) { // Short strings of printable ASCII - most object keys and many values - // are checked byte by byte against a table and then copied in one go. // For them, this is cheaper than starting the vector scanner below, // whose setup costs more than such a string takes to scan. The same // check serves both values of ensure_ascii. + // + // Where put_short_quoted() exists and takes strings of up to 32 bytes, + // the callers have tried it already, and repeating the attempt only + // costs time for strings that need escaping or are not ASCII. +#if !(JSON_DUMP_SHORT_STRINGS && JSON_SCAN_VECTOR) if (s.size() <= 32) { const auto& copyable = ascii_copyable(); @@ -719,6 +888,7 @@ class serializer return; } } +#endif // dispatch once here rather than test the flag inside the loop: it does // not change while a string is written, and folding it lets each of the diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 89e89c12a..1dbf1fd48 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -30178,6 +30178,14 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// the short-string fast path of the serializer assembles words from little-endian +// loads, see serializer::put_short_quoted() +#if defined(_MSC_VER) || defined(__x86_64__) || defined(__i386__) || (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define JSON_DUMP_SHORT_STRINGS 1 +#else + #define JSON_DUMP_SHORT_STRINGS 0 +#endif + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { @@ -30313,9 +30321,7 @@ class serializer for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); + put_quoted(i->first, "\": "); dump_internal(i->second, new_indent, depth + 1); put_literal(",\n"); } @@ -30324,9 +30330,7 @@ class serializer JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); put_indent(new_indent); - put_char('"'); - dump_escaped(i->first); - put_literal("\": "); + put_quoted(i->first, "\": "); dump_internal(i->second, new_indent, depth + 1); put_char('\n'); @@ -30341,9 +30345,7 @@ class serializer auto i = val.m_data.m_value.object->cbegin(); for (std::size_t cnt = 0; cnt < val.m_data.m_value.object->size() - 1; ++cnt, ++i) { - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); + put_quoted(i->first, "\":"); dump_internal(i->second, current_indent, depth + 1); put_char(','); } @@ -30351,9 +30353,7 @@ class serializer // last element JSON_ASSERT(i != val.m_data.m_value.object->cend()); JSON_ASSERT(std::next(i) == val.m_data.m_value.object->cend()); - put_char('"'); - dump_escaped(i->first); - put_literal("\":"); + put_quoted(i->first, "\":"); dump_internal(i->second, current_indent, depth + 1); put_char('}'); @@ -30497,16 +30497,13 @@ class serializer put_indent(frame.child_indent); } - put_char('"'); - dump_escaped(frame.object_it->first); - if (pretty_print) { - put_literal("\": "); + put_quoted(frame.object_it->first, "\": "); } else { - put_literal("\":"); + put_quoted(frame.object_it->first, "\":"); } const BasicJsonType& element = frame.object_it->second; @@ -30681,78 +30678,13 @@ class serializer { case value_t::string: { - put_char('"'); - dump_escaped(*val.m_data.m_value.string); - put_char('"'); + dump_string_value(*val.m_data.m_value.string); return; } case value_t::binary: { - if (pretty_print) - { - put_literal("{\n"); - - // variable to hold indentation for the bytes - const auto new_indent = next_indent(current_indent, indent_step); - - put_indent(new_indent); - - put_literal("\"bytes\": ["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_literal(", "); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\n"); - put_indent(new_indent); - - put_literal("\"subtype\": "); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - } - else - { - put_literal("null"); - } - put_char('\n'); - put_indent(current_indent); - put_char('}'); - } - else - { - put_literal("{\"bytes\":["); - - if (!val.m_data.m_value.binary->empty()) - { - for (auto i = val.m_data.m_value.binary->cbegin(); - i != val.m_data.m_value.binary->cend() - 1; ++i) - { - dump_byte(*i); - put_char(','); - } - dump_byte(val.m_data.m_value.binary->back()); - } - - put_literal("],\"subtype\":"); - if (val.m_data.m_value.binary->has_subtype()) - { - dump_integer(val.m_data.m_value.binary->subtype()); - put_char('}'); - } - else - { - put_literal("null}"); - } - } + dump_binary_value(current_indent, *val.m_data.m_value.binary); return; } @@ -30771,19 +30703,19 @@ class serializer case value_t::number_integer: { - dump_integer(val.m_data.m_value.number_integer); + dump_signed_value(val.m_data.m_value.number_integer); return; } case value_t::number_unsigned: { - dump_integer(val.m_data.m_value.number_unsigned); + dump_unsigned_value(val.m_data.m_value.number_unsigned); return; } case value_t::number_float: { - dump_float(val.m_data.m_value.number_float); + dump_float_value(val.m_data.m_value.number_float); return; } @@ -30806,6 +30738,107 @@ class serializer } } + /// @brief write the string @a s in quotes; kept out of line, so that the + /// other scalars do not pay for its register needs (and the other way round) + JSON_HEDLEY_NEVER_INLINE void dump_string_value(const string_t& s) + { +#if JSON_DUMP_SHORT_STRINGS + if (s.size() <= 16 && put_short_quoted(s, "\"")) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_char('"'); + } + + // The numbers, and the rest of what dump_scalar() writes, are left to + // functions of their own as well: a single function holding all of them + // saves and restores every register any of them needs, on every call. Now + // dump_scalar() only dispatches, and each writer pays for what it uses. + JSON_HEDLEY_NEVER_INLINE void dump_signed_value(const number_integer_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_unsigned_value(const number_unsigned_t x) + { + dump_integer(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_float_value(const number_float_t x) + { + dump_float(x); + } + + JSON_HEDLEY_NEVER_INLINE void dump_binary_value(const std::size_t current_indent, const typename BasicJsonType::binary_t& binary) + { + if (pretty_print) + { + put_literal("{\n"); + + // variable to hold indentation for the bytes + const auto new_indent = next_indent(current_indent, indent_step); + + put_indent(new_indent); + + put_literal("\"bytes\": ["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); + i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_literal(", "); + } + dump_byte(binary.back()); + } + + put_literal("],\n"); + put_indent(new_indent); + + put_literal("\"subtype\": "); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + } + else + { + put_literal("null"); + } + put_char('\n'); + put_indent(current_indent); + put_char('}'); + } + else + { + put_literal("{\"bytes\":["); + + if (!binary.empty()) + { + for (auto i = binary.cbegin(); + i != binary.cend() - 1; ++i) + { + dump_byte(*i); + put_char(','); + } + dump_byte(binary.back()); + } + + put_literal("],\"subtype\":"); + if (binary.has_subtype()) + { + dump_integer(binary.subtype()); + put_char('}'); + } + else + { + put_literal("null}"); + } + } + } /*! @brief the indentation level to use for the children of the current value @@ -30822,6 +30855,133 @@ class serializer return new_indent; } + private: + /*! + @brief append the string @a s in quotes, followed by @a tail + + The quote, the string and @a tail (the closing quote, possibly with the + colon of an object key) are the bulk of what most documents consist of, and + most keys and many values are short and plain. Those are handled in one go + by put_short_quoted(); everything else takes the general path. + */ + template + void put_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { +#if JSON_DUMP_SHORT_STRINGS + if (put_short_quoted(s, tail)) + { + return; + } +#endif + put_char('"'); + dump_escaped(s); + put_literal(tail); + } + +#if JSON_DUMP_SHORT_STRINGS + /// the high bit of every byte of @a v that is not printable ASCII or is a + /// quote, a backslash, or DEL (zero if the bytes can be copied as they are) + static std::uint64_t swar_not_plain(const std::uint64_t v) noexcept + { + constexpr std::uint64_t ones = 0x0101010101010101ull; + const std::uint64_t q = v ^ 0x2222222222222222ull; // '"' + const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' + return (((q - ones) & ~q) | ((b - ones) & ~b) // == '"', == '\\' + | ((v - 0x2020202020202020ull) & ~v) // < 0x20 + | v | (v + ones)) // >= 0x7F + & 0x8080808080808080ull; + } + + template + static WordType load_word(const char* p) noexcept + { + WordType w; + std::memcpy(&w, p, sizeof(w)); + return w; + } + + template + static void store_word(char* p, const WordType w) noexcept + { + std::memcpy(p, &w, sizeof(w)); + } + + /*! + @brief write `"`, @a s and @a tail if @a s is short and needs no escaping + + A string of up to 16 printable ASCII characters (no quote, backslash, or DEL) + is read with two overlapping loads and written with two overlapping stores, + instead of scanning it and calling memcpy with a length that is only known + at run time. Returns false, having changed nothing, if @a s does not + qualify or the write buffer lacks the room. + */ + template + bool put_short_quoted(const string_t& s, const char (&tail)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + { + const std::size_t n = s.size(); + if (n > 16 || write_buffer_pos + n + N > write_buffer.size()) + { + return false; + } + + char* d = write_buffer.data() + write_buffer_pos + 1; + const char* const p = s.data(); + + if (n >= 8) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 8); + if ((swar_not_plain(a) | swar_not_plain(b)) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 8, b); + } + else if (n >= 4) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 4); + // the bytes above the string are filled with a plain character + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 4))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 4, b); + } + else if (n >= 2) + { + const auto a = load_word(p); + const auto b = load_word(p + n - 2); + const std::uint64_t w = a | (static_cast(b) << (8 * (n - 2))) + | (0x4141414141414141ull << (8 * n)); + if (swar_not_plain(w) != 0) + { + return false; + } + store_word(d, a); + store_word(d + n - 2, b); + } + else if (n == 1) + { + const std::uint64_t w = static_cast(static_cast(p[0])) | (0x4141414141414141ull << 8u); + if (swar_not_plain(w) != 0) + { + return false; + } + *d = p[0]; + } + + d[-1] = '"'; + std::memcpy(d + n, tail, N - 1); + write_buffer_pos += n + N; + return true; + } +#endif + JSON_PRIVATE_UNLESS_TESTED: /*! @brief dump escaped string @@ -30834,14 +30994,23 @@ class serializer @param[in] s the string to escape Complexity: Linear in the length of string @a s. + + @note kept out of line: with the short-string attempts in front of it gone + from here, compilers would otherwise inline it into @ref put_quoted + and then no longer into the loops that call that */ - void dump_escaped(const string_t& s) + JSON_HEDLEY_NEVER_INLINE void dump_escaped(const string_t& s) { // Short strings of printable ASCII - most object keys and many values - // are checked byte by byte against a table and then copied in one go. // For them, this is cheaper than starting the vector scanner below, // whose setup costs more than such a string takes to scan. The same // check serves both values of ensure_ascii. + // + // Where put_short_quoted() exists and takes strings of up to 32 bytes, + // the callers have tried it already, and repeating the attempt only + // costs time for strings that need escaping or are not ASCII. +#if !(JSON_DUMP_SHORT_STRINGS && JSON_SCAN_VECTOR) if (s.size() <= 32) { const auto& copyable = ascii_copyable(); @@ -30863,6 +31032,7 @@ class serializer return; } } +#endif // dispatch once here rather than test the flag inside the loop: it does // not change while a string is written, and folding it lets each of the @@ -40701,6 +40871,7 @@ struct formatter // NOLINT(cert-dcl58-c #undef JSON_SCAN_SSSE3_TARGET #undef JSON_SCAN_VECTOR #undef JSON_SCAN_VECTOR_UTF8 +#undef JSON_DUMP_SHORT_STRINGS #ifndef JSON_TEST_KEEP_MACROS #undef JSON_CATCH diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index 5722c75a0..338eef291 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -1580,3 +1580,54 @@ TEST_CASE("invalid UTF-8 at every position is handled like the reference") } } } + +TEST_CASE("short strings are written like the reference, as values and as keys") +{ + // strings of up to 16 characters that need no escaping are written as a + // whole (together with the quotes and, for a key, the colon); everything + // else, including a string that is merely short, takes the general path. + // Every length around the edges of the size classes of the fast path is + // combined with each kind of byte that must send a string to the general + // path, at each position, and compared with the reference for values and, + // via the value, for keys. + const std::vector specials = + { + "\"", "\\", "\n", "\x1f", "\x7f", "\xc3\xa9", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xff", "\x80", "\xc3" + }; + + for (std::size_t length = 0; length <= 20; ++length) + { + const std::string plain(length, 'k'); + check_dump_against_reference(plain, false); + + for (const auto& special : specials) + { + for (std::size_t pos = 0; pos <= length; ++pos) + { + std::string s = plain; + s.insert(pos, special); + check_dump_against_reference(s, false); + + for (const bool ensure_ascii : + { + false, true + }) + { + // a key and a value are written alike, in compact and + // in pretty-printed form + const json value = s; + const json object = json::object({{s, 1}}); + for (const auto error_handler : + { + json::error_handler_t::replace, json::error_handler_t::keep + }) + { + const std::string quoted = value.dump(-1, ' ', ensure_ascii, error_handler); + CHECK(object.dump(-1, ' ', ensure_ascii, error_handler) == "{" + quoted + ":1}"); + CHECK(object.dump(2, ' ', ensure_ascii, error_handler) == "{\n " + quoted + ": 1\n}"); + } + } + } + } + } +}