diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 6d3fc6c9e..9ca47cb7a 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2839,6 +2839,24 @@ class binary_writer write_msgpack(BasicJsonType(key)); } + /// the longest string that is written, with its head of one or two + /// characters, in a single reservation from the sink + static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2; + + /// copy the characters of a string to space in the sink; a short one + /// without a call of memcpy() + static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept + { + if (length <= 32) + { + copy_short(dst, src, length, std::integral_constant()); + } + else + { + std::memcpy(dst, src, length * sizeof(CharType)); + } + } + /*! @brief write a CBOR text string @@ -2855,6 +2873,28 @@ class binary_writer */ void write_cbor_string(const string_t& value, const BasicJsonType& context) { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -2883,6 +2923,30 @@ class binary_writer */ void write_msgpack_string(const string_t& value, const BasicJsonType& context) { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index 917c8e648..a5b3ec9d4 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -55,6 +55,63 @@ using output_adapter_t = std::shared_ptr>; /// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; +/// @brief copy @a length characters, at most 32, without a call of memcpy() +/// +/// Short strings are the typical payload of a binary value, and a call of +/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two +/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and +/// so on down to a single character; each is a fixed-size load and store. +/// Nothing outside of [src, src + length) is read or written. +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept +{ + JSON_ASSERT(length <= 32); + if (length >= 16) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 16); + std::memcpy(last.data(), src + (length - 16), 16); + std::memcpy(dst, first.data(), 16); + std::memcpy(dst + (length - 16), last.data(), 16); + } + else if (length >= 8) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 8); + std::memcpy(last.data(), src + (length - 8), 8); + std::memcpy(dst, first.data(), 8); + std::memcpy(dst + (length - 8), last.data(), 8); + } + else if (length >= 4) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 4); + std::memcpy(last.data(), src + (length - 4), 4); + std::memcpy(dst, first.data(), 4); + std::memcpy(dst + (length - 4), last.data(), 4); + } + else if (length != 0) + { + // 1 to 3: the first, the middle, and the last character + const CharType a = src[0]; + const CharType b = src[length >> 1]; + const CharType c = src[length - 1]; + dst[0] = a; + dst[length >> 1] = b; + dst[length - 1] = c; + } +} + +/// @brief copy @a length characters, of a character type wider than a byte +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept +{ + std::memcpy(dst, src, length * sizeof(CharType)); +} + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -126,7 +183,12 @@ class output_vector_sink } grow(length); } - if (length != 0) + if (length <= 32) + { + copy_short(v->data() + position, s, length, std::integral_constant()); + position += length; + } + else { std::memcpy(v->data() + position, s, length * sizeof(CharType)); position += length; @@ -349,7 +411,12 @@ class output_adapter_sink write_large(s, length); return; } - if (length != 0) + if (length <= 32) + { + copy_short(buffer.data() + used, s, length, std::integral_constant()); + used += length; + } + else { std::memcpy(buffer.data() + used, s, length * sizeof(CharType)); used += length; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3ca284a61..718060947 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -24321,6 +24321,63 @@ using output_adapter_t = std::shared_ptr>; /// output_vector_sink also appends writes of at least this size with insert() constexpr std::size_t max_reserve_size = 256; +/// @brief copy @a length characters, at most 32, without a call of memcpy() +/// +/// Short strings are the typical payload of a binary value, and a call of +/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two +/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and +/// so on down to a single character; each is a fixed-size load and store. +/// Nothing outside of [src, src + length) is read or written. +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept +{ + JSON_ASSERT(length <= 32); + if (length >= 16) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 16); + std::memcpy(last.data(), src + (length - 16), 16); + std::memcpy(dst, first.data(), 16); + std::memcpy(dst + (length - 16), last.data(), 16); + } + else if (length >= 8) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 8); + std::memcpy(last.data(), src + (length - 8), 8); + std::memcpy(dst, first.data(), 8); + std::memcpy(dst + (length - 8), last.data(), 8); + } + else if (length >= 4) + { + std::array first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::array last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + std::memcpy(first.data(), src, 4); + std::memcpy(last.data(), src + (length - 4), 4); + std::memcpy(dst, first.data(), 4); + std::memcpy(dst + (length - 4), last.data(), 4); + } + else if (length != 0) + { + // 1 to 3: the first, the middle, and the last character + const CharType a = src[0]; + const CharType b = src[length >> 1]; + const CharType c = src[length - 1]; + dst[0] = a; + dst[length >> 1] = b; + dst[length - 1] = c; + } +} + +/// @brief copy @a length characters, of a character type wider than a byte +template +inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept +{ + std::memcpy(dst, src, length * sizeof(CharType)); +} + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is @@ -24392,7 +24449,12 @@ class output_vector_sink } grow(length); } - if (length != 0) + if (length <= 32) + { + copy_short(v->data() + position, s, length, std::integral_constant()); + position += length; + } + else { std::memcpy(v->data() + position, s, length * sizeof(CharType)); position += length; @@ -24615,7 +24677,12 @@ class output_adapter_sink write_large(s, length); return; } - if (length != 0) + if (length <= 32) + { + copy_short(buffer.data() + used, s, length, std::integral_constant()); + used += length; + } + else { std::memcpy(buffer.data() + used, s, length * sizeof(CharType)); used += length; @@ -27600,6 +27667,24 @@ class binary_writer write_msgpack(BasicJsonType(key)); } + /// the longest string that is written, with its head of one or two + /// characters, in a single reservation from the sink + static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2; + + /// copy the characters of a string to space in the sink; a short one + /// without a call of memcpy() + static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept + { + if (length <= 32) + { + copy_short(dst, src, length, std::integral_constant()); + } + else + { + std::memcpy(dst, src, length * sizeof(CharType)); + } + } + /*! @brief write a CBOR text string @@ -27616,6 +27701,28 @@ class binary_writer */ void write_cbor_string(const string_t& value, const BasicJsonType& context) { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 0x17) + { + *p++ = to_char_type(static_cast(0x60 + value.size())); + } + else + { + *p++ = to_char_type(0x78); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage); @@ -27644,6 +27751,30 @@ class binary_writer */ void write_msgpack_string(const string_t& value, const BasicJsonType& context) { + if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep)) + { + // the string is written as it is: head and characters in one reservation + if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation)) + { + CharType* const out = oa.reserve(value.size() + 2); + CharType* p = out; + if (value.size() <= 31) + { + // fixstr + *p++ = to_char_type(static_cast(0xA0 | value.size())); + } + else + { + // str 8 + *p++ = to_char_type(0xD9); + *p++ = to_char_type(static_cast(value.size())); + } + copy_characters(p, reinterpret_cast(value.data()), value.size()); + oa.commit(static_cast(p - out) + value.size()); + return; + } + } + string_t storage; const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);