Write a CBOR or MessagePack string with its head in one reservation

A string or object key took a string_t constructed for the UTF-8 error
handler, a call of the (out of line) sanitize_utf8_for_write() that returns
the string unchanged for the default handler, a call that builds the head from
a range-checked length and writes it with a std::array and a sink call of its
own, and a sink write for the characters, which copied it with a call of
memcpy() through a stub with its own dispatch on the length.

With the default error handler (keep) and a string of up to 254 characters,
the head and the characters are now written into space reserved from the sink
in one go. Longer strings and the other handlers take the previous code. The
sinks copy up to 32 characters with fixed-size loads and stores - two possibly
overlapping 16, 8 or 4 byte copies, or the first, middle and last character of
a string of 1 to 3 - instead of calling memcpy(); nothing outside of the
source or destination range is touched.

Instructions per call (Linux, GCC 13, thousands), to_msgpack: twitter 4414 -> 3383, 20000 small objects 23895 -> 18342, citm_catalog 8575 -> 7031; to_cbor: 20000 small objects 20695 -> 17182, twitter 3842 -> 3204.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/binary_writer.hpp

@@ -2839,6 +2839,24 @@ class binary_writer
         write_msgpack(BasicJsonType(key));
     }
 
+    /// the longest string that is written, with its head of one or two
+    /// characters, in a single reservation from the sink
+    static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2;
+
+    /// copy the characters of a string to space in the sink; a short one
+    /// without a call of memcpy()
+    static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept
+    {
+        if (length <= 32)
+        {
+            copy_short(dst, src, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+        }
+        else
+        {
+            std::memcpy(dst, src, length * sizeof(CharType));
+        }
+    }
+
     /*!
     @brief write a CBOR text string
 
@@ -2855,6 +2873,28 @@ class binary_writer
     */
     void write_cbor_string(const string_t& value, const BasicJsonType& context)
     {
+        if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep))
+        {
+            // the string is written as it is: head and characters in one reservation
+            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
+            {
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                return;
+            }
+        }
+
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -2883,6 +2923,30 @@ class binary_writer
     */
     void write_msgpack_string(const string_t& value, const BasicJsonType& context)
     {
+        if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep))
+        {
+            // the string is written as it is: head and characters in one reservation
+            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
+            {
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                return;
+            }
+        }
+
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 

include/nlohmann/detail/output/output_adapters.hpp

@@ -55,6 +55,63 @@ using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
 /// output_vector_sink also appends writes of at least this size with insert()
 constexpr std::size_t max_reserve_size = 256;
 
+/// @brief copy @a length characters, at most 32, without a call of memcpy()
+///
+/// Short strings are the typical payload of a binary value, and a call of
+/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two
+/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and
+/// so on down to a single character; each is a fixed-size load and store.
+/// Nothing outside of [src, src + length) is read or written.
+template<typename CharType>
+inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept
+{
+    JSON_ASSERT(length <= 32);
+    if (length >= 16)
+    {
+        std::array<char, 16> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 16> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 16);
+        std::memcpy(last.data(), src + (length - 16), 16);
+        std::memcpy(dst, first.data(), 16);
+        std::memcpy(dst + (length - 16), last.data(), 16);
+    }
+    else if (length >= 8)
+    {
+        std::array<char, 8> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 8> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 8);
+        std::memcpy(last.data(), src + (length - 8), 8);
+        std::memcpy(dst, first.data(), 8);
+        std::memcpy(dst + (length - 8), last.data(), 8);
+    }
+    else if (length >= 4)
+    {
+        std::array<char, 4> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 4> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 4);
+        std::memcpy(last.data(), src + (length - 4), 4);
+        std::memcpy(dst, first.data(), 4);
+        std::memcpy(dst + (length - 4), last.data(), 4);
+    }
+    else if (length != 0)
+    {
+        // 1 to 3: the first, the middle, and the last character
+        const CharType a = src[0];
+        const CharType b = src[length >> 1];
+        const CharType c = src[length - 1];
+        dst[0] = a;
+        dst[length >> 1] = b;
+        dst[length - 1] = c;
+    }
+}
+
+/// @brief copy @a length characters, of a character type wider than a byte
+template<typename CharType>
+inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept
+{
+    std::memcpy(dst, src, length * sizeof(CharType));
+}
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
@@ -126,7 +183,12 @@ class output_vector_sink
             }
             grow(length);
         }
-        if (length != 0)
+        if (length <= 32)
+        {
+            copy_short(v->data() + position, s, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+            position += length;
+        }
+        else
         {
             std::memcpy(v->data() + position, s, length * sizeof(CharType));
             position += length;
@@ -349,7 +411,12 @@ class output_adapter_sink
             write_large(s, length);
             return;
         }
-        if (length != 0)
+        if (length <= 32)
+        {
+            copy_short(buffer.data() + used, s, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+            used += length;
+        }
+        else
         {
             std::memcpy(buffer.data() + used, s, length * sizeof(CharType));
             used += length;

single_include/nlohmann/json.hpp

@@ -24321,6 +24321,63 @@ using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
 /// output_vector_sink also appends writes of at least this size with insert()
 constexpr std::size_t max_reserve_size = 256;
 
+/// @brief copy @a length characters, at most 32, without a call of memcpy()
+///
+/// Short strings are the typical payload of a binary value, and a call of
+/// memcpy() costs more than copying them. A copy of 16 to 32 characters is two
+/// possibly overlapping 16-byte copies, one of 8 to 15 two 8-byte copies, and
+/// so on down to a single character; each is a fixed-size load and store.
+/// Nothing outside of [src, src + length) is read or written.
+template<typename CharType>
+inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::true_type /*is_byte*/) noexcept
+{
+    JSON_ASSERT(length <= 32);
+    if (length >= 16)
+    {
+        std::array<char, 16> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 16> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 16);
+        std::memcpy(last.data(), src + (length - 16), 16);
+        std::memcpy(dst, first.data(), 16);
+        std::memcpy(dst + (length - 16), last.data(), 16);
+    }
+    else if (length >= 8)
+    {
+        std::array<char, 8> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 8> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 8);
+        std::memcpy(last.data(), src + (length - 8), 8);
+        std::memcpy(dst, first.data(), 8);
+        std::memcpy(dst + (length - 8), last.data(), 8);
+    }
+    else if (length >= 4)
+    {
+        std::array<char, 4> first; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::array<char, 4> last; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read
+        std::memcpy(first.data(), src, 4);
+        std::memcpy(last.data(), src + (length - 4), 4);
+        std::memcpy(dst, first.data(), 4);
+        std::memcpy(dst + (length - 4), last.data(), 4);
+    }
+    else if (length != 0)
+    {
+        // 1 to 3: the first, the middle, and the last character
+        const CharType a = src[0];
+        const CharType b = src[length >> 1];
+        const CharType c = src[length - 1];
+        dst[0] = a;
+        dst[length >> 1] = b;
+        dst[length - 1] = c;
+    }
+}
+
+/// @brief copy @a length characters, of a character type wider than a byte
+template<typename CharType>
+inline void copy_short(CharType* dst, const CharType* src, const std::size_t length, std::false_type /*is_byte*/) noexcept
+{
+    std::memcpy(dst, src, length * sizeof(CharType));
+}
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
@@ -24392,7 +24449,12 @@ class output_vector_sink
             }
             grow(length);
         }
-        if (length != 0)
+        if (length <= 32)
+        {
+            copy_short(v->data() + position, s, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+            position += length;
+        }
+        else
         {
             std::memcpy(v->data() + position, s, length * sizeof(CharType));
             position += length;
@@ -24615,7 +24677,12 @@ class output_adapter_sink
             write_large(s, length);
             return;
         }
-        if (length != 0)
+        if (length <= 32)
+        {
+            copy_short(buffer.data() + used, s, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+            used += length;
+        }
+        else
         {
             std::memcpy(buffer.data() + used, s, length * sizeof(CharType));
             used += length;
@@ -27600,6 +27667,24 @@ class binary_writer
         write_msgpack(BasicJsonType(key));
     }
 
+    /// the longest string that is written, with its head of one or two
+    /// characters, in a single reservation from the sink
+    static constexpr std::size_t max_text_in_reservation = max_reserve_size - 2;
+
+    /// copy the characters of a string to space in the sink; a short one
+    /// without a call of memcpy()
+    static void copy_characters(CharType* dst, const CharType* src, const std::size_t length) noexcept
+    {
+        if (length <= 32)
+        {
+            copy_short(dst, src, length, std::integral_constant<bool, sizeof(CharType) == 1>());
+        }
+        else
+        {
+            std::memcpy(dst, src, length * sizeof(CharType));
+        }
+    }
+
     /*!
     @brief write a CBOR text string
 
@@ -27616,6 +27701,28 @@ class binary_writer
     */
     void write_cbor_string(const string_t& value, const BasicJsonType& context)
     {
+        if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep))
+        {
+            // the string is written as it is: head and characters in one reservation
+            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
+            {
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                return;
+            }
+        }
+
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -27644,6 +27751,30 @@ class binary_writer
     */
     void write_msgpack_string(const string_t& value, const BasicJsonType& context)
     {
+        if (JSON_HEDLEY_LIKELY(error_handler == error_handler_t::keep))
+        {
+            // the string is written as it is: head and characters in one reservation
+            if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
+            {
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
+                return;
+            }
+        }
+
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);