Append to the output vector through a position-tracking sink

The vector-returning to_cbor(), to_msgpack(), ... wrote through
output_vector_sink, which appended each write with push_back() or
insert(). A number's 1 to 8 payload bytes went through vector::insert(),
which libc++ and libstdc++ both run as an out-of-line insert-anywhere
routine; on integer arrays that call cost more than encoding the number,
so the devirtualization of #5286 brought nothing there, and with GCC the
writer was slower than in 3.12.0.

output_vector_sink now keeps its own write position: it grows the vector
geometrically ahead of the position (resize(), using any capacity the
vector already has), copies each write there, and shrinks the vector to
the written length in its destructor, also when a write throws. Writes of
256 characters or more that do not fit are appended with insert(), so a
large string or binary value is copied once instead of zero-filled and
copied. The sink is move-only, as it owns that trim.

output_vector_adapter (the to_*(j, vector) overloads) appends with
push_back()/insert() again, so a caller holding the adapter never sees
the space kept ahead.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/output_adapters.hpp

@@ -8,7 +8,9 @@
 
 #pragma once
 
+#include <algorithm> // max
 #include <cstddef> // size_t
+#include <cstring> // memcpy
 #include <memory> // shared_ptr, make_shared
 #include <string> // basic_string
 #include <type_traits> // conditional, integral_constant, is_same
@@ -48,32 +50,145 @@ template<typename CharType> struct output_adapter_protocol
 template<typename CharType>
 using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
 
+/// output_vector_sink appends writes of at least this many characters with
+/// insert() instead of copying them to the space it keeps ahead
+constexpr std::size_t max_reserve_size = 256;
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
 /// passed to binary_writer by value as a template parameter, so
 /// write_character()/write_characters() are ordinary (inlinable) calls with no
 /// vtable lookup and no shared_ptr. It is used for the common
-/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below
-/// wraps this same sink to provide the virtual interface.
+/// `to_cbor`/`to_msgpack`/... into a std::vector.
+///
+/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8
+/// bytes - and appending those with push_back() or insert() cost more than
+/// encoding them: insert() is an out-of-line call into a routine that can insert
+/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
+/// its own write position: it grows the vector geometrically ahead of the
+/// position and copies each write to it, so a write is a capacity check and a
+/// short copy. The destructor shrinks the vector to the bytes actually written,
+/// also when a write throws; shrinking a vector never allocates. The sink owns
+/// that trim, so it can be moved but not copied, and the vector must not be used
+/// while the sink is alive.
 template<typename CharType, typename AllocatorType = std::allocator<CharType>>
 class output_vector_sink
 {
   public:
     explicit output_vector_sink(std::vector<CharType, AllocatorType>& vec) noexcept
+        : v(&vec), position(vec.size())
+    {}
+
+    output_vector_sink(const output_vector_sink&) = delete;
+    output_vector_sink& operator=(const output_vector_sink&) = delete;
+    output_vector_sink& operator=(output_vector_sink&&) = delete;
+
+    output_vector_sink(output_vector_sink&& other) noexcept
+        : v(other.v), position(other.position)
+    {
+        other.v = nullptr;
+    }
+
+    ~output_vector_sink()
+    {
+        if (v != nullptr)
+        {
+            v->resize(position);
+        }
+    }
+
+    // always inlined: GCC otherwise keeps these out of line in a large
+    // translation unit, and a call per byte or number costs more than the write
+    JSON_HEDLEY_ALWAYS_INLINE
+    void write_character(CharType c)
+    {
+        if (JSON_HEDLEY_UNLIKELY(position == v->size()))
+        {
+            grow(1);
+        }
+        (*v)[position++] = c;
+    }
+
+    // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
+    // pointer with length 0 for empty strings/binary values; memcpy must not
+    // see it.
+    JSON_HEDLEY_ALWAYS_INLINE
+    void write_characters(const CharType* s, std::size_t length)
+    {
+        if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
+        {
+            if (length >= max_reserve_size)
+            {
+                append(s, length);
+                return;
+            }
+            grow(length);
+        }
+        if (length != 0)
+        {
+            std::memcpy(v->data() + position, s, length * sizeof(CharType));
+            position += length;
+        }
+    }
+
+  private:
+    /// make room for at least @a length more characters after the position;
+    /// kept out of line so that the write functions stay small enough for
+    /// GCC to inline them into the writer
+    JSON_HEDLEY_NEVER_INLINE
+    void grow(const std::size_t length)
+    {
+        // use the capacity the vector already has, else at least double it;
+        // never less than 64 characters, so that a small value does not
+        // reallocate for every few bytes
+        const std::size_t needed = position + length;
+        v->resize((std::max)(std::size_t{64}, needed <= v->capacity()
+                             ? v->capacity()
+                             : (std::max)(needed, v->capacity() * 2)));
+    }
+
+    /// append @a length (at least max_reserve_size) characters that do not fit
+    /// the space ahead of the position: insert() copies them once into the
+    /// grown vector, where grow() would first zero the new space and then copy
+    /// over it - noticeable for a large string or binary value
+    JSON_HEDLEY_NEVER_INLINE
+    void append(const CharType* s, const std::size_t length)
+    {
+        v->resize(position);
+        if (v->capacity() - position < length)
+        {
+            // grow geometrically, with room for the writes that follow, so
+            // that the next grow() need not reallocate right away
+            v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size));
+        }
+        v->insert(v->end(), s, s + length);
+        position = v->size();
+    }
+
+    std::vector<CharType, AllocatorType>* v;
+    /// number of characters of *v written so far
+    std::size_t position;
+};
+
+/// output adapter for byte vectors
+template<typename CharType, typename AllocatorType = std::allocator<CharType>>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
         : v(vec)
     {}
 
-    void write_character(CharType c)
+    void write_character(CharType c) override
     {
         v.push_back(c);
     }
 
     // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
-    // pointer with length 0 for empty strings/binary values. Appending an empty
-    // range is a no-op; the type-erased path tolerates this via the (unattributed)
-    // virtual base, and the concrete sink must do the same.
-    void write_characters(const CharType* s, std::size_t length)
+    // pointer with length 0 for empty strings/binary values; appending an empty
+    // range is a no-op
+    void write_characters(const CharType* s, std::size_t length) override
     {
         v.insert(v.end(), s, s + length);
     }
@@ -82,33 +197,6 @@ class output_vector_sink
     std::vector<CharType, AllocatorType>& v;
 };
 
-/// output adapter for byte vectors
-///
-/// The appending itself lives in output_vector_sink; this class only adds the
-/// virtual output_adapter_protocol interface on top of it, so both the
-/// type-erased and the templated path share one implementation.
-template<typename CharType, typename AllocatorType = std::allocator<CharType>>
-class output_vector_adapter : public output_adapter_protocol<CharType>
-{
-  public:
-    explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
-        : sink(vec)
-    {}
-
-    void write_character(CharType c) override
-    {
-        sink.write_character(c);
-    }
-
-    void write_characters(const CharType* s, std::size_t length) override
-    {
-        sink.write_characters(s, length);
-    }
-
-  private:
-    output_vector_sink<CharType, AllocatorType> sink;
-};
-
 #ifndef JSON_NO_IO
 /// output adapter for output streams
 template<typename CharType>

single_include/nlohmann/json.hpp

@@ -21638,7 +21638,9 @@ NLOHMANN_JSON_NAMESPACE_END
 
 
 
+#include <algorithm> // max
 #include <cstddef> // size_t
+#include <cstring> // memcpy
 #include <memory> // shared_ptr, make_shared
 #include <string> // basic_string
 #include <type_traits> // conditional, integral_constant, is_same
@@ -21679,32 +21681,145 @@ template<typename CharType> struct output_adapter_protocol
 template<typename CharType>
 using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
 
+/// output_vector_sink appends writes of at least this many characters with
+/// insert() instead of copying them to the space it keeps ahead
+constexpr std::size_t max_reserve_size = 256;
+
 /// @brief non-virtual output sink writing into a std::vector
 ///
 /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is
 /// passed to binary_writer by value as a template parameter, so
 /// write_character()/write_characters() are ordinary (inlinable) calls with no
 /// vtable lookup and no shared_ptr. It is used for the common
-/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below
-/// wraps this same sink to provide the virtual interface.
+/// `to_cbor`/`to_msgpack`/... into a std::vector.
+///
+/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8
+/// bytes - and appending those with push_back() or insert() cost more than
+/// encoding them: insert() is an out-of-line call into a routine that can insert
+/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps
+/// its own write position: it grows the vector geometrically ahead of the
+/// position and copies each write to it, so a write is a capacity check and a
+/// short copy. The destructor shrinks the vector to the bytes actually written,
+/// also when a write throws; shrinking a vector never allocates. The sink owns
+/// that trim, so it can be moved but not copied, and the vector must not be used
+/// while the sink is alive.
 template<typename CharType, typename AllocatorType = std::allocator<CharType>>
 class output_vector_sink
 {
   public:
     explicit output_vector_sink(std::vector<CharType, AllocatorType>& vec) noexcept
+        : v(&vec), position(vec.size())
+    {}
+
+    output_vector_sink(const output_vector_sink&) = delete;
+    output_vector_sink& operator=(const output_vector_sink&) = delete;
+    output_vector_sink& operator=(output_vector_sink&&) = delete;
+
+    output_vector_sink(output_vector_sink&& other) noexcept
+        : v(other.v), position(other.position)
+    {
+        other.v = nullptr;
+    }
+
+    ~output_vector_sink()
+    {
+        if (v != nullptr)
+        {
+            v->resize(position);
+        }
+    }
+
+    // always inlined: GCC otherwise keeps these out of line in a large
+    // translation unit, and a call per byte or number costs more than the write
+    JSON_HEDLEY_ALWAYS_INLINE
+    void write_character(CharType c)
+    {
+        if (JSON_HEDLEY_UNLIKELY(position == v->size()))
+        {
+            grow(1);
+        }
+        (*v)[position++] = c;
+    }
+
+    // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
+    // pointer with length 0 for empty strings/binary values; memcpy must not
+    // see it.
+    JSON_HEDLEY_ALWAYS_INLINE
+    void write_characters(const CharType* s, std::size_t length)
+    {
+        if (JSON_HEDLEY_UNLIKELY(v->size() - position < length))
+        {
+            if (length >= max_reserve_size)
+            {
+                append(s, length);
+                return;
+            }
+            grow(length);
+        }
+        if (length != 0)
+        {
+            std::memcpy(v->data() + position, s, length * sizeof(CharType));
+            position += length;
+        }
+    }
+
+  private:
+    /// make room for at least @a length more characters after the position;
+    /// kept out of line so that the write functions stay small enough for
+    /// GCC to inline them into the writer
+    JSON_HEDLEY_NEVER_INLINE
+    void grow(const std::size_t length)
+    {
+        // use the capacity the vector already has, else at least double it;
+        // never less than 64 characters, so that a small value does not
+        // reallocate for every few bytes
+        const std::size_t needed = position + length;
+        v->resize((std::max)(std::size_t{64}, needed <= v->capacity()
+                             ? v->capacity()
+                             : (std::max)(needed, v->capacity() * 2)));
+    }
+
+    /// append @a length (at least max_reserve_size) characters that do not fit
+    /// the space ahead of the position: insert() copies them once into the
+    /// grown vector, where grow() would first zero the new space and then copy
+    /// over it - noticeable for a large string or binary value
+    JSON_HEDLEY_NEVER_INLINE
+    void append(const CharType* s, const std::size_t length)
+    {
+        v->resize(position);
+        if (v->capacity() - position < length)
+        {
+            // grow geometrically, with room for the writes that follow, so
+            // that the next grow() need not reallocate right away
+            v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size));
+        }
+        v->insert(v->end(), s, s + length);
+        position = v->size();
+    }
+
+    std::vector<CharType, AllocatorType>* v;
+    /// number of characters of *v written so far
+    std::size_t position;
+};
+
+/// output adapter for byte vectors
+template<typename CharType, typename AllocatorType = std::allocator<CharType>>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
         : v(vec)
     {}
 
-    void write_character(CharType c)
+    void write_character(CharType c) override
     {
         v.push_back(c);
     }
 
     // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
-    // pointer with length 0 for empty strings/binary values. Appending an empty
-    // range is a no-op; the type-erased path tolerates this via the (unattributed)
-    // virtual base, and the concrete sink must do the same.
-    void write_characters(const CharType* s, std::size_t length)
+    // pointer with length 0 for empty strings/binary values; appending an empty
+    // range is a no-op
+    void write_characters(const CharType* s, std::size_t length) override
     {
         v.insert(v.end(), s, s + length);
     }
@@ -21713,33 +21828,6 @@ class output_vector_sink
     std::vector<CharType, AllocatorType>& v;
 };
 
-/// output adapter for byte vectors
-///
-/// The appending itself lives in output_vector_sink; this class only adds the
-/// virtual output_adapter_protocol interface on top of it, so both the
-/// type-erased and the templated path share one implementation.
-template<typename CharType, typename AllocatorType = std::allocator<CharType>>
-class output_vector_adapter : public output_adapter_protocol<CharType>
-{
-  public:
-    explicit output_vector_adapter(std::vector<CharType, AllocatorType>& vec) noexcept
-        : sink(vec)
-    {}
-
-    void write_character(CharType c) override
-    {
-        sink.write_character(c);
-    }
-
-    void write_characters(const CharType* s, std::size_t length) override
-    {
-        sink.write_characters(s, length);
-    }
-
-  private:
-    output_vector_sink<CharType, AllocatorType> sink;
-};
-
 #ifndef JSON_NO_IO
 /// output adapter for output streams
 template<typename CharType>

tests/src/unit-binary_writer_sinks.cpp

@@ -35,6 +35,17 @@ std::vector<json> test_values()
         big_object[std::to_string(i)] = i;
     }
 
+    // large strings and binary values between small ones: a write too large
+    // for the space the vector sink has ahead takes a different path than a
+    // small one, and the small writes after it must still land behind it
+    json large_payloads = json::array();
+    for (int i = 0; i < 50; ++i)
+    {
+        large_payloads.push_back(std::string(300, static_cast<char>('a' + (i % 26))));
+        large_payloads.push_back(i);
+        large_payloads.push_back(json::binary(std::vector<std::uint8_t>(static_cast<std::size_t>(100 * i), static_cast<std::uint8_t>(i))));
+    }
+
     return
     {
         json(nullptr), json(true), json(false),
@@ -46,7 +57,8 @@ std::vector<json> test_values()
         json::array(), json::object(),
         json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}),
         json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}),
-        big_array, big_object
+        big_array, big_object, large_payloads,
+        json::binary(std::vector<std::uint8_t>(5000, 0x42))
     };
 }