diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index 53a14d42f..dc2c06587 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -8,7 +8,9 @@ #pragma once +#include // max #include // size_t +#include // memcpy #include // shared_ptr, make_shared #include // basic_string #include // conditional, integral_constant, is_same @@ -48,32 +50,145 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; +/// output_vector_sink appends writes of at least this many characters with +/// insert() instead of copying them to the space it keeps ahead +constexpr std::size_t max_reserve_size = 256; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is /// passed to binary_writer by value as a template parameter, so /// write_character()/write_characters() are ordinary (inlinable) calls with no /// vtable lookup and no shared_ptr. It is used for the common -/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below -/// wraps this same sink to provide the virtual interface. +/// `to_cbor`/`to_msgpack`/... into a std::vector. +/// +/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8 +/// bytes - and appending those with push_back() or insert() cost more than +/// encoding them: insert() is an out-of-line call into a routine that can insert +/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps +/// its own write position: it grows the vector geometrically ahead of the +/// position and copies each write to it, so a write is a capacity check and a +/// short copy. The destructor shrinks the vector to the bytes actually written, +/// also when a write throws; shrinking a vector never allocates. The sink owns +/// that trim, so it can be moved but not copied, and the vector must not be used +/// while the sink is alive. template> class output_vector_sink { public: explicit output_vector_sink(std::vector& vec) noexcept + : v(&vec), position(vec.size()) + {} + + output_vector_sink(const output_vector_sink&) = delete; + output_vector_sink& operator=(const output_vector_sink&) = delete; + output_vector_sink& operator=(output_vector_sink&&) = delete; + + output_vector_sink(output_vector_sink&& other) noexcept + : v(other.v), position(other.position) + { + other.v = nullptr; + } + + ~output_vector_sink() + { + if (v != nullptr) + { + v->resize(position); + } + } + + // always inlined: GCC otherwise keeps these out of line in a large + // translation unit, and a call per byte or number costs more than the write + JSON_HEDLEY_ALWAYS_INLINE + void write_character(CharType c) + { + if (JSON_HEDLEY_UNLIKELY(position == v->size())) + { + grow(1); + } + (*v)[position++] = c; + } + + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values; memcpy must not + // see it. + JSON_HEDLEY_ALWAYS_INLINE + void write_characters(const CharType* s, std::size_t length) + { + if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + { + if (length >= max_reserve_size) + { + append(s, length); + return; + } + grow(length); + } + if (length != 0) + { + std::memcpy(v->data() + position, s, length * sizeof(CharType)); + position += length; + } + } + + private: + /// make room for at least @a length more characters after the position; + /// kept out of line so that the write functions stay small enough for + /// GCC to inline them into the writer + JSON_HEDLEY_NEVER_INLINE + void grow(const std::size_t length) + { + // use the capacity the vector already has, else at least double it; + // never less than 64 characters, so that a small value does not + // reallocate for every few bytes + const std::size_t needed = position + length; + v->resize((std::max)(std::size_t{64}, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2))); + } + + /// append @a length (at least max_reserve_size) characters that do not fit + /// the space ahead of the position: insert() copies them once into the + /// grown vector, where grow() would first zero the new space and then copy + /// over it - noticeable for a large string or binary value + JSON_HEDLEY_NEVER_INLINE + void append(const CharType* s, const std::size_t length) + { + v->resize(position); + if (v->capacity() - position < length) + { + // grow geometrically, with room for the writes that follow, so + // that the next grow() need not reallocate right away + v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size)); + } + v->insert(v->end(), s, s + length); + position = v->size(); + } + + std::vector* v; + /// number of characters of *v written so far + std::size_t position; +}; + +/// output adapter for byte vectors +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept : v(vec) {} - void write_character(CharType c) + void write_character(CharType c) override { v.push_back(c); } // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null - // pointer with length 0 for empty strings/binary values. Appending an empty - // range is a no-op; the type-erased path tolerates this via the (unattributed) - // virtual base, and the concrete sink must do the same. - void write_characters(const CharType* s, std::size_t length) + // pointer with length 0 for empty strings/binary values; appending an empty + // range is a no-op + void write_characters(const CharType* s, std::size_t length) override { v.insert(v.end(), s, s + length); } @@ -82,33 +197,6 @@ class output_vector_sink std::vector& v; }; -/// output adapter for byte vectors -/// -/// The appending itself lives in output_vector_sink; this class only adds the -/// virtual output_adapter_protocol interface on top of it, so both the -/// type-erased and the templated path share one implementation. -template> -class output_vector_adapter : public output_adapter_protocol -{ - public: - explicit output_vector_adapter(std::vector& vec) noexcept - : sink(vec) - {} - - void write_character(CharType c) override - { - sink.write_character(c); - } - - void write_characters(const CharType* s, std::size_t length) override - { - sink.write_characters(s, length); - } - - private: - output_vector_sink sink; -}; - #ifndef JSON_NO_IO /// output adapter for output streams template diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 16c87bf36..f83725215 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -21638,7 +21638,9 @@ NLOHMANN_JSON_NAMESPACE_END +#include // max #include // size_t +#include // memcpy #include // shared_ptr, make_shared #include // basic_string #include // conditional, integral_constant, is_same @@ -21679,32 +21681,145 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; +/// output_vector_sink appends writes of at least this many characters with +/// insert() instead of copying them to the space it keeps ahead +constexpr std::size_t max_reserve_size = 256; + /// @brief non-virtual output sink writing into a std::vector /// /// This sink is not part of the virtual output_adapter_protocol hierarchy: it is /// passed to binary_writer by value as a template parameter, so /// write_character()/write_characters() are ordinary (inlinable) calls with no /// vtable lookup and no shared_ptr. It is used for the common -/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below -/// wraps this same sink to provide the virtual interface. +/// `to_cbor`/`to_msgpack`/... into a std::vector. +/// +/// Most writes are a few bytes - a type marker, or a number of 1, 2, 4, or 8 +/// bytes - and appending those with push_back() or insert() cost more than +/// encoding them: insert() is an out-of-line call into a routine that can insert +/// anywhere, and resize() is one as well with libstdc++. The sink therefore keeps +/// its own write position: it grows the vector geometrically ahead of the +/// position and copies each write to it, so a write is a capacity check and a +/// short copy. The destructor shrinks the vector to the bytes actually written, +/// also when a write throws; shrinking a vector never allocates. The sink owns +/// that trim, so it can be moved but not copied, and the vector must not be used +/// while the sink is alive. template> class output_vector_sink { public: explicit output_vector_sink(std::vector& vec) noexcept + : v(&vec), position(vec.size()) + {} + + output_vector_sink(const output_vector_sink&) = delete; + output_vector_sink& operator=(const output_vector_sink&) = delete; + output_vector_sink& operator=(output_vector_sink&&) = delete; + + output_vector_sink(output_vector_sink&& other) noexcept + : v(other.v), position(other.position) + { + other.v = nullptr; + } + + ~output_vector_sink() + { + if (v != nullptr) + { + v->resize(position); + } + } + + // always inlined: GCC otherwise keeps these out of line in a large + // translation unit, and a call per byte or number costs more than the write + JSON_HEDLEY_ALWAYS_INLINE + void write_character(CharType c) + { + if (JSON_HEDLEY_UNLIKELY(position == v->size())) + { + grow(1); + } + (*v)[position++] = c; + } + + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values; memcpy must not + // see it. + JSON_HEDLEY_ALWAYS_INLINE + void write_characters(const CharType* s, std::size_t length) + { + if (JSON_HEDLEY_UNLIKELY(v->size() - position < length)) + { + if (length >= max_reserve_size) + { + append(s, length); + return; + } + grow(length); + } + if (length != 0) + { + std::memcpy(v->data() + position, s, length * sizeof(CharType)); + position += length; + } + } + + private: + /// make room for at least @a length more characters after the position; + /// kept out of line so that the write functions stay small enough for + /// GCC to inline them into the writer + JSON_HEDLEY_NEVER_INLINE + void grow(const std::size_t length) + { + // use the capacity the vector already has, else at least double it; + // never less than 64 characters, so that a small value does not + // reallocate for every few bytes + const std::size_t needed = position + length; + v->resize((std::max)(std::size_t{64}, needed <= v->capacity() + ? v->capacity() + : (std::max)(needed, v->capacity() * 2))); + } + + /// append @a length (at least max_reserve_size) characters that do not fit + /// the space ahead of the position: insert() copies them once into the + /// grown vector, where grow() would first zero the new space and then copy + /// over it - noticeable for a large string or binary value + JSON_HEDLEY_NEVER_INLINE + void append(const CharType* s, const std::size_t length) + { + v->resize(position); + if (v->capacity() - position < length) + { + // grow geometrically, with room for the writes that follow, so + // that the next grow() need not reallocate right away + v->reserve((std::max)(v->capacity() * 2, position + length + max_reserve_size)); + } + v->insert(v->end(), s, s + length); + position = v->size(); + } + + std::vector* v; + /// number of characters of *v written so far + std::size_t position; +}; + +/// output adapter for byte vectors +template> +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept : v(vec) {} - void write_character(CharType c) + void write_character(CharType c) override { v.push_back(c); } // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null - // pointer with length 0 for empty strings/binary values. Appending an empty - // range is a no-op; the type-erased path tolerates this via the (unattributed) - // virtual base, and the concrete sink must do the same. - void write_characters(const CharType* s, std::size_t length) + // pointer with length 0 for empty strings/binary values; appending an empty + // range is a no-op + void write_characters(const CharType* s, std::size_t length) override { v.insert(v.end(), s, s + length); } @@ -21713,33 +21828,6 @@ class output_vector_sink std::vector& v; }; -/// output adapter for byte vectors -/// -/// The appending itself lives in output_vector_sink; this class only adds the -/// virtual output_adapter_protocol interface on top of it, so both the -/// type-erased and the templated path share one implementation. -template> -class output_vector_adapter : public output_adapter_protocol -{ - public: - explicit output_vector_adapter(std::vector& vec) noexcept - : sink(vec) - {} - - void write_character(CharType c) override - { - sink.write_character(c); - } - - void write_characters(const CharType* s, std::size_t length) override - { - sink.write_characters(s, length); - } - - private: - output_vector_sink sink; -}; - #ifndef JSON_NO_IO /// output adapter for output streams template diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp index b780dc0b5..cad77172e 100644 --- a/tests/src/unit-binary_writer_sinks.cpp +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -35,6 +35,17 @@ std::vector test_values() big_object[std::to_string(i)] = i; } + // large strings and binary values between small ones: a write too large + // for the space the vector sink has ahead takes a different path than a + // small one, and the small writes after it must still land behind it + json large_payloads = json::array(); + for (int i = 0; i < 50; ++i) + { + large_payloads.push_back(std::string(300, static_cast('a' + (i % 26)))); + large_payloads.push_back(i); + large_payloads.push_back(json::binary(std::vector(static_cast(100 * i), static_cast(i)))); + } + return { json(nullptr), json(true), json(false), @@ -46,7 +57,8 @@ std::vector test_values() json::array(), json::object(), json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), - big_array, big_object + big_array, big_object, large_payloads, + json::binary(std::vector(5000, 0x42)) }; }