amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-07-23 02:44:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reserve output capacity up front for binary serialization

Browse Source 
The vector-returning to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson grew
the output buffer purely by geometric reallocation. Reserving an estimate
up front avoids the early reallocations, which is the dominant per-byte
cost for array/object-heavy output.

The estimate (binary_reserve_hint) is deliberately conservative and safe
against untrusted input: it consults only the top-level element count
(O(1), no walk of the DOM), guards the multiplication against overflow,
and clamps the result to a fixed 1 MiB ceiling, so a large or hostile DOM
can never force an oversized allocation here. The buffer still grows
geometrically past the hint, so an underestimate only costs a few later
reallocations; scalars/strings/binary are written in one shot and get no
hint. Reserving capacity does not change the bytes produced.

Throughput (g++/clang -O3, vs the previous commit):
  cbor int array     +10% / +13%
  cbor object array  +20% / +38%

Output is byte-for-byte identical to develop across the binary
differential corpus.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-07-22 11:01:03 +00:00
co-authored by Claude Opus 4.8
parent ac6b505923
commit 9ea2d28ef9
3 changed files with 68 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/output/binary_writer.hpp
+29
View File
@@ -39,6 +39,35 @@ enum class bjdata_version_t
// binary writer //
///////////////////
/*!
@brief conservative capacity hint for binary serialization into a std::vector
Returns an approximate number of bytes to reserve up front so that serializing
an array/object of many elements does not repeatedly reallocate the output
buffer. Only the top-level element count is consulted (O(1), no walk of the
DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can
therefore never trigger an oversized allocation here, and the multiplication
cannot overflow. The buffer still grows geometrically beyond the hint, so a hint
that is too small only costs a few later reallocations. A single scalar, string,
or binary value is written in one shot and needs no hint.
*/
template<typename BasicJsonType>
std::size_t binary_reserve_hint(const BasicJsonType& j)
{
constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB
if (j.is_array() || j.is_object())
{
const std::size_t elements = j.size();
// guard the multiplication against overflow and cap the reservation
if (elements > max_hint / 4)
{
return max_hint;
}
return (elements * 4) + 2;
}
return 0;
}
/*!
@brief serialization to CBOR and MessagePack values
*/
include/nlohmann/json.hpp
+5
View File
@@ -4327,6 +4327,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_cbor(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j);
return result;
@@ -4351,6 +4352,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_msgpack(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j);
return result;
@@ -4377,6 +4379,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool use_type = false)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type);
return result;
@@ -4406,6 +4409,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bjdata_version_t version = bjdata_version_t::draft2)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version);
return result;
@@ -4434,6 +4438,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_bson(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_bson(j);
return result;
single_include/nlohmann/json.hpp
+34
View File
@@ -16853,6 +16853,35 @@ enum class bjdata_version_t
// binary writer //
///////////////////
/*!
@brief conservative capacity hint for binary serialization into a std::vector
Returns an approximate number of bytes to reserve up front so that serializing
an array/object of many elements does not repeatedly reallocate the output
buffer. Only the top-level element count is consulted (O(1), no walk of the
DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can
therefore never trigger an oversized allocation here, and the multiplication
cannot overflow. The buffer still grows geometrically beyond the hint, so a hint
that is too small only costs a few later reallocations. A single scalar, string,
or binary value is written in one shot and needs no hint.
*/
template<typename BasicJsonType>
std::size_t binary_reserve_hint(const BasicJsonType& j)
{
constexpr std::size_t max_hint = static_cast<std::size_t>(1) << 20; // 1 MiB
if (j.is_array() || j.is_object())
{
const std::size_t elements = j.size();
// guard the multiplication against overflow and cap the reservation
if (elements > max_hint / 4)
{
return max_hint;
}
return (elements * 4) + 2;
}
return 0;
}
/*!
@brief serialization to CBOR and MessagePack values
*/
@@ -25571,6 +25600,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_cbor(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j);
return result;
@@ -25595,6 +25625,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_msgpack(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j);
return result;
@@ -25621,6 +25652,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bool use_type = false)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type);
return result;
@@ -25650,6 +25682,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
const bjdata_version_t version = bjdata_version_t::draft2)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version);
return result;
@@ -25678,6 +25711,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
static std::vector<std::uint8_t> to_bson(const basic_json& j)
{
std::vector<std::uint8_t> result;
result.reserve(detail::binary_reserve_hint(j));
detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
detail::output_vector_sink<std::uint8_t>(result)).write_bson(j);
return result;