Compare commits

..
Author SHA1 Message Date
Niels Lohmann e59b455616 Update doc example outputs for new object-conversion iteration order
Reserving capacity in from_json()'s object-conversion path before
inserting elements changes libstdc++'s std::unordered_map bucket
layout, which changes the iteration order used by
get__ValueType_const.cpp, get_to.cpp and operator__ValueType.cpp to
print the elements of a converted std::unordered_map<std::string,
json>. Verified against a clean develop checkout (built with the
same GCC/libstdc++ used in CI) that the old order was produced
without this PR's change and the new order is produced with it, and
that the three affected examples now match their updated expected
output byte-for-byte.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-06 11:34:53 +02:00
Niels Lohmann e29ac4f043 Reserve capacity in from_json() object conversion when supported
The object-to-container from_json() overload filled the target
container one element at a time without reserving capacity, even
when the target type supports reserve() (e.g. std::unordered_map)
and the number of elements is already known. This caused unnecessary
rehashing while parsing large objects into such containers.

Add a reserve-detecting overload (from_json_object_impl), mirroring
the priority_tag-based SFINAE technique already used by the array
conversion path (from_json_array_impl), so that reserve(size()) is
called up front when available and the loop falls back unchanged
otherwise (e.g. for std::map).

Fixes #5406

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-05 20:43:23 +02:00
11 changed files with 92 additions and 438 deletions
@@ -4,8 +4,8 @@
Hello, world!
1 2 3 4 5
string: "Hello, world!"
number: {"floating-point":17.23,"integer":42}
null: null
string: "Hello, world!"
boolean: true
array: [1,2,3,4,5]
+1 -1
View File
@@ -4,8 +4,8 @@
Hello, world!
1 2 3 4 5
string: "Hello, world!"
number: {"floating-point":17.23,"integer":42}
null: null
string: "Hello, world!"
boolean: true
array: [1,2,3,4,5]
@@ -4,9 +4,9 @@
Hello, world!
1 2 3 4 5
string: "Hello, world!"
number: {"floating-point":17.23,"integer":42}
null: null
string: "Hello, world!"
boolean: true
array: [1,2,3,4,5]
[json.exception.type_error.302] type must be boolean, but is string
@@ -398,6 +398,34 @@ inline void from_json(const BasicJsonType& j, CompatibleArrayType& bin)
}
}
template<typename BasicJsonType, typename ConstructibleObjectType>
auto from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<1> /*unused*/)
-> decltype(
obj.reserve(std::declval<typename ConstructibleObjectType::size_type>()),
void())
{
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
ret.reserve(inner_object->size());
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
}
template<typename BasicJsonType, typename ConstructibleObjectType>
inline void from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<0> /*unused*/)
{
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
}
template<typename BasicJsonType, typename ConstructibleObjectType,
enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
@@ -407,13 +435,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
JSON_THROW(type_error::create(302, concat("type must be object, but is ", j.type_name()), &j));
}
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
from_json_object_impl(j, obj, priority_tag<1> {});
}
// overload for arithmetic types, not chosen for basic_json template arguments
@@ -301,16 +301,6 @@ class json_sax_dom_parser
JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
}
if (len != detail::unknown_size())
{
// reserve upfront to avoid repeated reallocations while adding elements,
// but cap the reservation so a bogus/hostile length (which is not bounded
// by max_size(), unlike e.g. std::vector) cannot trigger an oversized
// allocation for a small or truncated input
constexpr std::size_t reserve_cap = 16384;
ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
}
return true;
}
@@ -671,16 +661,6 @@ class json_sax_dom_callback_parser
{
JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
}
if (len != detail::unknown_size())
{
// reserve upfront to avoid repeated reallocations while adding elements,
// but cap the reservation so a bogus/hostile length (which is not bounded
// by max_size(), unlike e.g. std::vector) cannot trigger an oversized
// allocation for a small or truncated input
constexpr std::size_t reserve_cap = 16384;
ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
}
}
return true;
+29 -27
View File
@@ -5693,6 +5693,34 @@ inline void from_json(const BasicJsonType& j, CompatibleArrayType& bin)
}
}
template<typename BasicJsonType, typename ConstructibleObjectType>
auto from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<1> /*unused*/)
-> decltype(
obj.reserve(std::declval<typename ConstructibleObjectType::size_type>()),
void())
{
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
ret.reserve(inner_object->size());
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
}
template<typename BasicJsonType, typename ConstructibleObjectType>
inline void from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<0> /*unused*/)
{
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
}
template<typename BasicJsonType, typename ConstructibleObjectType,
enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
@@ -5702,13 +5730,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
JSON_THROW(type_error::create(302, concat("type must be object, but is ", j.type_name()), &j));
}
ConstructibleObjectType ret;
const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
for (const auto& p : *inner_object)
{
ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
}
obj = std::move(ret);
from_json_object_impl(j, obj, priority_tag<1> {});
}
// overload for arithmetic types, not chosen for basic_json template arguments
@@ -9829,16 +9851,6 @@ class json_sax_dom_parser
JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
}
if (len != detail::unknown_size())
{
// reserve upfront to avoid repeated reallocations while adding elements,
// but cap the reservation so a bogus/hostile length (which is not bounded
// by max_size(), unlike e.g. std::vector) cannot trigger an oversized
// allocation for a small or truncated input
constexpr std::size_t reserve_cap = 16384;
ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
}
return true;
}
@@ -10199,16 +10211,6 @@ class json_sax_dom_callback_parser
{
JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
}
if (len != detail::unknown_size())
{
// reserve upfront to avoid repeated reallocations while adding elements,
// but cap the reservation so a bogus/hostile length (which is not bounded
// by max_size(), unlike e.g. std::vector) cannot trigger an oversized
// allocation for a small or truncated input
constexpr std::size_t reserve_cap = 16384;
ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap);
}
}
return true;
-105
View File
@@ -3489,111 +3489,6 @@ TEST_CASE("BJData")
}
}
TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays")
{
#if !defined(JSON_NOEXCEPTION)
// this SECTION relies on catching a thrown exception to distinguish
// which of two acceptable, bounded rejections a hostile header took;
// under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
// exception (it aborts instead), so this cannot be tested that way here
SECTION("a huge claimed length with no element data must not over-allocate")
{
// optimized form [$type#count: type 'i' (int8), count as a four-byte
// little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no
// element data at all. max_size() for a std::vector is far larger
// than this count, so it does not reject the header outright; the
// (capped) reservation must not attempt to allocate space for
// billions of elements before the missing data is detected.
json _;
const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F};
// On a platform where std::vector<json>::max_size() is smaller than
// the claimed count (e.g. 32-bit, where max_size() is bounded by a
// 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
// check rejects the header outright (out_of_range.408, with the
// claimed count in the message) instead of accepting it and only
// finding it short of data once the (capped) reservation looks for
// element bytes that were never provided (parse_error.110). Either
// is an acceptable, bounded rejection of the hostile header -- the
// property under test is that no path attempts to allocate space
// for billions of elements.
bool threw = false;
try
{
_ = json::from_bjdata(input);
}
catch (const json::parse_error& e)
{
threw = true;
CHECK(e.id == 110);
CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input");
}
catch (const json::out_of_range& e)
{
threw = true;
CHECK(e.id == 408);
CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
}
CHECK(threw);
// json_sax_dom_parser::start_array()'s max_size() check (unlike the
// scanner's own parse_error path) throws unconditionally via
// JSON_THROW rather than going through sax->parse_error(), so it is
// not gated by allow_exceptions=false on a platform where this
// header hits that check (e.g. 32-bit, see above) -- allow either
// a discarded result or the same out_of_range it throws with
// exceptions enabled.
try
{
CHECK(json::from_bjdata(input, true, false).is_discarded());
}
catch (const json::out_of_range& e)
{
CHECK(e.id == 408);
}
}
#endif
SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
{
for (const auto size :
{
std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
std::size_t{16384}, // exactly at the reserve cap
std::size_t{20000} // above the reserve cap
})
{
CAPTURE(size)
json j = json::array();
for (std::size_t i = 0; i < size; ++i)
{
j.push_back(static_cast<int>(i % 1000));
}
// exercise both the plain and the optimized [$type#count encoding
const auto packed_plain = json::to_bjdata(j);
CHECK(json::from_bjdata(packed_plain) == j);
const auto packed_optimized = json::to_bjdata(j, true, true);
CHECK(json::from_bjdata(packed_optimized) == j);
}
}
SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
{
// the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
// a custom SAX consumer that does not touch a DOM array sees identical events
json j = json::array();
for (int i = 0; i < 100; ++i)
{
j.push_back(i);
}
const auto packed = json::to_bjdata(j, true, true);
SaxCountdown scp(1000000); // large enough to never trigger an abort
CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata));
}
}
TEST_CASE("Universal Binary JSON Specification Examples 1")
{
SECTION("Null Value")
-86
View File
@@ -2035,92 +2035,6 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel")
}
}
TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays")
{
#if !defined(JSON_NOEXCEPTION)
// this SECTION relies on catching a thrown exception to distinguish
// which of two acceptable, bounded rejections a hostile header took;
// under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
// exception (it aborts instead), so this cannot be tested that way here
SECTION("a huge claimed length with no element data must not over-allocate")
{
// 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295)
// elements but provides none. max_size() for a std::vector is far
// larger than this count, so it does not reject the header outright;
// the (capped) reservation must not attempt to allocate space for
// billions of elements before the missing data is detected.
json _;
const std::vector<uint8_t> input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
// On a platform where std::size_t is narrower than 64 bits (e.g.
// 32-bit), the claimed count 0xFFFFFFFF coincides with that
// platform's detail::unknown_size() sentinel (SIZE_MAX), so the
// format-level size check rejects it outright (out_of_range.408,
// "excessive ... size") before the SAX consumer's own max_size()
// check would even run; on a 64-bit platform it passes both of
// those checks and is only found short of data once the (capped)
// reservation looks for element bytes that were never provided
// (parse_error.110). Either is an acceptable, bounded rejection of
// the hostile header -- the property under test is that no path
// attempts to allocate space for billions of elements.
bool threw = false;
try
{
_ = json::from_cbor(input);
}
catch (const json::parse_error& e)
{
threw = true;
CHECK(e.id == 110);
CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input");
}
catch (const json::out_of_range& e)
{
threw = true;
CHECK(e.id == 408);
CHECK(std::string(e.what()).find("excessive") != std::string::npos);
}
CHECK(threw);
CHECK(json::from_cbor(input, true, false).is_discarded());
}
#endif
SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
{
for (const auto size :
{
std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
std::size_t{16384}, // exactly at the reserve cap
std::size_t{20000} // above the reserve cap
})
{
CAPTURE(size)
json j = json::array();
for (std::size_t i = 0; i < size; ++i)
{
j.push_back(static_cast<int>(i % 1000));
}
const auto packed = json::to_cbor(j);
CHECK(json::from_cbor(packed) == j);
}
}
SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
{
// the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
// a custom SAX consumer that does not touch a DOM array sees identical events
json j = json::array();
for (int i = 0; i < 100; ++i)
{
j.push_back(i);
}
const auto packed = json::to_cbor(j);
SaxCountdown scp(1000000); // large enough to never trigger an abort
CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor));
}
}
TEST_CASE("CBOR roundtrips" * doctest::skip())
{
SECTION("input from flynn")
+31
View File
@@ -1389,6 +1389,37 @@ TEST_CASE("value conversion")
// CHECK(m5["one"] == "eins");
}
SECTION("reserve is called on containers that support it (#5406)")
{
// build a larger object so that a missing/incorrect reserve()
// call would be more likely to corrupt or drop elements
json j_large;
for (int i = 0; i < 100; ++i)
{
j_large[std::to_string(i)] = i;
}
SECTION("std::unordered_map (supports reserve)")
{
const auto m = j_large.get<std::unordered_map<std::string, int>>();
CHECK(m.size() == 100);
for (int i = 0; i < 100; ++i)
{
CHECK(m.at(std::to_string(i)) == i);
}
}
SECTION("std::map (no reserve, fallback path)")
{
const auto m = j_large.get<std::map<std::string, int>>();
CHECK(m.size() == 100);
for (int i = 0; i < 100; ++i)
{
CHECK(m.at(std::to_string(i)) == i);
}
}
}
SECTION("std::multimap")
{
j1.get<std::multimap<std::string, int>>();
-85
View File
@@ -1597,91 +1597,6 @@ TEST_CASE("MessagePack")
}
}
TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays")
{
#if !defined(JSON_NOEXCEPTION)
// this SECTION relies on catching a thrown exception to distinguish
// which of two acceptable, bounded rejections a hostile header took;
// under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
// exception (it aborts instead), so this cannot be tested that way here
SECTION("a huge claimed length with no element data must not over-allocate")
{
// 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295)
// elements but provides none. max_size() for a std::vector is far
// larger than this count, so it does not reject the header outright;
// the (capped) reservation must not attempt to allocate space for
// billions of elements before the missing data is detected.
json _;
const std::vector<uint8_t> input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF};
// On a platform where std::size_t is narrower than 64 bits (e.g.
// 32-bit), the claimed count 0xFFFFFFFF coincides with that
// platform's SIZE_MAX, which some size-narrowing checks treat the
// same as detail::unknown_size(); it may then be rejected before
// the SAX consumer's own max_size() check (out_of_range.408) rather
// than being accepted and only found short of data once the
// (capped) reservation looks for element bytes that were never
// provided (parse_error.110). Either is an acceptable, bounded
// rejection of the hostile header -- the property under test is
// that no path attempts to allocate space for billions of elements.
bool threw = false;
try
{
_ = json::from_msgpack(input);
}
catch (const json::parse_error& e)
{
threw = true;
CHECK(e.id == 110);
CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input");
}
catch (const json::out_of_range& e)
{
threw = true;
CHECK(e.id == 408);
CHECK(std::string(e.what()).find("excessive") != std::string::npos);
}
CHECK(threw);
CHECK(json::from_msgpack(input, true, false).is_discarded());
}
#endif
SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
{
for (const auto size :
{
std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
std::size_t{16384}, // exactly at the reserve cap
std::size_t{20000} // above the reserve cap
})
{
CAPTURE(size)
json j = json::array();
for (std::size_t i = 0; i < size; ++i)
{
j.push_back(static_cast<int>(i % 1000));
}
const auto packed = json::to_msgpack(j);
CHECK(json::from_msgpack(packed) == j);
}
}
SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
{
// the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
// a custom SAX consumer that does not touch a DOM array sees identical events
json j = json::array();
for (int i = 0; i < 100; ++i)
{
j.push_back(i);
}
const auto packed = json::to_msgpack(j);
SaxCountdown scp(1000000); // large enough to never trigger an abort
CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack));
}
}
// use this testcase outside [hide] to run it with Valgrind
TEST_CASE("single MessagePack roundtrip")
{
-105
View File
@@ -2149,111 +2149,6 @@ TEST_CASE("UBJSON")
}
}
TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays")
{
#if !defined(JSON_NOEXCEPTION)
// this SECTION relies on catching a thrown exception to distinguish
// which of two acceptable, bounded rejections a hostile header took;
// under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++
// exception (it aborts instead), so this cannot be tested that way here
SECTION("a huge claimed length with no element data must not over-allocate")
{
// optimized form [$type#count: type 'i' (int8), count as a four-byte
// 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all.
// max_size() for a std::vector is far larger than this count, so it
// does not reject the header outright; the (capped) reservation must
// not attempt to allocate space for billions of elements before the
// missing data is detected.
json _;
const std::vector<uint8_t> input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
// On a platform where std::vector<json>::max_size() is smaller than
// the claimed count (e.g. 32-bit, where max_size() is bounded by a
// 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own
// check rejects the header outright (out_of_range.408, with the
// claimed count in the message) instead of accepting it and only
// finding it short of data once the (capped) reservation looks for
// element bytes that were never provided (parse_error.110). Either
// is an acceptable, bounded rejection of the hostile header -- the
// property under test is that no path attempts to allocate space
// for billions of elements.
bool threw = false;
try
{
_ = json::from_ubjson(input);
}
catch (const json::parse_error& e)
{
threw = true;
CHECK(e.id == 110);
CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input");
}
catch (const json::out_of_range& e)
{
threw = true;
CHECK(e.id == 408);
CHECK(std::string(e.what()).find("excessive array size") != std::string::npos);
}
CHECK(threw);
// json_sax_dom_parser::start_array()'s max_size() check (unlike the
// scanner's own parse_error path) throws unconditionally via
// JSON_THROW rather than going through sax->parse_error(), so it is
// not gated by allow_exceptions=false on a platform where this
// header hits that check (e.g. 32-bit, see above) -- allow either
// a discarded result or the same out_of_range it throws with
// exceptions enabled.
try
{
CHECK(json::from_ubjson(input, true, false).is_discarded());
}
catch (const json::out_of_range& e)
{
CHECK(e.id == 408);
}
}
#endif
SECTION("arrays of various sizes decode to the same value as before the reserve optimization")
{
for (const auto size :
{
std::size_t{0}, std::size_t{1}, std::size_t{5}, // small
std::size_t{16384}, // exactly at the reserve cap
std::size_t{20000} // above the reserve cap
})
{
CAPTURE(size)
json j = json::array();
for (std::size_t i = 0; i < size; ++i)
{
j.push_back(static_cast<int>(i % 1000));
}
// exercise both the plain and the optimized [$type#count encoding
const auto packed_plain = json::to_ubjson(j);
CHECK(json::from_ubjson(packed_plain) == j);
const auto packed_optimized = json::to_ubjson(j, true, true);
CHECK(json::from_ubjson(packed_optimized) == j);
}
}
SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization")
{
// the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser;
// a custom SAX consumer that does not touch a DOM array sees identical events
json j = json::array();
for (int i = 0; i < 100; ++i)
{
j.push_back(i);
}
const auto packed = json::to_ubjson(j, true, true);
SaxCountdown scp(1000000); // large enough to never trigger an abort
CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson));
}
}
TEST_CASE("Universal Binary JSON Specification Examples 1")
{
SECTION("Null Value")