From a50c2537ebf639570c6bbae0a7c9543e3c44f9da Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 12 Sep 2026 21:09:33 +0200 Subject: [PATCH] Reserve capped array capacity in json_sax_dom_parser::start_array() for definite-length binary arrays (#5476) * Reserve capped array capacity for definite-length binary arrays CBOR, MessagePack, and the optimized [$type#count UBJSON/BJData form all pass an exact element count to sax->start_array(len), but json_sax_dom_parser::start_array() (and the callback variant) only used len for an overflow check against max_size() and never reserved the underlying vector, so each element triggered a reallocation cascade via emplace_back(). Reserve upfront, but cap the reservation at 16384 elements: max_size() for a std::vector is far larger than any realistic input, so an unbounded reserve(len) would let a crafted/truncated header (e.g. CBOR 0x9A + a huge uint32 count with no data) trigger a multi-gigabyte allocation attempt instead of the normal graceful parse_error. With the cap, a hostile length still fails fast with the existing parse_error, while realistic arrays get a single up-front allocation. Signed-off-by: Niels Lohmann * Make the huge-claimed-length DoS regression tests portable across size_t widths On a platform where size_t is narrower than 64 bits (e.g. 32-bit mingw/msvc x86), the previously-hardcoded huge test lengths either collide with that platform's unknown_size() sentinel (CBOR/MessagePack, both using exactly SIZE_MAX) or exceed the platform's smaller vector::max_size() (UBJSON/BJData's 0x7FFFFFFF), so the header is now rejected outright (out_of_range.408) instead of being accepted and only found short of data (parse_error.110). Both are safe, bounded rejections of the hostile input; the property under test -- no attempt to allocate space for billions of elements -- holds either way. Accept both outcomes instead of pinning the 64-bit-only exact result. Also fixed an unrelated clang-tidy finding (google-readability-casting) on the functional-style std::size_t(...) casts in the neighboring "arrays of various sizes" section. Signed-off-by: Niels Lohmann * Fix remaining CI failures in the huge-claimed-length DoS regression tests - Apply the same google-readability-casting fix (std::size_t{N} instead of std::size_t(N)) to the "arrays of various sizes" section in unit-msgpack.cpp, unit-ubjson.cpp, and unit-bjdata.cpp; only unit-cbor.cpp had been fixed previously, since clang-tidy's build didn't get far enough to report the other three in the same pass. - json_sax_dom_parser::start_array()'s max_size() check calls JSON_THROW directly rather than going through sax->parse_error(), so unlike the scanner's own "not enough data" parse_error it is not gated by allow_exceptions=false. On a platform where a header's claimed count exceeds max_size() (e.g. 32-bit, for UBJSON/BJData's 0x7FFFFFFF test value), from_ubjson/from_bjdata(input, true, false) can therefore still throw instead of returning a discarded value. Make that assertion tolerant of either outcome, same as the main exception-catching check above it. - Guard all four "a huge claimed length..." SECTIONs with #if !defined(JSON_NOEXCEPTION), matching this test suite's existing convention for exception-dependent tests: under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ exception at all (it aborts the process), so a section that relies on try/catch to distinguish between two acceptable outcomes cannot be expressed under that build configuration regardless of platform. Signed-off-by: Niels Lohmann * Use (std::min)(len, reserve_cap) instead of a ternary in start_array() Addresses review feedback from @gregmarr on PR #5476. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 21 ++++ single_include/nlohmann/json.hpp | 21 ++++ tests/src/unit-bjdata.cpp | 105 ++++++++++++++++++++ tests/src/unit-cbor.cpp | 86 +++++++++++++++++ tests/src/unit-msgpack.cpp | 85 +++++++++++++++++ tests/src/unit-ubjson.cpp | 106 +++++++++++++++++++++ 6 files changed, 424 insertions(+) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 1627d2326..60c468f30 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,6 +8,7 @@ #pragma once +#include // min #include #include // string #include // enable_if_t @@ -305,6 +306,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } + return true; } @@ -683,6 +694,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } } return true; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3b2233a57..6eb006b68 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7892,6 +7892,7 @@ NLOHMANN_JSON_NAMESPACE_END +#include // min #include #include // string #include // enable_if_t @@ -11018,6 +11019,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } + return true; } @@ -11396,6 +11407,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } } return true; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 3130bb720..9338e663e 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3551,6 +3551,111 @@ TEST_CASE("BJData") } } +TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no + // element data at all. max_size() for a std::vector is far larger + // than this count, so it does not reject the header outright; the + // (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_bjdata(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_bjdata(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_bjdata(j); + CHECK(json::from_bjdata(packed_plain) == j); + + const auto packed_optimized = json::to_bjdata(j, true, true); + CHECK(json::from_bjdata(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_bjdata(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 032e6641b..4c9107517 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2174,6 +2174,92 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") } } +TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's detail::unknown_size() sentinel (SIZE_MAX), so the + // format-level size check rejects it outright (out_of_range.408, + // "excessive ... size") before the SAX consumer's own max_size() + // check would even run; on a 64-bit platform it passes both of + // those checks and is only found short of data once the (capped) + // reservation looks for element bytes that were never provided + // (parse_error.110). Either is an acceptable, bounded rejection of + // the hostile header -- the property under test is that no path + // attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_cbor(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_cbor(j); + CHECK(json::from_cbor(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_cbor(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor)); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 75c2ae464..74f7f4969 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1597,6 +1597,91 @@ TEST_CASE("MessagePack") } } +TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's SIZE_MAX, which some size-narrowing checks treat the + // same as detail::unknown_size(); it may then be rejected before + // the SAX consumer's own max_size() check (out_of_range.408) rather + // than being accepted and only found short of data once the + // (capped) reservation looks for element bytes that were never + // provided (parse_error.110). Either is an acceptable, bounded + // rejection of the hostile header -- the property under test is + // that no path attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_msgpack(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_msgpack(j); + CHECK(json::from_msgpack(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_msgpack(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack)); + } +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("MessagePack nesting does not consume the call stack") { diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 2ebb55b90..aafbbf5a4 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2315,6 +2315,112 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } +TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all. + // max_size() for a std::vector is far larger than this count, so it + // does not reject the header outright; the (capped) reservation must + // not attempt to allocate space for billions of elements before the + // missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_ubjson(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_ubjson(j); + CHECK(json::from_ubjson(packed_plain) == j); + + const auto packed_optimized = json::to_ubjson(j, true, true); + CHECK(json::from_ubjson(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_ubjson(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson)); + } +} + + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value")