From aee9421883395ea886242b137a19dd78239b366b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 5 Sep 2026 20:51:47 +0200 Subject: [PATCH] Reserve capped array capacity for definite-length binary arrays CBOR, MessagePack, and the optimized [$type#count UBJSON/BJData form all pass an exact element count to sax->start_array(len), but json_sax_dom_parser::start_array() (and the callback variant) only used len for an overflow check against max_size() and never reserved the underlying vector, so each element triggered a reallocation cascade via emplace_back(). Reserve upfront, but cap the reservation at 16384 elements: max_size() for a std::vector is far larger than any realistic input, so an unbounded reserve(len) would let a crafted/truncated header (e.g. CBOR 0x9A + a huge uint32 count with no data) trigger a multi-gigabyte allocation attempt instead of the normal graceful parse_error. With the cap, a hostile length still fails fast with the existing parse_error, while realistic arrays get a single up-front allocation. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 20 ++++++++ single_include/nlohmann/json.hpp | 20 ++++++++ tests/src/unit-bjdata.cpp | 59 ++++++++++++++++++++++ tests/src/unit-cbor.cpp | 54 ++++++++++++++++++++ tests/src/unit-msgpack.cpp | 54 ++++++++++++++++++++ tests/src/unit-ubjson.cpp | 59 ++++++++++++++++++++++ 6 files changed, 266 insertions(+) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 8a98ee728..1efe3e493 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -301,6 +301,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap); + } + return true; } @@ -661,6 +671,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap); + } } return true; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 31bfb869d..dedc2a7ad 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9829,6 +9829,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap); + } + return true; } @@ -10189,6 +10199,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve(len < reserve_cap ? len : reserve_cap); + } } return true; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 7d0dd5ff2..00b3b62bb 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3489,6 +3489,65 @@ TEST_CASE("BJData") } } +TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") +{ + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no + // element data at all. max_size() for a std::vector is far larger + // than this count, so it does not reject the header outright; the + // (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(input), + "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input", + json::parse_error&); + CHECK(json::from_bjdata(input, true, false).is_discarded()); + } + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t(0), std::size_t(1), std::size_t(5), // small + std::size_t(16384), // exactly at the reserve cap + std::size_t(20000) // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_bjdata(j); + CHECK(json::from_bjdata(packed_plain) == j); + + const auto packed_optimized = json::to_bjdata(j, true, true); + CHECK(json::from_bjdata(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_bjdata(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 2a6bd41d7..ee5651b46 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2035,6 +2035,60 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel") } } +TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") +{ + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), + "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input", + json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t(0), std::size_t(1), std::size_t(5), // small + std::size_t(16384), // exactly at the reserve cap + std::size_t(20000) // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_cbor(j); + CHECK(json::from_cbor(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_cbor(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor)); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 4358cd464..bd83cb4b3 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1597,6 +1597,60 @@ TEST_CASE("MessagePack") } } +TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") +{ + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_msgpack(input), + "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input", + json::parse_error&); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t(0), std::size_t(1), std::size_t(5), // small + std::size_t(16384), // exactly at the reserve cap + std::size_t(20000) // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_msgpack(j); + CHECK(json::from_msgpack(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_msgpack(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack)); + } +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("single MessagePack roundtrip") { diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index b59ac6b38..779b59306 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,65 @@ TEST_CASE("UBJSON") } } +TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") +{ + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all. + // max_size() for a std::vector is far larger than this count, so it + // does not reject the header outright; the (capped) reservation must + // not attempt to allocate space for billions of elements before the + // missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), + "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", + json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t(0), std::size_t(1), std::size_t(5), // small + std::size_t(16384), // exactly at the reserve cap + std::size_t(20000) // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_ubjson(j); + CHECK(json::from_ubjson(packed_plain) == j); + + const auto packed_optimized = json::to_ubjson(j, true, true); + CHECK(json::from_ubjson(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_ubjson(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value")