From 7dc308d95277c993a4d4cd257d3c0b442470f176 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 6 Sep 2026 18:29:42 +0200 Subject: [PATCH] Read UBJSON and BJData containers without recursing per nesting level get_ubjson_array() and get_ubjson_object() read their elements by calling back into the value reader, which called them again for a nested container, so the native call stack grew with the nesting depth of the input. '[' alone opens a container, so half a million of them crashes the process before the input runs out (#5104). The optimized forms reach the same path through a size or type annotation, and in plain UBJSON '[' and '{' are permitted as the type of an optimized container, so "[$[#i\x01" repeated nests just as deeply at six bytes a level. Both readers now only open their container, and parse_ubjson_internal() loops: it closes the containers that have ended, claims the next element of the innermost one, reads its key when it is an object, and works out the marker of the value to read next. That last part is where the formats differ, and the loop follows what the four element loops used to do: - a sized, typed container gives its elements no marker of their own - a sized, untyped container reads one for each element - a container that ends at a marker has the byte already, from the test against ']' or '}'; for an object it is the first byte of the key The ND-array wrapper and the 'B' binary shortcut stay as they are. Both read a complete value rather than opening a container, and their elements are always scalars: BJData does not permit '[' or '{' as an optimized type, which is also why only plain UBJSON needed the type-marker case above. A container of no-ops keeps its behaviour of holding no elements while still announcing its declared size to the SAX parser, by opening it and then setting its count to zero. unit-ubjson and unit-bjdata pass unchanged, 1.39 million assertions between them, and a behaviour comparison against the previous commit over every container form -- sized, unsized, typed, untyped, empty, no-op, ND-array, binary, and the forms nested inside one another -- gives identical values, error codes, messages and byte offsets. 500,000 levels of each vector now report a parse error instead of crashing, and a well-formed 100,000-level value is read to completion. The driver costs about 3 % on parsing 60,000 small objects and one array of a million integers, for the reason given in the previous commit; reading the frame once per element rather than per branch halved what it cost before. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 209 +++++++++--------- single_include/nlohmann/json.hpp | 209 +++++++++--------- tests/src/unit-ubjson.cpp | 105 +++++++++ 3 files changed, 325 insertions(+), 198 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 4845b73ca..a4e336f81 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -199,8 +199,12 @@ class binary_reader */ struct container_frame { - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker = 0; /// whether to close this container with end_object() or end_array() bool is_object = false; }; @@ -217,7 +221,8 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { @@ -226,6 +231,7 @@ class binary_reader container_frame frame; frame.remaining = len; + frame.type_marker = type_marker; frame.is_object = is_object; container_stack.push_back(frame); return true; @@ -2158,7 +2164,99 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // The reference is not held across get_ubjson_value() above, which + // can push onto the stack and reallocate it. + for (;;) + { + container_frame& top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --top.remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + else if (current != (top.is_object ? '}' : ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -2930,53 +3028,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(/*is_object*/false, size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_container(/*is_object*/false, detail::unknown_size()); } /*! @@ -2998,68 +3065,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_container(/*is_object*/true, size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_container(/*is_object*/true, detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0dba23eed..5a5cd150f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10886,8 +10886,12 @@ class binary_reader */ struct container_frame { - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker = 0; /// whether to close this container with end_object() or end_array() bool is_object = false; }; @@ -10904,7 +10908,8 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { @@ -10913,6 +10918,7 @@ class binary_reader container_frame frame; frame.remaining = len; + frame.type_marker = type_marker; frame.is_object = is_object; container_stack.push_back(frame); return true; @@ -12845,7 +12851,99 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // The reference is not held across get_ubjson_value() above, which + // can push onto the stack and reallocate it. + for (;;) + { + container_frame& top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --top.remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + else if (current != (top.is_object ? '}' : ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -13617,53 +13715,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(/*is_object*/false, size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_container(/*is_object*/false, detail::unknown_size()); } /*! @@ -13685,68 +13752,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_container(/*is_object*/true, size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_container(/*is_object*/true, detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 8534e62a2..e4f8c1ca1 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,111 @@ TEST_CASE("UBJSON") } } +TEST_CASE("UBJSON nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, so the native call stack grew with the nesting depth of the + // input. '[' alone opens a container, so a payload of repeated '[' crashed + // the process (#5104), as did the optimized forms, which reach the same + // path through a type or size annotation. The containers are kept on a + // heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("containers that end at a marker") + { + const std::vector input(500000, '['); + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a size") + { + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + input.push_back('['); + input.push_back('#'); + input.push_back('i'); + input.push_back(1); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a type and a size") + { + // '[' is a permitted optimized type in UBJSON, so each element of such + // a container is itself a container, read without a marker of its own + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + const std::vector level = {'[', '$', '[', '#', 'i', 1}; + input.insert(input.end(), level.begin(), level.end()); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(100000, '['); + input.insert(input.end(), 100000, ']'); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, '['); + input.insert(input.end(), depth, ']'); + + json j = json::from_ubjson(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + // the innermost array is empty, so the descent stops one level short + CHECK(measured == depth - 1); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_ubjson(std::vector({'[', ']'})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '}'})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 0})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '#', 'i', 0})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}})); + // a no-op is not a value, so a container of them holds none + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + // sized and unsized forms nested inside one another + CHECK(json::from_ubjson(std::vector({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}})); + // an optimized container of containers + CHECK(json::from_ubjson(std::vector({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}})); + } + + SECTION("BJData containers are still read the same way") + { + // the ND-array wrapper and the binary shortcut are complete values, + // not containers the reader descends into + CHECK(json::from_bjdata(std::vector({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) == + json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}})); + CHECK(json::from_bjdata(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_bjdata(std::vector({'[', '[', 'i', 1, ']', ']'})) == json({{1}})); + } +} + TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") { // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an