diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 4845b73ca..a4e336f81 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -199,8 +199,12 @@ class binary_reader */ struct container_frame { - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker = 0; /// whether to close this container with end_object() or end_array() bool is_object = false; }; @@ -217,7 +221,8 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { @@ -226,6 +231,7 @@ class binary_reader container_frame frame; frame.remaining = len; + frame.type_marker = type_marker; frame.is_object = is_object; container_stack.push_back(frame); return true; @@ -2158,7 +2164,99 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // The reference is not held across get_ubjson_value() above, which + // can push onto the stack and reallocate it. + for (;;) + { + container_frame& top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --top.remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + else if (current != (top.is_object ? '}' : ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -2930,53 +3028,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(/*is_object*/false, size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_container(/*is_object*/false, detail::unknown_size()); } /*! @@ -2998,68 +3065,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_container(/*is_object*/true, size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_container(/*is_object*/true, detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 0dba23eed..5a5cd150f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10886,8 +10886,12 @@ class binary_reader */ struct container_frame { - /// number of elements that have not been read yet + /// number of elements that have not been read yet, or npos when the + /// container is not sized and ends at a marker instead std::size_t remaining = 0; + /// UBJSON/BJData: the type marker of an optimized container, so that + /// its elements are read without one of their own; 0 otherwise + char_int_type type_marker = 0; /// whether to close this container with end_object() or end_array() bool is_object = false; }; @@ -10904,7 +10908,8 @@ class binary_reader @return whether the SAX parser accepted the start event */ - bool enter_container(const bool is_object, const std::size_t len) + bool enter_container(const bool is_object, const std::size_t len, + const char_int_type type_marker = 0) { if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len))) { @@ -10913,6 +10918,7 @@ class binary_reader container_frame frame; frame.remaining = len; + frame.type_marker = type_marker; frame.is_object = is_object; container_stack.push_back(frame); return true; @@ -12845,7 +12851,99 @@ class binary_reader */ bool parse_ubjson_internal(const bool get_char = true) { - return get_ubjson_value(get_char ? get_ignore_noop() : current); + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + // the type marker of the value to read next + char_int_type prefix = get_char ? get_ignore_noop() : current; + + while (true) + { + const std::size_t depth = container_stack.size(); + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix))) + { + return false; + } + + // the value begun here is complete once it is not inside anything + if (container_stack.empty()) + { + return true; + } + + // a value was completed rather than a container opened; a + // container that ends at a marker needs the next byte to test + if (container_stack.size() == depth && container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + + // advance to the next element, closing the containers that ended. + // The reference is not held across get_ubjson_value() above, which + // can push onto the stack and reallocate it. + for (;;) + { + container_frame& top = container_stack.back(); + + if (top.remaining != npos) + { + if (top.remaining != 0) + { + --top.remaining; + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) + { + return false; + } + } + // an optimized container gives its elements no marker + prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop(); + break; + } + } + else if (current != (top.is_object ? '}' : ']')) + { + // a container that ends at a marker is never optimized, so + // every element carries its own marker; for an object the + // byte tested above is the first byte of the key + if (top.is_object) + { + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) + { + return false; + } + prefix = get_ignore_noop(); + } + else + { + prefix = current; + } + break; + } + + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + if (container_stack.empty()) + { + return true; + } + // the container that just ended was an element of the one + // below it, which may need the next byte for its own test + if (container_stack.back().remaining == npos) + { + get_ignore_noop(); + } + } + } } /*! @@ -13617,53 +13715,22 @@ class binary_reader exception_message(input_format, "excessive array size", "size"), nullptr)); } - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(/*is_object*/false, size_and_type.first, size_and_type.second))) { return false; } - if (size_and_type.second != 0) + if (size_and_type.second == 'N') { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; + // a no-op is not a value, so a container of them holds none; + // the declared size has already been passed to the SAX parser + container_stack.back().remaining = 0; } - while (current != ']') - { - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } + return true; } - return sax->end_array(); + return enter_container(/*is_object*/false, detail::unknown_size()); } /*! @@ -13685,68 +13752,12 @@ class binary_reader exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr)); } - string_t key; if (size_and_type.first != npos) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) - { - return false; - } - - while (current != '}') - { - if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } + return enter_container(/*is_object*/true, size_and_type.first, size_and_type.second); } - return sax->end_object(); + return enter_container(/*is_object*/true, detail::unknown_size()); } // Note, no reader for UBJSON binary types is implemented because they do diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 8534e62a2..e4f8c1ca1 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,111 @@ TEST_CASE("UBJSON") } } +TEST_CASE("UBJSON nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, so the native call stack grew with the nesting depth of the + // input. '[' alone opens a container, so a payload of repeated '[' crashed + // the process (#5104), as did the optimized forms, which reach the same + // path through a type or size annotation. The containers are kept on a + // heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("containers that end at a marker") + { + const std::vector input(500000, '['); + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a size") + { + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + input.push_back('['); + input.push_back('#'); + input.push_back('i'); + input.push_back(1); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("containers with a type and a size") + { + // '[' is a permitted optimized type in UBJSON, so each element of such + // a container is itself a container, read without a marker of its own + std::vector input; + for (std::size_t i = 0; i < 100000; ++i) + { + const std::vector level = {'[', '$', '[', '#', 'i', 1}; + input.insert(input.end(), level.begin(), level.end()); + } + CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(100000, '['); + input.insert(input.end(), 100000, ']'); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, '['); + input.insert(input.end(), depth, ']'); + + json j = json::from_ubjson(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + // the innermost array is empty, so the descent stops one level short + CHECK(measured == depth - 1); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_ubjson(std::vector({'[', ']'})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '}'})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 0})) == json::array()); + CHECK(json::from_ubjson(std::vector({'{', '#', 'i', 0})) == json::object()); + CHECK(json::from_ubjson(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2})); + CHECK(json::from_ubjson(std::vector({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}})); + // a no-op is not a value, so a container of them holds none + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + // sized and unsized forms nested inside one another + CHECK(json::from_ubjson(std::vector({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}})); + CHECK(json::from_ubjson(std::vector({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}})); + // an optimized container of containers + CHECK(json::from_ubjson(std::vector({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}})); + } + + SECTION("BJData containers are still read the same way") + { + // the ND-array wrapper and the binary shortcut are complete values, + // not containers the reader descends into + CHECK(json::from_bjdata(std::vector({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) == + json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}})); + CHECK(json::from_bjdata(std::vector({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2})); + CHECK(json::from_bjdata(std::vector({'[', '[', 'i', 1, ']', ']'})) == json({{1}})); + } +} + TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") { // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an