diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 252143978..35f56dec8 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -206,9 +206,14 @@ class binary_reader /// number of elements that have not been read yet, or npos when the /// container is not sized and ends at a marker instead std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; /// UBJSON/BJData: the type marker of an optimized container, so that /// its elements are read without one of their own; 0 otherwise char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -283,8 +288,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -292,22 +299,89 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + const container_frame& top = container_stack.back(); + const bool is_object = top.is_object; + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -419,12 +493,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -474,82 +548,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b8edda7ad..dae41b56d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10893,9 +10893,14 @@ class binary_reader /// number of elements that have not been read yet, or npos when the /// container is not sized and ends at a marker instead std::size_t remaining; + /// BSON: value of chars_read before this document's size prefix, which + /// check_bson_document_size() needs once the document has been read + std::size_t start_position = 0; /// UBJSON/BJData: the type marker of an optimized container, so that /// its elements are read without one of their own; 0 otherwise char_int_type type_marker; + /// BSON: the size this document declares, in bytes + std::int32_t declared_size = 0; /// whether to close this container with end_object() or end_array() bool is_object; }; @@ -10970,8 +10975,10 @@ class binary_reader @brief Reads in a BSON-object and passes it to the SAX-parser. @return whether a valid BSON-value was passed to the SAX parser */ - bool parse_bson_internal() + bool open_bson_document(const bool is_object) { + // recorded before the size prefix is read, because + // check_bson_document_size() measures the document from here const std::size_t document_start = chars_read; std::int32_t document_size{}; if (!get_number(input_format_t::bson, document_size)) @@ -10979,22 +10986,89 @@ class binary_reader return false; } - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size()))) + if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size()))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false))) + container_frame& frame = container_stack.back(); + frame.start_position = document_start; + frame.declared_size = document_size; + return true; + } + + /*! + @brief read a BSON document and everything nested inside it + + Reads elements until the document that was begun here is complete, + resuming the enclosing document each time an embedded one ends, so that + the nesting depth of the input costs heap rather than native stack + (see #5104). + + @return whether reading the document succeeded + */ + bool parse_bson_internal() + { + if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true))) { return false; } - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; - return sax->end_object(); + while (true) + { + const auto element_type = get(); + + if (element_type == 0) // end of the innermost document + { + const container_frame& top = container_stack.back(); + const bool is_object = top.is_object; + + if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size))) + { + return false; + } + + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the document begun here is complete once it is not inside one + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + key.clear(); + if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) + { + return false; + } + + // an array's elements are named "0", "1", ... in the wire format, + // and those names are not passed on + if (container_stack.back().is_object && !sax->key(key)) + { + return false; + } + + if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + } } /*! @@ -11106,12 +11180,12 @@ class binary_reader case 0x03: // object { - return parse_bson_internal(); + return open_bson_document(/*is_object*/true); } case 0x04: // array { - return parse_bson_array(); + return open_bson_document(/*is_object*/false); } case 0x05: // binary @@ -11161,82 +11235,7 @@ class binary_reader } } - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - - while (auto element_type = get()) - { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key))) - { - return false; - } - - if (!is_array && !sax->key(key)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - const std::size_t document_start = chars_read; - std::int32_t document_size{}; - if (!get_number(input_format_t::bson, document_size)) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size()))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size))) - { - return false; - } - - return sax->end_array(); - } ////////// // CBOR // diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 7896b9f18..d892393ad 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -1011,6 +1011,91 @@ TEST_CASE("BSON document size mismatch") } } +TEST_CASE("BSON nesting does not consume the call stack") +{ + // An embedded document or array used to be read by calling back into the + // document reader, so the native call stack grew with the nesting depth of + // the input (#5104). The open documents are kept on a heap stack now. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + + // A document nested deeply enough to have crashed. The bytes are built + // here rather than with to_bson(), because the writer still recurses once + // per level and would overflow the stack before the reader is ever + // reached. Every level is + // 0x03 'a' 0x00 0x00 + // so a level is eight bytes larger than the one it holds, and the sizes + // can be filled in from the outside in. + const std::size_t depth = 30000; + std::vector input; + input.reserve(5 + (8 * depth)); + for (std::size_t i = 0; i < depth; ++i) + { + const std::uint32_t size = static_cast(5 + (8 * (depth - i))); + input.push_back(static_cast(size & 0xFF)); + input.push_back(static_cast((size >> 8) & 0xFF)); + input.push_back(static_cast((size >> 16) & 0xFF)); + input.push_back(static_cast((size >> 24) & 0xFF)); + input.push_back(0x03); // embedded document + input.push_back('a'); + input.push_back(0x00); + } + // the innermost document is empty, then one terminator closes each level + input.insert(input.end(), {0x05, 0x00, 0x00, 0x00, 0x00}); + input.insert(input.end(), depth, 0x00); + + SECTION("a well-formed deep document is read through the SAX interface") + { + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::bson)); + } + + SECTION("a well-formed deep document is read into a value") + { + json j = json::from_bson(input); + + // walked rather than compared: comparing, copying or dumping a value + // this deep is still recursive + std::size_t measured = 0; + const json* q = &j; + while (q->is_object() && !q->empty()) + { + q = &q->begin().value(); + ++measured; + } + CHECK(measured == depth); + } + + SECTION("embedded documents and arrays are still read the same way") + { + const json values = {{"a", {{"b", {{"c", 1}}}}}}; + CHECK(json::from_bson(json::to_bson(values)) == values); + + const json array = {{"a", {1, 2, 3}}}; + CHECK(json::from_bson(json::to_bson(array)) == array); + + const json mixed = {{"a", {json{{"x", 1}}, json{{"y", 2}}}}}; + CHECK(json::from_bson(json::to_bson(mixed)) == mixed); + + CHECK(json::from_bson(json::to_bson(json::object())) == json::object()); + } + + SECTION("a size that does not match is still reported per document") + { + // the embedded document claims one byte too many + std::vector const bad = + { + 0x15, 0x00, 0x00, 0x00, 0x03, 'a', 0x00, + 0x0D, 0x00, 0x00, 0x00, 0x08, 'b', 0x00, 0x01, 0x00, + 0x00 + }; + json _; + CHECK_THROWS_AS(_ = json::from_bson(bad), json::parse_error&); + CHECK(json::from_bson(bad, true, false).is_discarded()); + } +} + TEST_CASE("BSON numerical data") { SECTION("number")