diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 1cc3e10c9..4845b73ca 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -563,9 +563,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -757,37 +760,37 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_container(/*is_object*/false, + conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/false, static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/false, static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_container(/*is_object*/false, size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_container(/*is_object*/false, size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_container(/*is_object*/false, detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -814,36 +817,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_container(/*is_object*/true, conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/true, static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/true, static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_container(/*is_object*/true, size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_container(/*is_object*/true, size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_container(/*is_object*/true, detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -927,7 +930,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -977,7 +983,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -1375,96 +1385,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // the reference is not held across parse_cbor_value() below, + // which can push onto the stack and reallocate it + container_frame& top = container_stack.back(); + bool at_end; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --top.remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1a9a9a092..0dba23eed 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11250,9 +11250,12 @@ class binary_reader @return whether a valid CBOR value was passed to the SAX parser */ - bool parse_cbor_internal(const bool get_char, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_value(const bool get_char, + const cbor_tag_handler_t tag_handler, + bool& tag_pending) { + tag_pending = false; + switch (get_char ? get() : current) { // EOF @@ -11444,37 +11447,37 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return get_cbor_array( - conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_container(/*is_object*/false, + conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/false, static_cast(len)); } case 0x99: // array (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_array(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/false, static_cast(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_container(/*is_object*/false, size); } case 0x9B: // array (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && get_cbor_array(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "array") && enter_container(/*is_object*/false, size); } case 0x9F: // array (indefinite length) - return get_cbor_array(detail::unknown_size(), tag_handler); + return enter_container(/*is_object*/false, detail::unknown_size()); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -11501,36 +11504,36 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - return get_cbor_object(conditional_static_cast(static_cast(current) & 0x1Fu), tag_handler); + return enter_container(/*is_object*/true, conditional_static_cast(static_cast(current) & 0x1Fu)); case 0xB8: // map (one-byte uint8_t for n follows) { std::uint8_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/true, static_cast(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { std::uint16_t len{}; - return get_number(input_format_t::cbor, len) && get_cbor_object(static_cast(len), tag_handler); + return get_number(input_format_t::cbor, len) && enter_container(/*is_object*/true, static_cast(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { std::uint32_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_container(/*is_object*/true, size); } case 0xBB: // map (eight-byte uint64_t for n follow) { std::uint64_t len{}; std::size_t size{}; - return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && get_cbor_object(size, tag_handler); + return get_number(input_format_t::cbor, len) && get_cbor_container_size(len, size, "map") && enter_container(/*is_object*/true, size); } case 0xBF: // map (indefinite length) - return get_cbor_object(detail::unknown_size(), tag_handler); + return enter_container(/*is_object*/true, detail::unknown_size()); case 0xC0: // tagged item case 0xC1: @@ -11614,7 +11617,10 @@ class binary_reader default: break; } - return parse_cbor_internal(true, tag_handler); + // the tagged value follows; it is read by the loop in + // parse_cbor_internal() rather than by recursing here + tag_pending = true; + return true; } case cbor_tag_handler_t::store: @@ -11664,7 +11670,11 @@ class binary_reader break; } default: - return parse_cbor_internal(true, tag_handler); + { + // as above, the tagged value is read by the caller + tag_pending = true; + return true; + } } get(); return get_cbor_binary(b) && sax->binary(b); @@ -12062,96 +12072,110 @@ class binary_reader } /*! - @param[in] len the length of the array or detail::unknown_size() for an - array of indefinite size + @brief read a CBOR value and everything nested inside it + + Reads values until the one that was begun here is complete, resuming the + enclosing container after each element, so that the nesting depth of the + input costs heap rather than native stack (see #5104). + + @param[in] get_char whether a new character should be retrieved from the + input (true) or whether the last read character + @a current should be considered instead @param[in] tag_handler how CBOR tags should be treated - @return whether array creation completed + + @return whether reading the value succeeded */ - bool get_cbor_array(const std::size_t len, - const cbor_tag_handler_t tag_handler) + bool parse_cbor_internal(const bool get_char, + const cbor_tag_handler_t tag_handler) { - if (JSON_HEDLEY_UNLIKELY(!sax->start_array(len))) - { - return false; - } + // whether the next value starts at a fresh byte or at the one already + // read into `current` + bool fetch = get_char; - if (len != detail::unknown_size()) + // the key currently being read; hoisted out of the loop so that its + // capacity is reused across elements and across nesting levels + string_t key; + + while (true) { - for (std::size_t i = 0; i < len; ++i) + if (!container_stack.empty()) { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) + // the reference is not held across parse_cbor_value() below, + // which can push onto the stack and reallocate it + container_frame& top = container_stack.back(); + bool at_end; + + if (top.remaining != npos) { - return false; + // definite length: the container ends once its elements + // have been read + at_end = (top.remaining == 0); + if (!at_end) + { + // claim the element about to be read + --top.remaining; + if (top.is_object) + { + get(); + } + } + fetch = true; } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(false, tag_handler))) + else { - return false; + // indefinite length: the container ends at a break marker. + // Testing for it consumes a byte, which is the first byte + // of the next element when it is not one. + at_end = (get() == 0xFF); + fetch = top.is_object; } - } - } - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or detail::unknown_size() for an - object of indefinite size - @param[in] tag_handler how CBOR tags should be treated - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len, - const cbor_tag_handler_t tag_handler) - { - if (JSON_HEDLEY_UNLIKELY(!sax->start_object(len))) - { - return false; - } - - if (len != 0) - { - string_t key; - if (len != detail::unknown_size()) - { - for (std::size_t i = 0; i < len; ++i) + if (at_end) { - get(); + const bool is_object = top.is_object; + container_stack.pop_back(); + if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array())) + { + return false; + } + // the value begun here is complete once its container is + if (container_stack.empty()) + { + return true; + } + continue; + } + + if (top.is_object) + { + key.clear(); if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) { return false; } - - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + fetch = true; } } - else - { - while (get() != 0xFF) - { - if (JSON_HEDLEY_UNLIKELY(!get_cbor_string(key) || !sax->key(key))) - { - return false; - } - if (JSON_HEDLEY_UNLIKELY(!parse_cbor_internal(true, tag_handler))) - { - return false; - } - key.clear(); + // a tag is not a value of its own: read on until the tagged value + bool tag_pending; + do + { + if (JSON_HEDLEY_UNLIKELY(!parse_cbor_value(fetch, tag_handler, tag_pending))) + { + return false; } + fetch = true; + } + while (tag_pending); + + // a value that opened a container left it on the stack; one that + // did not, and that was not inside a container, was the whole value + if (container_stack.empty()) + { + return true; } } - - return sax->end_object(); } ///////////// diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 96b30e142..032e6641b 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2035,6 +2035,93 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel") } } +TEST_CASE("CBOR nesting does not consume the call stack") +{ + // Containers used to be read by calling back into the value reader once + // per element, and a tag by calling it for the tagged value, so the native + // call stack grew with the nesting depth of the input. Each of the three + // costs a single byte to encode -- 0x9F, 0x81 and 0xC2 -- so a payload of + // repeated bytes crashed the process (#5104). The containers are kept on a + // heap stack now, and a tag is read in a loop. + // + // Deeply nested values must not be compared, copied or dumped here: those + // operations are still recursive and would reintroduce the crash. + json _; + + SECTION("indefinite-length containers") + { + const std::vector input(500000, 0x9F); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("definite-length containers") + { + const std::vector input(500000, 0x81); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } + + SECTION("tags") + { + // a tag is not a value of its own, so a chain of them used to recurse + const std::vector input(500000, 0xC2); + CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, json::cbor_tag_handler_t::ignore), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&); + CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded()); + } + + SECTION("a well-formed deep value is read through the SAX interface") + { + std::vector input(200000, 0x9F); + input.insert(input.end(), 200000, 0xFF); + + SaxCountdown accept_all(1000000); + CHECK(json::sax_parse(input, &accept_all, json::input_format_t::cbor)); + } + + SECTION("a well-formed deep value is read into a value") + { + const std::size_t depth = 10000; + std::vector input(depth, 0x81); + input.push_back(0x00); + + json j = json::from_cbor(input); + + std::size_t measured = 0; + const json* p = &j; + while (p->is_array() && !p->empty()) + { + p = &p->front(); + ++measured; + } + CHECK(measured == depth); + CHECK(p->is_number()); + } + + SECTION("containers are still read the same way") + { + CHECK(json::from_cbor(std::vector({0x80})) == json::array()); + CHECK(json::from_cbor(std::vector({0xA0})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0xFF})) == json::array()); + CHECK(json::from_cbor(std::vector({0xBF, 0xFF})) == json::object()); + CHECK(json::from_cbor(std::vector({0x9F, 0x01, 0x02, 0xFF})) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xBF, 0x61, 'a', 0x01, 0xFF})) == json({{"a", 1}})); + // definite and indefinite forms nested inside each other + CHECK(json::from_cbor(std::vector({0x9F, 0x82, 0x01, 0x02, 0xA1, 0x61, 'k', 0xBF, 0xFF, 0xFF})) == json({{1, 2}, {{"k", json::object()}}})); + } + + SECTION("tagged values are still read the same way") + { + const auto ignore = json::cbor_tag_handler_t::ignore; + CHECK(json::from_cbor(std::vector({0xC2, 0x01}), true, true, ignore) == json(1)); + // a chain of tags resolves to the value that follows it + CHECK(json::from_cbor(std::vector({0xC2, 0xC2, 0xC2, 0x01}), true, true, ignore) == json(1)); + // a tag inside a container, and one in front of a container + CHECK(json::from_cbor(std::vector({0x82, 0xC2, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + CHECK(json::from_cbor(std::vector({0xC2, 0x82, 0x01, 0x02}), true, true, ignore) == json({1, 2})); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself