diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index a355f1c15..69f9bc0bc 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -143,10 +143,26 @@ class binary_writer } /*! - @param[in] j JSON value to serialize + @param[in] j JSON value to serialize + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_cbor + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref binary_write_depth_limit + levels have been entered, @ref write_cbor_iterative writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5392 */ - void write_cbor(const BasicJsonType& j) + void write_cbor(const BasicJsonType& j, const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_cbor_iterative(j); + return; + } + switch (j.type()) { case value_t::null: @@ -320,39 +336,11 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); - if (N <= 0x17) - { - write_number(static_cast(0x80 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x98)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x99)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x9A)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x9B)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP + write_cbor_array_prefix(j.m_data.m_value.array->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.array) { - write_cbor(el); + write_cbor(el, depth + 1); } break; } @@ -422,40 +410,12 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); - if (N <= 0x17) - { - write_number(static_cast(0xA0 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xB8)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xB9)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xBA)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xBB)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP + write_cbor_object_prefix(j.m_data.m_value.object->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { write_cbor(el.first); - write_cbor(el.second); + write_cbor(el.second, depth + 1); } break; } @@ -467,10 +427,20 @@ class binary_writer } /*! - @param[in] j JSON value to serialize + @param[in] j JSON value to serialize + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_msgpack + + @sa @ref write_cbor + @sa https://github.com/nlohmann/json/issues/5392 */ - void write_msgpack(const BasicJsonType& j) + void write_msgpack(const BasicJsonType& j, const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_msgpack_iterative(j); + return; + } + switch (j.type()) { case value_t::null: // nil @@ -640,30 +610,11 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); - if (N <= 15) - { - // fixarray - write_number(static_cast(0x90 | N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 16 - oa.write_character(to_char_type(0xDC)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 32 - oa.write_character(to_char_type(0xDD)); - write_number(static_cast(N)); - } + write_msgpack_array_prefix(j.m_data.m_value.array->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.array) { - write_msgpack(el); + write_msgpack(el, depth + 1); } break; } @@ -758,31 +709,12 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); - if (N <= 15) - { - // fixmap - write_number(static_cast(0x80 | (N & 0xF))); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 16 - oa.write_character(to_char_type(0xDE)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 32 - oa.write_character(to_char_type(0xDF)); - write_number(static_cast(N)); - } + write_msgpack_object_prefix(j.m_data.m_value.object->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { write_msgpack(el.first); - write_msgpack(el.second); + write_msgpack(el.second, depth + 1); } break; } @@ -800,11 +732,22 @@ class binary_writer @param[in] add_prefix whether prefixes need to be used for this value @param[in] use_bjdata whether write in BJData format, default is false @param[in] bjdata_version which BJData version to use, default is draft2 + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_ubjson + + @sa @ref write_cbor + @sa https://github.com/nlohmann/json/issues/5392 */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, - const bool use_bjdata = false, const bjdata_version_t bjdata_version = bjdata_version_t::draft2) + const bool use_bjdata = false, const bjdata_version_t bjdata_version = bjdata_version_t::draft2, + const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_ubjson_iterative(j, use_count, use_type, add_prefix, use_bjdata, bjdata_version); + return; + } + const bool bjdata_draft3 = use_bjdata && bjdata_version == bjdata_version_t::draft3; switch (j.type()) @@ -862,57 +805,15 @@ class binary_writer case value_t::array: { - if (add_prefix) - { - oa.write_character(to_char_type('[')); - } - bool prefix_required = true; - if (use_type && !j.m_data.m_value.array->empty()) - { - if (!use_count) - { - JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); - } - const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); - const bool same_prefix = std::all_of(j.begin() + 1, j.end(), - [this, first_prefix, use_bjdata](const BasicJsonType & v) - { - return ubjson_prefix(v, use_bjdata) == first_prefix; - }); - - std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - - // an optimized array of a valueless type carries no payload, so a - // reader has nothing but the declared count to bound the allocation - // by and refuses an excessive one. Write the unoptimized form for - // those, at one byte per element, so the result can be read back. - // Objects are not affected: every element is preceded by its key. - const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); - const bool excessive_valueless = valueless_type - && j.m_data.m_value.array->size() > detail::max_valueless_container_size; - - if (same_prefix && !excessive_valueless - && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) - { - prefix_required = false; - oa.write_character(to_char_type('$')); - oa.write_character(first_prefix); - } - } - - if (use_count) - { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); - } + const bool write_closer = write_ubjson_start_array(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); } - if (!use_count) + if (write_closer) { oa.write_character(to_char_type(']')); } @@ -978,40 +879,8 @@ class binary_writer } } - if (add_prefix) - { - oa.write_character(to_char_type('{')); - } - bool prefix_required = true; - if (use_type && !j.m_data.m_value.object->empty()) - { - if (!use_count) - { - JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); - } - const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); - const bool same_prefix = std::all_of(j.begin(), j.end(), - [this, first_prefix, use_bjdata](const BasicJsonType & v) - { - return ubjson_prefix(v, use_bjdata) == first_prefix; - }); - - std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) - { - prefix_required = false; - oa.write_character(to_char_type('$')); - oa.write_character(first_prefix); - } - } - - if (use_count) - { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); - } + const bool write_closer = write_ubjson_start_object(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); for (const auto& el : *j.m_data.m_value.object) { @@ -1019,10 +888,10 @@ class binary_writer oa.write_characters( reinterpret_cast(el.first.data()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); } - if (!use_count) + if (write_closer) { oa.write_character(to_char_type('}')); } @@ -1037,6 +906,495 @@ class binary_writer } private: + /// the number of levels the recursive writers descend into before handing + /// over to the iterative writers below + static constexpr std::size_t binary_write_depth_limit() + { + return 128; + } + + void write_cbor_array_prefix(const std::size_t N) + { + // step 1: write control byte and the array size + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + } + + void write_cbor_object_prefix(const std::size_t N) + { + // step 1: write control byte and the object size + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + } + + /*! + @brief write out @a root and everything below it without the call stack + + Emits the same bytes as @ref write_cbor, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref binary_write_depth_limit, which is why it + is not written for speed. + */ + void write_cbor_iterative(const BasicJsonType& root) + { + // which of the two iterators is live follows from the type of value. + // They are kept side by side rather than in a union, which would need + // its special members written out by hand, see + // detail/iterators/internal_iterator.hpp + struct frame + { + explicit frame(const BasicJsonType* value_) noexcept + : value(value_) + , started(false) + {} + + const BasicJsonType* value; + bool started; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + if (!j.is_array() && !j.is_object()) + { + write_cbor(j); + stack.pop_back(); + continue; + } + + if (j.is_array()) + { + write_cbor_array_prefix(j.m_data.m_value.array->size()); + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + write_cbor_object_prefix(j.m_data.m_value.object->size()); + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (j.is_array()) + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + stack.pop_back(); + continue; + } + + // read the child before pushing: entering it can move every frame + const BasicJsonType* child = &(*current.array_it); + ++current.array_it; + stack.emplace_back(child); + } + else + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + stack.pop_back(); + continue; + } + + write_cbor(current.object_it->first); + const BasicJsonType* child = &(current.object_it->second); + ++current.object_it; + stack.emplace_back(child); + } + } + } + + void write_msgpack_array_prefix(const std::size_t N) + { + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa.write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa.write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + } + + void write_msgpack_object_prefix(const std::size_t N) + { + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa.write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa.write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + } + + /*! + @brief write out @a root and everything below it without the call stack + + @sa @ref write_cbor_iterative + */ + void write_msgpack_iterative(const BasicJsonType& root) + { + struct frame + { + explicit frame(const BasicJsonType* value_) noexcept + : value(value_) + , started(false) + {} + + const BasicJsonType* value; + bool started; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + if (!j.is_array() && !j.is_object()) + { + write_msgpack(j); + stack.pop_back(); + continue; + } + + if (j.is_array()) + { + write_msgpack_array_prefix(j.m_data.m_value.array->size()); + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + write_msgpack_object_prefix(j.m_data.m_value.object->size()); + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (j.is_array()) + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + stack.pop_back(); + continue; + } + + const BasicJsonType* child = &(*current.array_it); + ++current.array_it; + stack.emplace_back(child); + } + else + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + stack.pop_back(); + continue; + } + + write_msgpack(current.object_it->first); + const BasicJsonType* child = &(current.object_it->second); + ++current.object_it; + stack.emplace_back(child); + } + } + } + + /// @return true when a closing ']' still has to be written after the elements + bool write_ubjson_start_array(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, bool& prefix_required) + { + prefix_required = true; + if (add_prefix) + { + oa.write_character(to_char_type('[')); + } + + if (use_type && !j.m_data.m_value.array->empty()) + { + if (!use_count) + { + JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); + } + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix, use_bjdata](const BasicJsonType & v) + { + return ubjson_prefix(v, use_bjdata) == first_prefix; + }); + + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + { + prefix_required = false; + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); + } + } + + if (use_count) + { + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); + } + + return !use_count; + } + + /// @return true when a closing '}' still has to be written after the elements + bool write_ubjson_start_object(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, bool& prefix_required) + { + prefix_required = true; + if (add_prefix) + { + oa.write_character(to_char_type('{')); + } + + if (use_type && !j.m_data.m_value.object->empty()) + { + if (!use_count) + { + JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); + } + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix, use_bjdata](const BasicJsonType & v) + { + return ubjson_prefix(v, use_bjdata) == first_prefix; + }); + + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + { + prefix_required = false; + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); + } + } + + if (use_count) + { + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); + } + + return !use_count; + } + + /*! + @brief write out @a root and everything below it without the call stack + + @sa @ref write_cbor_iterative + */ + void write_ubjson_iterative(const BasicJsonType& root, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, const bjdata_version_t bjdata_version) + { + struct frame + { + frame(const BasicJsonType* value_, const bool add_prefix_) noexcept + : value(value_) + , add_prefix(add_prefix_) + , prefix_required(true) + , started(false) + , write_closer(false) + , is_object(false) + {} + + const BasicJsonType* value; + bool add_prefix; + bool prefix_required; + bool started; + bool write_closer; + bool is_object; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root, add_prefix); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + const bool this_prefix = stack.back().add_prefix; + if (!j.is_array() && !j.is_object()) + { + write_ubjson(j, use_count, use_type, this_prefix, use_bjdata, bjdata_version); + stack.pop_back(); + continue; + } + + if (j.is_object() && use_bjdata && j.m_data.m_value.object->size() == 3 && + j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && + j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && + j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()) + { + if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type, bjdata_version)) + { + stack.pop_back(); + continue; + } + } + + bool prefix_required = true; + if (j.is_array()) + { + stack.back().write_closer = write_ubjson_start_array(j, use_count, use_type, this_prefix, use_bjdata, prefix_required); + stack.back().is_object = false; + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + stack.back().write_closer = write_ubjson_start_object(j, use_count, use_type, this_prefix, use_bjdata, prefix_required); + stack.back().is_object = true; + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().prefix_required = prefix_required; + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (current.is_object) + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + if (current.write_closer) + { + oa.write_character(to_char_type('}')); + } + stack.pop_back(); + continue; + } + + write_number_with_ubjson_prefix(current.object_it->first.size(), true, use_bjdata); + oa.write_characters( + reinterpret_cast(current.object_it->first.data()), + current.object_it->first.size()); + const BasicJsonType* child = &(current.object_it->second); + const bool child_prefix = current.prefix_required; + ++current.object_it; + stack.emplace_back(child, child_prefix); + } + else + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + if (current.write_closer) + { + oa.write_character(to_char_type(']')); + } + stack.pop_back(); + continue; + } + + const BasicJsonType* child = &(*current.array_it); + const bool child_prefix = current.prefix_required; + ++current.array_it; + stack.emplace_back(child, child_prefix); + } + } + } + ////////// // BSON // ////////// diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4af4af1e0..d81b4d791 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18972,10 +18972,26 @@ class binary_writer } /*! - @param[in] j JSON value to serialize + @param[in] j JSON value to serialize + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_cbor + + Serializing a container descends into its elements, so a value nested deeply + enough used to exhaust the call stack and terminate the process with no + exception to catch. The descent is bounded here: once @ref binary_write_depth_limit + levels have been entered, @ref write_cbor_iterative writes out what is left + without the call stack. A value nested less deeply than that - all but a + vanishing minority - is written by exactly the code that always wrote it. + + @sa https://github.com/nlohmann/json/issues/5392 */ - void write_cbor(const BasicJsonType& j) + void write_cbor(const BasicJsonType& j, const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_cbor_iterative(j); + return; + } + switch (j.type()) { case value_t::null: @@ -19149,39 +19165,11 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); - if (N <= 0x17) - { - write_number(static_cast(0x80 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x98)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x99)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x9A)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0x9B)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP + write_cbor_array_prefix(j.m_data.m_value.array->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.array) { - write_cbor(el); + write_cbor(el, depth + 1); } break; } @@ -19251,40 +19239,12 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); - if (N <= 0x17) - { - write_number(static_cast(0xA0 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xB8)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xB9)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xBA)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa.write_character(to_char_type(0xBB)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP + write_cbor_object_prefix(j.m_data.m_value.object->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { write_cbor(el.first); - write_cbor(el.second); + write_cbor(el.second, depth + 1); } break; } @@ -19296,10 +19256,20 @@ class binary_writer } /*! - @param[in] j JSON value to serialize + @param[in] j JSON value to serialize + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_msgpack + + @sa @ref write_cbor + @sa https://github.com/nlohmann/json/issues/5392 */ - void write_msgpack(const BasicJsonType& j) + void write_msgpack(const BasicJsonType& j, const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_msgpack_iterative(j); + return; + } + switch (j.type()) { case value_t::null: // nil @@ -19469,30 +19439,11 @@ class binary_writer case value_t::array: { - // step 1: write control byte and the array size - const auto N = j.m_data.m_value.array->size(); - if (N <= 15) - { - // fixarray - write_number(static_cast(0x90 | N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 16 - oa.write_character(to_char_type(0xDC)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 32 - oa.write_character(to_char_type(0xDD)); - write_number(static_cast(N)); - } + write_msgpack_array_prefix(j.m_data.m_value.array->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.array) { - write_msgpack(el); + write_msgpack(el, depth + 1); } break; } @@ -19587,31 +19538,12 @@ class binary_writer case value_t::object: { - // step 1: write control byte and the object size - const auto N = j.m_data.m_value.object->size(); - if (N <= 15) - { - // fixmap - write_number(static_cast(0x80 | (N & 0xF))); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 16 - oa.write_character(to_char_type(0xDE)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 32 - oa.write_character(to_char_type(0xDF)); - write_number(static_cast(N)); - } + write_msgpack_object_prefix(j.m_data.m_value.object->size()); - // step 2: write each element for (const auto& el : *j.m_data.m_value.object) { write_msgpack(el.first); - write_msgpack(el.second); + write_msgpack(el.second, depth + 1); } break; } @@ -19629,11 +19561,22 @@ class binary_writer @param[in] add_prefix whether prefixes need to be used for this value @param[in] use_bjdata whether write in BJData format, default is false @param[in] bjdata_version which BJData version to use, default is draft2 + @param[in] depth nesting level of @a j, counted from the value passed to @ref write_ubjson + + @sa @ref write_cbor + @sa https://github.com/nlohmann/json/issues/5392 */ void write_ubjson(const BasicJsonType& j, const bool use_count, const bool use_type, const bool add_prefix = true, - const bool use_bjdata = false, const bjdata_version_t bjdata_version = bjdata_version_t::draft2) + const bool use_bjdata = false, const bjdata_version_t bjdata_version = bjdata_version_t::draft2, + const std::size_t depth = 0) { + if (JSON_HEDLEY_UNLIKELY(depth >= binary_write_depth_limit()) && (j.is_array() || j.is_object())) + { + write_ubjson_iterative(j, use_count, use_type, add_prefix, use_bjdata, bjdata_version); + return; + } + const bool bjdata_draft3 = use_bjdata && bjdata_version == bjdata_version_t::draft3; switch (j.type()) @@ -19691,57 +19634,15 @@ class binary_writer case value_t::array: { - if (add_prefix) - { - oa.write_character(to_char_type('[')); - } - bool prefix_required = true; - if (use_type && !j.m_data.m_value.array->empty()) - { - if (!use_count) - { - JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); - } - const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); - const bool same_prefix = std::all_of(j.begin() + 1, j.end(), - [this, first_prefix, use_bjdata](const BasicJsonType & v) - { - return ubjson_prefix(v, use_bjdata) == first_prefix; - }); - - std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - - // an optimized array of a valueless type carries no payload, so a - // reader has nothing but the declared count to bound the allocation - // by and refuses an excessive one. Write the unoptimized form for - // those, at one byte per element, so the result can be read back. - // Objects are not affected: every element is preceded by its key. - const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); - const bool excessive_valueless = valueless_type - && j.m_data.m_value.array->size() > detail::max_valueless_container_size; - - if (same_prefix && !excessive_valueless - && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) - { - prefix_required = false; - oa.write_character(to_char_type('$')); - oa.write_character(first_prefix); - } - } - - if (use_count) - { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); - } + const bool write_closer = write_ubjson_start_array(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); for (const auto& el : *j.m_data.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); } - if (!use_count) + if (write_closer) { oa.write_character(to_char_type(']')); } @@ -19807,40 +19708,8 @@ class binary_writer } } - if (add_prefix) - { - oa.write_character(to_char_type('{')); - } - bool prefix_required = true; - if (use_type && !j.m_data.m_value.object->empty()) - { - if (!use_count) - { - JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); - } - const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); - const bool same_prefix = std::all_of(j.begin(), j.end(), - [this, first_prefix, use_bjdata](const BasicJsonType & v) - { - return ubjson_prefix(v, use_bjdata) == first_prefix; - }); - - std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) - { - prefix_required = false; - oa.write_character(to_char_type('$')); - oa.write_character(first_prefix); - } - } - - if (use_count) - { - oa.write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); - } + const bool write_closer = write_ubjson_start_object(j, use_count, use_type, add_prefix, use_bjdata, prefix_required); for (const auto& el : *j.m_data.m_value.object) { @@ -19848,10 +19717,10 @@ class binary_writer oa.write_characters( reinterpret_cast(el.first.data()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version, depth + 1); } - if (!use_count) + if (write_closer) { oa.write_character(to_char_type('}')); } @@ -19866,6 +19735,495 @@ class binary_writer } private: + /// the number of levels the recursive writers descend into before handing + /// over to the iterative writers below + static constexpr std::size_t binary_write_depth_limit() + { + return 128; + } + + void write_cbor_array_prefix(const std::size_t N) + { + // step 1: write control byte and the array size + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + } + + void write_cbor_object_prefix(const std::size_t N) + { + // step 1: write control byte and the object size + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa.write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + } + + /*! + @brief write out @a root and everything below it without the call stack + + Emits the same bytes as @ref write_cbor, keeping the containers it has + entered on an explicit stack instead of descending into them. Only reached + for values nested deeper than @ref binary_write_depth_limit, which is why it + is not written for speed. + */ + void write_cbor_iterative(const BasicJsonType& root) + { + // which of the two iterators is live follows from the type of value. + // They are kept side by side rather than in a union, which would need + // its special members written out by hand, see + // detail/iterators/internal_iterator.hpp + struct frame + { + explicit frame(const BasicJsonType* value_) noexcept + : value(value_) + , started(false) + {} + + const BasicJsonType* value; + bool started; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + if (!j.is_array() && !j.is_object()) + { + write_cbor(j); + stack.pop_back(); + continue; + } + + if (j.is_array()) + { + write_cbor_array_prefix(j.m_data.m_value.array->size()); + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + write_cbor_object_prefix(j.m_data.m_value.object->size()); + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (j.is_array()) + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + stack.pop_back(); + continue; + } + + // read the child before pushing: entering it can move every frame + const BasicJsonType* child = &(*current.array_it); + ++current.array_it; + stack.emplace_back(child); + } + else + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + stack.pop_back(); + continue; + } + + write_cbor(current.object_it->first); + const BasicJsonType* child = &(current.object_it->second); + ++current.object_it; + stack.emplace_back(child); + } + } + } + + void write_msgpack_array_prefix(const std::size_t N) + { + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa.write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa.write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + } + + void write_msgpack_object_prefix(const std::size_t N) + { + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa.write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa.write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + } + + /*! + @brief write out @a root and everything below it without the call stack + + @sa @ref write_cbor_iterative + */ + void write_msgpack_iterative(const BasicJsonType& root) + { + struct frame + { + explicit frame(const BasicJsonType* value_) noexcept + : value(value_) + , started(false) + {} + + const BasicJsonType* value; + bool started; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + if (!j.is_array() && !j.is_object()) + { + write_msgpack(j); + stack.pop_back(); + continue; + } + + if (j.is_array()) + { + write_msgpack_array_prefix(j.m_data.m_value.array->size()); + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + write_msgpack_object_prefix(j.m_data.m_value.object->size()); + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (j.is_array()) + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + stack.pop_back(); + continue; + } + + const BasicJsonType* child = &(*current.array_it); + ++current.array_it; + stack.emplace_back(child); + } + else + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + stack.pop_back(); + continue; + } + + write_msgpack(current.object_it->first); + const BasicJsonType* child = &(current.object_it->second); + ++current.object_it; + stack.emplace_back(child); + } + } + } + + /// @return true when a closing ']' still has to be written after the elements + bool write_ubjson_start_array(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, bool& prefix_required) + { + prefix_required = true; + if (add_prefix) + { + oa.write_character(to_char_type('[')); + } + + if (use_type && !j.m_data.m_value.array->empty()) + { + if (!use_count) + { + JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); + } + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix, use_bjdata](const BasicJsonType & v) + { + return ubjson_prefix(v, use_bjdata) == first_prefix; + }); + + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + { + prefix_required = false; + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); + } + } + + if (use_count) + { + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); + } + + return !use_count; + } + + /// @return true when a closing '}' still has to be written after the elements + bool write_ubjson_start_object(const BasicJsonType& j, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, bool& prefix_required) + { + prefix_required = true; + if (add_prefix) + { + oa.write_character(to_char_type('{')); + } + + if (use_type && !j.m_data.m_value.object->empty()) + { + if (!use_count) + { + JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); + } + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix, use_bjdata](const BasicJsonType & v) + { + return ubjson_prefix(v, use_bjdata) == first_prefix; + }); + + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + { + prefix_required = false; + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); + } + } + + if (use_count) + { + oa.write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); + } + + return !use_count; + } + + /*! + @brief write out @a root and everything below it without the call stack + + @sa @ref write_cbor_iterative + */ + void write_ubjson_iterative(const BasicJsonType& root, const bool use_count, const bool use_type, + const bool add_prefix, const bool use_bjdata, const bjdata_version_t bjdata_version) + { + struct frame + { + frame(const BasicJsonType* value_, const bool add_prefix_) noexcept + : value(value_) + , add_prefix(add_prefix_) + , prefix_required(true) + , started(false) + , write_closer(false) + , is_object(false) + {} + + const BasicJsonType* value; + bool add_prefix; + bool prefix_required; + bool started; + bool write_closer; + bool is_object; + typename BasicJsonType::object_t::const_iterator object_it{}; + typename BasicJsonType::array_t::const_iterator array_it{}; + }; + + std::vector stack; + stack.emplace_back(&root, add_prefix); + + while (!stack.empty()) + { + if (!stack.back().started) + { + const BasicJsonType& j = *stack.back().value; + const bool this_prefix = stack.back().add_prefix; + if (!j.is_array() && !j.is_object()) + { + write_ubjson(j, use_count, use_type, this_prefix, use_bjdata, bjdata_version); + stack.pop_back(); + continue; + } + + if (j.is_object() && use_bjdata && j.m_data.m_value.object->size() == 3 && + j.m_data.m_value.object->find("_ArrayType_") != j.m_data.m_value.object->end() && + j.m_data.m_value.object->find("_ArraySize_") != j.m_data.m_value.object->end() && + j.m_data.m_value.object->find("_ArrayData_") != j.m_data.m_value.object->end()) + { + if (!write_bjdata_ndarray(*j.m_data.m_value.object, use_count, use_type, bjdata_version)) + { + stack.pop_back(); + continue; + } + } + + bool prefix_required = true; + if (j.is_array()) + { + stack.back().write_closer = write_ubjson_start_array(j, use_count, use_type, this_prefix, use_bjdata, prefix_required); + stack.back().is_object = false; + stack.back().array_it = j.m_data.m_value.array->cbegin(); + } + else + { + stack.back().write_closer = write_ubjson_start_object(j, use_count, use_type, this_prefix, use_bjdata, prefix_required); + stack.back().is_object = true; + stack.back().object_it = j.m_data.m_value.object->cbegin(); + } + stack.back().prefix_required = prefix_required; + stack.back().started = true; + continue; + } + + frame& current = stack.back(); + const BasicJsonType& j = *current.value; + if (current.is_object) + { + if (current.object_it == j.m_data.m_value.object->cend()) + { + if (current.write_closer) + { + oa.write_character(to_char_type('}')); + } + stack.pop_back(); + continue; + } + + write_number_with_ubjson_prefix(current.object_it->first.size(), true, use_bjdata); + oa.write_characters( + reinterpret_cast(current.object_it->first.data()), + current.object_it->first.size()); + const BasicJsonType* child = &(current.object_it->second); + const bool child_prefix = current.prefix_required; + ++current.object_it; + stack.emplace_back(child, child_prefix); + } + else + { + if (current.array_it == j.m_data.m_value.array->cend()) + { + if (current.write_closer) + { + oa.write_character(to_char_type(']')); + } + stack.pop_back(); + continue; + } + + const BasicJsonType* child = &(*current.array_it); + const bool child_prefix = current.prefix_required; + ++current.array_it; + stack.emplace_back(child, child_prefix); + } + } + } + ////////// // BSON // ////////// diff --git a/tests/src/unit-large_json.cpp b/tests/src/unit-large_json.cpp index 98d16e336..e3f8a8d4c 100644 --- a/tests/src/unit-large_json.cpp +++ b/tests/src/unit-large_json.cpp @@ -12,6 +12,9 @@ using nlohmann::json; #include +#include +#include +#include TEST_CASE("tests on very large JSONs") { @@ -27,3 +30,124 @@ TEST_CASE("tests on very large JSONs") } } +namespace +{ +json nested_array(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json a = json::array(); + a.push_back(std::move(j)); + j = std::move(a); + } + return j; +} + +json nested_object(const std::size_t depth, json leaf) +{ + json j = std::move(leaf); + for (std::size_t i = 0; i < depth; ++i) + { + json o = json::object(); + o["k"] = std::move(j); + j = std::move(o); + } + return j; +} +} // namespace + +TEST_CASE("issue #5392 - binary writers on deeply nested values") +{ + // 200 is past the point where the writers stop recursing, and still + // shallow enough that from_* and operator== (which still recurse) are fine. + const json deep_array = nested_array(200, json(0)); + const json deep_object = nested_object(200, json("x")); + const json empty_array = nested_array(200, json::array()); + const json empty_object = nested_object(200, json::object()); + const json mixed = nested_object(80, nested_array(80, json(true))); + + SECTION("roundtrip past the recursion bound") + { + CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array); + CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array); + CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array); + CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array); + + CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object); + CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object); + CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object); + CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object); + + CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array); + CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array); + CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array); + + CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object); + CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object); + CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object); + + CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed); + CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed); + CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed); + CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed); + } + + SECTION("the two ways of writing a value meet at the bound") + { + for (std::size_t depth = 120; depth <= 140; ++depth) + { + CAPTURE(depth); + + const json array = nested_array(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(array)) == array); + CHECK(json::from_msgpack(json::to_msgpack(array)) == array); + CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array); + + const json object = nested_object(depth, json(7)); + CHECK(json::from_cbor(json::to_cbor(object)) == object); + CHECK(json::from_msgpack(json::to_msgpack(object)) == object); + CHECK(json::from_bjdata(json::to_bjdata(object)) == object); + } + } + + SECTION("a BJData ndarray below the bound is still an ndarray") + { + const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}); + const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}}); + + const json deep_ndarray = nested_array(140, ndarray); + const json deep_invalid = nested_array(140, invalid); + + CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray); + CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid); + CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray); + } + + SECTION("does not overflow the C++ stack") + { + const std::size_t depth = 100000; + const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']')); + + std::vector packed; + CHECK_NOTHROW(packed = json::to_cbor(j)); + CHECK(packed.size() > depth); + + CHECK_NOTHROW(packed = json::to_msgpack(j)); + CHECK(packed.size() > depth); + + CHECK_NOTHROW(packed = json::to_ubjson(j)); + CHECK(packed.size() > depth); + + CHECK_NOTHROW(packed = json::to_ubjson(j, true, false)); + CHECK(packed.size() > depth); + + CHECK_NOTHROW(packed = json::to_bjdata(j)); + CHECK(packed.size() > depth); + } +} +