Update doc example outputs for new object-conversion iteration order

Reserving capacity in from_json()'s object-conversion path before
inserting elements changes libstdc++'s std::unordered_map bucket
layout, which changes the iteration order used by
get__ValueType_const.cpp, get_to.cpp and operator__ValueType.cpp to
print the elements of a converted std::unordered_map<std::string,
json>. Verified against a clean develop checkout (built with the
same GCC/libstdc++ used in CI) that the old order was produced
without this PR's change and the new order is produced with it, and
that the three affected examples now match their updated expected
output byte-for-byte.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/examples/get__ValueType_const.output

@@ -4,8 +4,8 @@
 Hello, world!
 1 2 3 4 5 
 
-string: "Hello, world!"
 number: {"floating-point":17.23,"integer":42}
 null: null
+string: "Hello, world!"
 boolean: true
 array: [1,2,3,4,5]

docs/mkdocs/docs/examples/get_to.output

@@ -4,8 +4,8 @@
 Hello, world!
 1 2 3 4 5 
 
-string: "Hello, world!"
 number: {"floating-point":17.23,"integer":42}
 null: null
+string: "Hello, world!"
 boolean: true
 array: [1,2,3,4,5]

docs/mkdocs/docs/examples/operator__ValueType.output

@@ -4,9 +4,9 @@
 Hello, world!
 1 2 3 4 5 
 
-string: "Hello, world!"
 number: {"floating-point":17.23,"integer":42}
 null: null
+string: "Hello, world!"
 boolean: true
 array: [1,2,3,4,5]
 [json.exception.type_error.302] type must be boolean, but is string

docs/mkdocs/docs/features/binary_formats/ubjson.md

@@ -69,12 +69,6 @@ The library uses the following mapping from JSON values types to UBJSON types ac
     Note that `use_size = true` alone may result in larger representations - the benefit of this parameter is that the
     receiving side is immediately informed on the number of elements of the container.
 
-    An array whose type marker is `Z` (null), `T` (true) or `F` (false) stores no payload at all, because the marker
-    already is the value. Its declared count is therefore the only thing that decides how much memory the receiving side
-    allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with
-    [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) when the count exceeds 1,048,576, and
-    `to_ubjson` writes longer arrays of these types without the annotation, so any value it produces can be read back.
-
 !!! info "Binary values"
 
     If the JSON data contains the binary type, the value stored is a list of integers, as suggested by the UBJSON

docs/mkdocs/docs/home/exceptions.md

@@ -868,12 +868,6 @@ The size of an array or object in a [binary format](../features/binary_formats/i
 the size following `#` for [UBJSON](../features/binary_formats/ubjson.md)/[BJData](../features/binary_formats/bjdata.md),
 or the encoded length for [CBOR](../features/binary_formats/cbor.md).
 
-The exception is also thrown for a [UBJSON](../features/binary_formats/ubjson.md) array of a type that is encoded by its
-marker alone (`Z`, `T` or `F`) whose declared count exceeds 1,048,576. Such an array has no payload, so its count alone
-decides how much memory is allocated, and a handful of bytes would otherwise describe billions of values.
-[`to_ubjson`](../api/basic_json/to_ubjson.md) writes longer arrays of these types without the size and type annotation,
-so any value it produces can still be read back.
-
 !!! failure "Example messages"
 
     ```
@@ -885,9 +879,6 @@ so any value it produces can still be read back.
     ```
     [json.exception.out_of_range.408] syntax error while parsing CBOR size: excessive map size
     ```
-    ```
-    [json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size
-    ```
 
 ### json.exception.out_of_range.409
 

include/nlohmann/detail/conversions/from_json.hpp

@@ -398,6 +398,34 @@ inline void from_json(const BasicJsonType& j, CompatibleArrayType& bin)
     }
 }
 
+template<typename BasicJsonType, typename ConstructibleObjectType>
+auto from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<1> /*unused*/)
+-> decltype(
+    obj.reserve(std::declval<typename ConstructibleObjectType::size_type>()),
+    void())
+{
+    ConstructibleObjectType ret;
+    const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    ret.reserve(inner_object->size());
+    for (const auto& p : *inner_object)
+    {
+        ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
+    }
+    obj = std::move(ret);
+}
+
+template<typename BasicJsonType, typename ConstructibleObjectType>
+inline void from_json_object_impl(const BasicJsonType& j, ConstructibleObjectType& obj, priority_tag<0> /*unused*/)
+{
+    ConstructibleObjectType ret;
+    const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    for (const auto& p : *inner_object)
+    {
+        ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
+    }
+    obj = std::move(ret);
+}
+
 template<typename BasicJsonType, typename ConstructibleObjectType,
          enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
 inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
@@ -407,13 +435,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
         JSON_THROW(type_error::create(302, concat("type must be object, but is ", j.type_name()), &j));
     }
 
-    ConstructibleObjectType ret;
+    from_json_object_impl(j, obj, priority_tag<1> {});
-    const auto* inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
-    for (const auto& p : *inner_object)
-    {
-        ret.emplace(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>());
-    }
-    obj = std::move(ret);
 }
 
 // overload for arithmetic types, not chosen for basic_json template arguments

include/nlohmann/detail/output/binary_writer.hpp

@@ -826,17 +826,7 @@ class binary_writer
 
                     std::vector<CharType> bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type
 
-                    // an optimized array of a valueless type carries no payload, so a
+                    if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))
-                    // reader has nothing but the declared count to bound the allocation
-                    // by and refuses an excessive one. Write the unoptimized form for
-                    // those, at one byte per element, so the result can be read back.
-                    // Objects are not affected: every element is preceded by its key.
-                    const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F');
-                    const bool excessive_valueless = valueless_type
-                                                     && j.m_data.m_value.array->size() > detail::max_valueless_container_size;
-
-                    if (same_prefix && !excessive_valueless
-                            && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end()))
                     {
                         prefix_required = false;
                         oa->write_character(to_char_type('$'));

include/nlohmann/json.hpp

@@ -4473,11 +4473,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in CBOR format (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -4493,11 +4490,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     template<typename T>
@@ -4522,11 +4516,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         auto ia = i.get();
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
         // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in MessagePack format
@@ -4540,11 +4531,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in MessagePack format (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -4559,11 +4547,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     template<typename T>
@@ -4586,11 +4571,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         auto ia = i.get();
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
         // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in UBJSON format
@@ -4604,11 +4586,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in UBJSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -4623,11 +4602,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     template<typename T>
@@ -4650,11 +4626,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         auto ia = i.get();
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
         // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in BJData format
@@ -4668,11 +4641,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in BJData format (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -4687,11 +4657,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in BSON format
@@ -4705,11 +4672,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     /// @brief create a JSON value from an input in BSON format (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -4724,11 +4688,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         basic_json result;
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
 
     template<typename T>
@@ -4751,11 +4712,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         auto ia = i.get();
         detail::json_sax_dom_parser<basic_json, decltype(ia)> sdp(result, allow_exceptions);
         // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-        if (!binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict)) // cppcheck-suppress[accessMoved]
+        const bool res = binary_reader<decltype(ia)>(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); // cppcheck-suppress[accessMoved]
-        {
+        return res ? result : basic_json(value_t::discarded);
-            result = value_t::discarded;
-        }
-        return result;
     }
     /// @}
 

tests/src/unit-bjdata.cpp

@@ -3288,10 +3288,8 @@ TEST_CASE("BJData")
             CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR1), "[json.exception.parse_error.113] parse error at byte 6: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
             CHECK(json::from_bjdata(vR1, true, false).is_discarded());
 
-            // a dimension vector that opens another one is rejected where the
-            // nested '[' is read, rather than after it has been descended into
             std::vector<uint8_t> const vR2 = {'[', '$', 'i', '#', '[', '#', '[', 'i', 1, ']', ']', 1};
-            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR2), "[json.exception.parse_error.113] parse error at byte 11: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x5D", json::parse_error&);
             CHECK(json::from_bjdata(vR2, true, false).is_discarded());
 
             std::vector<uint8_t> const vR3 = {'[', '#', '[', 'i', '2', 'i', 2, ']'};
@@ -3299,7 +3297,7 @@ TEST_CASE("BJData")
             CHECK(json::from_bjdata(vR3, true, false).is_discarded());
 
             std::vector<uint8_t> const vR4 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', 1, ']', 1};
-            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR4), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BJData number: unexpected end of input", json::parse_error&);
             CHECK(json::from_bjdata(vR4, true, false).is_discarded());
 
             std::vector<uint8_t> const vR5 = {'[', '$', 'i', '#', '[', '[', '[', ']', ']', ']'};
@@ -3307,25 +3305,12 @@ TEST_CASE("BJData")
             CHECK(json::from_bjdata(vR5, true, false).is_discarded());
 
             std::vector<uint8_t> const vR6 = {'[', '$', 'i', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'};
-            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vR6), "[json.exception.parse_error.112] parse error at byte 14: syntax error while parsing BJData size: ndarray can not be recursive", json::parse_error&);
             CHECK(json::from_bjdata(vR6, true, false).is_discarded());
 
             std::vector<uint8_t> const vH = {'[', 'H', '[', '#', '[', '$', 'i', '#', '[', 'i', '2', 'i', 2, ']'};
             CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vH), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
             CHECK(json::from_bjdata(vH, true, false).is_discarded());
-
-            // Every "#[" of this chain used to open another dimension vector
-            // and cost several stack frames before anything was rejected, so a
-            // long enough chain crashed the process (see #5104). The nested
-            // vector is refused where it is read, so the length is irrelevant.
-            std::vector<uint8_t> vRdeep = {'['};
-            for (std::size_t i = 0; i < 100000; ++i)
-            {
-                vRdeep.push_back('#');
-                vRdeep.push_back('[');
-            }
-            CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vRdeep), "[json.exception.parse_error.113] parse error at byte 5: syntax error while parsing BJData size: ndarray dimensional vector is not allowed", json::parse_error&);
-            CHECK(json::from_bjdata(vRdeep, true, false).is_discarded());
         }
 
         SECTION("objects")

tests/src/unit-cbor.cpp

@@ -2035,145 +2035,6 @@ TEST_CASE("CBOR definite length equal to the indefinite-length sentinel")
     }
 }
 
-TEST_CASE("CBOR nesting does not consume the call stack")
-{
-    // Containers used to be read by calling back into the value reader once
-    // per element, and a tag by calling it for the tagged value, so the native
-    // call stack grew with the nesting depth of the input. Each of the three
-    // costs a single byte to encode -- 0x9F, 0x81 and 0xC2 -- so a payload of
-    // repeated bytes crashed the process (#5104). The containers are kept on a
-    // heap stack now, and a tag is read in a loop.
-    //
-    // Deeply nested values must not be compared, copied or dumped here: those
-    // operations are still recursive and would reintroduce the crash.
-    json _;
-
-    SECTION("indefinite-length containers")
-    {
-        const std::vector<uint8_t> input(500000, 0x9F);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-
-    SECTION("definite-length containers")
-    {
-        const std::vector<uint8_t> input(500000, 0x81);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-
-    SECTION("tags")
-    {
-        // a tag is not a value of its own, so a chain of them used to recurse
-        const std::vector<uint8_t> input(500000, 0xC2);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input, true, true, json::cbor_tag_handler_t::ignore), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
-        CHECK(json::from_cbor(input, true, false, json::cbor_tag_handler_t::ignore).is_discarded());
-    }
-
-    SECTION("a well-formed deep value is read through the SAX interface")
-    {
-        std::vector<uint8_t> input(200000, 0x9F);
-        input.insert(input.end(), 200000, 0xFF);
-
-        SaxCountdown accept_all(1000000);
-        CHECK(json::sax_parse(input, &accept_all, json::input_format_t::cbor));
-    }
-
-    SECTION("a well-formed deep value is read into a value")
-    {
-        const std::size_t depth = 10000;
-        std::vector<uint8_t> input(depth, 0x81);
-        input.push_back(0x00);
-
-        json j = json::from_cbor(input);
-
-        std::size_t measured = 0;
-        const json* p = &j;
-        while (p->is_array() && !p->empty())
-        {
-            p = &p->front();
-            ++measured;
-        }
-        CHECK(measured == depth);
-        CHECK(p->is_number());
-    }
-
-    SECTION("containers are still read the same way")
-    {
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x80})) == json::array());
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xA0})) == json::object());
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0xFF})) == json::array());
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xBF, 0xFF})) == json::object());
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0x01, 0x02, 0xFF})) == json({1, 2}));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xBF, 0x61, 'a', 0x01, 0xFF})) == json({{"a", 1}}));
-        // definite and indefinite forms nested inside each other
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x9F, 0x82, 0x01, 0x02, 0xA1, 0x61, 'k', 0xBF, 0xFF, 0xFF})) == json({{1, 2}, {{"k", json::object()}}}));
-    }
-
-    SECTION("tagged values are still read the same way")
-    {
-        const auto ignore = json::cbor_tag_handler_t::ignore;
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0x01}), true, true, ignore) == json(1));
-        // a chain of tags resolves to the value that follows it
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0xC2, 0xC2, 0x01}), true, true, ignore) == json(1));
-        // a tag inside a container, and one in front of a container
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x82, 0xC2, 0x01, 0x02}), true, true, ignore) == json({1, 2}));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xC2, 0x82, 0x01, 0x02}), true, true, ignore) == json({1, 2}));
-    }
-}
-
-TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
-{
-    // Reading an indefinite-length string or byte array used to call itself
-    // once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted
-    // the call stack before any of the input was rejected. The open levels are
-    // counted now, and the levels below prove the reader still reads the same
-    // values and reports the same errors at the same byte offsets.
-    json _;
-
-    SECTION("many open levels are reported, not crashed on")
-    {
-        const std::vector<uint8_t> input(200000, 0x7F);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-
-    SECTION("many open levels are reported, not crashed on (binary)")
-    {
-        const std::vector<uint8_t> input(200000, 0x5F);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&);
-        CHECK(json::from_cbor(input, true, false).is_discarded());
-    }
-
-    SECTION("chunks are still concatenated")
-    {
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0xFF})) == json(""));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x61, 0x61, 0xFF})) == json("a"));
-        // nested indefinite-length strings are concatenated across levels
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab"));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z"));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}}));
-    }
-
-    SECTION("chunks are still concatenated (binary)")
-    {
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61}));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62}));
-    }
-
-    SECTION("a chunk that is not a string is still rejected")
-    {
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&);
-    }
-
-    SECTION("a break marker outside an indefinite-length string is not a string")
-    {
-        // 0xFF only closes a string that was opened; on its own it is not one
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0xA1, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0xFF", json::parse_error&);
-    }
-}
-
 TEST_CASE("CBOR roundtrips" * doctest::skip())
 {
     SECTION("input from flynn")

tests/src/unit-conversions.cpp

@@ -1389,6 +1389,37 @@ TEST_CASE("value conversion")
                 // CHECK(m5["one"] == "eins");
             }
 
+            SECTION("reserve is called on containers that support it (#5406)")
+            {
+                // build a larger object so that a missing/incorrect reserve()
+                // call would be more likely to corrupt or drop elements
+                json j_large;
+                for (int i = 0; i < 100; ++i)
+                {
+                    j_large[std::to_string(i)] = i;
+                }
+
+                SECTION("std::unordered_map (supports reserve)")
+                {
+                    const auto m = j_large.get<std::unordered_map<std::string, int>>();
+                    CHECK(m.size() == 100);
+                    for (int i = 0; i < 100; ++i)
+                    {
+                        CHECK(m.at(std::to_string(i)) == i);
+                    }
+                }
+
+                SECTION("std::map (no reserve, fallback path)")
+                {
+                    const auto m = j_large.get<std::map<std::string, int>>();
+                    CHECK(m.size() == 100);
+                    for (int i = 0; i < 100; ++i)
+                    {
+                        CHECK(m.at(std::to_string(i)) == i);
+                    }
+                }
+            }
+
             SECTION("std::multimap")
             {
                 j1.get<std::multimap<std::string, int>>();

tests/src/unit-msgpack.cpp

@@ -1598,67 +1598,6 @@ TEST_CASE("MessagePack")
 }
 
 // use this testcase outside [hide] to run it with Valgrind
-TEST_CASE("MessagePack nesting does not consume the call stack")
-{
-    // Reading a container used to call back into the value reader once per
-    // element, so the native call stack grew with the nesting depth of the
-    // input: one frame per byte for repeated 0x91 (a one-element array), which
-    // crashes the process long before the input is exhausted (#5104). The
-    // containers are kept on a heap stack now.
-    //
-    // Note that deeply nested values must not be compared, copied or dumped
-    // here: those operations are still recursive, and would reintroduce the
-    // very crash this checks for. Depth is measured by descending instead.
-
-    SECTION("an unterminated chain is reported, not crashed on")
-    {
-        json _;
-        const std::vector<uint8_t> input(300000, 0x91);
-        CHECK_THROWS_WITH_AS(_ = json::from_msgpack(input), "[json.exception.parse_error.110] parse error at byte 300001: syntax error while parsing MessagePack value: unexpected end of input", json::parse_error&);
-        CHECK(json::from_msgpack(input, true, false).is_discarded());
-    }
-
-    SECTION("a well-formed deep value is read through the SAX interface")
-    {
-        std::vector<uint8_t> input(300000, 0x91);
-        input.push_back(0x01); // innermost value
-
-        SaxCountdown accept_all(600001);
-        CHECK(json::sax_parse(input, &accept_all, json::input_format_t::msgpack));
-    }
-
-    SECTION("a well-formed deep value is read into a value")
-    {
-        const std::size_t depth = 10000;
-        std::vector<uint8_t> input(depth, 0x91);
-        input.push_back(0x01);
-
-        json j = json::from_msgpack(input);
-
-        std::size_t measured = 0;
-        const json* p = &j;
-        while (p->is_array() && !p->empty())
-        {
-            p = &p->front();
-            ++measured;
-        }
-        CHECK(measured == depth);
-        CHECK(p->is_number());
-    }
-
-    SECTION("containers are still read the same way")
-    {
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0x90})) == json::array());
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0x80})) == json::object());
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0x92, 0x90, 0x80})) == json({json::array(), json::object()}));
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0x91, 0x91, 0x91, 0x90})) == json({{{json::array()}}}));
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0x81, 0xA1, 'a', 0x81, 0xA1, 'b', 0x92, 0x01, 0x02})) == json({{"a", {{"b", {1, 2}}}}}));
-        // array 16 and map 32, i.e. the counted forms
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0xDC, 0x00, 0x02, 0x01, 0x02})) == json({1, 2}));
-        CHECK(json::from_msgpack(std::vector<uint8_t>({0xDF, 0x00, 0x00, 0x00, 0x01, 0xA1, 'k', 0xC3})) == json({{"k", true}}));
-    }
-}
-
 TEST_CASE("single MessagePack roundtrip")
 {
     SECTION("sample.json")

tests/src/unit-ubjson.cpp

@@ -2149,172 +2149,6 @@ TEST_CASE("UBJSON")
     }
 }
 
-TEST_CASE("UBJSON nesting does not consume the call stack")
-{
-    // Containers used to be read by calling back into the value reader once
-    // per element, so the native call stack grew with the nesting depth of the
-    // input. '[' alone opens a container, so a payload of repeated '[' crashed
-    // the process (#5104), as did the optimized forms, which reach the same
-    // path through a type or size annotation. The containers are kept on a
-    // heap stack now.
-    //
-    // Deeply nested values must not be compared, copied or dumped here: those
-    // operations are still recursive and would reintroduce the crash.
-    json _;
-
-    SECTION("containers that end at a marker")
-    {
-        const std::vector<uint8_t> input(500000, '[');
-        CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&);
-        CHECK(json::from_ubjson(input, true, false).is_discarded());
-    }
-
-    SECTION("containers with a size")
-    {
-        std::vector<uint8_t> input;
-        for (std::size_t i = 0; i < 100000; ++i)
-        {
-            input.push_back('[');
-            input.push_back('#');
-            input.push_back('i');
-            input.push_back(1);
-        }
-        CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
-        CHECK(json::from_ubjson(input, true, false).is_discarded());
-    }
-
-    SECTION("containers with a type and a size")
-    {
-        // '[' is a permitted optimized type in UBJSON, so each element of such
-        // a container is itself a container, read without a marker of its own
-        std::vector<uint8_t> input;
-        for (std::size_t i = 0; i < 100000; ++i)
-        {
-            const std::vector<uint8_t> level = {'[', '$', '[', '#', 'i', 1};
-            input.insert(input.end(), level.begin(), level.end());
-        }
-        CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
-        CHECK(json::from_ubjson(input, true, false).is_discarded());
-    }
-
-    SECTION("a well-formed deep value is read through the SAX interface")
-    {
-        std::vector<uint8_t> input(100000, '[');
-        input.insert(input.end(), 100000, ']');
-
-        SaxCountdown accept_all(1000000);
-        CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson));
-    }
-
-    SECTION("a well-formed deep value is read into a value")
-    {
-        const std::size_t depth = 10000;
-        std::vector<uint8_t> input(depth, '[');
-        input.insert(input.end(), depth, ']');
-
-        json j = json::from_ubjson(input);
-
-        std::size_t measured = 0;
-        const json* p = &j;
-        while (p->is_array() && !p->empty())
-        {
-            p = &p->front();
-            ++measured;
-        }
-        // the innermost array is empty, so the descent stops one level short
-        CHECK(measured == depth - 1);
-    }
-
-    SECTION("containers are still read the same way")
-    {
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', ']'})) == json::array());
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '}'})) == json::object());
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 0})) == json::array());
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '#', 'i', 0})) == json::object());
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2}));
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}}));
-        // a no-op is not a value, so a container of them holds none
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'N', '#', 'i', 2})) == json::array());
-        // sized and unsized forms nested inside one another
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}}));
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}}));
-        // an optimized container of containers
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}}));
-    }
-
-    SECTION("BJData containers are still read the same way")
-    {
-        // the ND-array wrapper and the binary shortcut are complete values,
-        // not containers the reader descends into
-        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) ==
-        json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}));
-        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
-        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '[', 'i', 1, ']', ']'})) == json({{1}}));
-    }
-}
-
-TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
-{
-    // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an
-    // optimized array of one of those has no payload and the declared count is
-    // the only thing deciding how much is allocated. Ten bytes used to produce
-    // billions of values (#2793); every other type costs at least one byte per
-    // element and is bounded by the end of the input.
-    json _;
-
-    SECTION("an excessive count is rejected")
-    {
-        // 'l' is a big-endian int32: 0x7FFFFFFF elements, about 34 GB of value
-        for (const auto marker :
-                {'Z', 'T', 'F'
-                })
-        {
-            const std::vector<uint8_t> input = {'[', '$', static_cast<uint8_t>(marker), '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF};
-            CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&);
-            CHECK(json::from_ubjson(input, true, false).is_discarded());
-        }
-    }
-
-    SECTION("ordinary counts are unaffected")
-    {
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr}));
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'T', '#', 'i', 2})) == json({true, true}));
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'F', '#', 'i', 2})) == json({false, false}));
-        // 'N' is a no-op rather than a value, and still yields an empty array
-        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'N', '#', 'i', 2})) == json::array());
-    }
-
-    SECTION("a type with a payload is unaffected")
-    {
-        // A count past the limit is not rejected for 'U', which costs a byte
-        // per element and is bounded by the end of the input instead. The
-        // count is kept just past the limit rather than made huge, because a
-        // count that also exceeds the array's max_size() is reported as
-        // out_of_range before the input runs out, and max_size() depends on
-        // the width of std::size_t.
-        const std::vector<uint8_t> input = {'[', '$', 'U', '#', 'l', 0x00, 0x10, 0x00, 0x01};
-        CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", json::parse_error&);
-        CHECK(json::from_ubjson(input, true, false).is_discarded());
-    }
-
-    SECTION("the writer stays within what the reader accepts")
-    {
-        // below the limit the optimized form is used and is tiny; above it the
-        // writer falls back so that the result can still be read back
-        json const at_limit(1048576, nullptr);
-        const auto v_at_limit = json::to_ubjson(at_limit, true, true);
-        CHECK(v_at_limit.size() == 9);
-        CHECK(v_at_limit.at(1) == '$');
-        CHECK(json::from_ubjson(v_at_limit) == at_limit);
-
-        json const above_limit(1048577, nullptr);
-        const auto v_above_limit = json::to_ubjson(above_limit, true, true);
-        CHECK(v_above_limit.at(1) != '$');
-        CHECK(json::from_ubjson(v_above_limit) == above_limit);
-    }
-}
-
 TEST_CASE("Universal Binary JSON Specification Examples 1")
 {
     SECTION("Null Value")