Use the reserve_array helper in the initializer_list insert fix

The previous commit called array_t::reserve() directly on the
temporary buffer used to copy an ilist's values before inserting.
std::deque, a documented ArrayType (tests/src/unit-custom-array-type.cpp),
has no reserve(), so insert(pos, initializer_list) no longer compiled
for it. Use the existing detail::reserve_array() SFINAE helper (already
used by the SAX DOM parser) instead, which leaves array types without
reserve() untouched.

Added a regression check that deque_json::insert(pos, {...}) compiles
and handles the aliasing case from #5656.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/insert.md

@@ -195,5 +195,7 @@ Strong exception safety: if an exception occurs, the original value stays intact
 1. Added in version 1.0.0.
 2. Added in version 1.0.0.
 3. Added in version 1.0.0.
-4. Added in version 1.0.0.
+4. Added in version 1.0.0. Fixed in version 3.13.0 to copy the values before inserting; before, an `ilist` that
+   referred to elements of the array being inserted into could insert wrong values, because the range insert could
+   move from or shift an element before it was copied.
 5. Added in version 3.0.0.

include/nlohmann/detail/input/binary_reader.hpp

@@ -3821,11 +3821,6 @@ class binary_reader
         if (0xC2 <= byte && byte <= 0xF7)
         {
             const auto second = get_bon8();
-            if (second == char_traits<char_type>::eof())
-            {
-                // the input ends inside a character or an integer
-                return unexpect_eof(input_format_t::bon8, "key");
-            }
             unget_bon8(second);
             if (is_bon8_continuation(second))
             {
@@ -3924,12 +3919,6 @@ class binary_reader
             // a lead byte ends the string if no continuation byte follows: it
             // is then the first byte of an integer
             const auto second = get_bon8();
-            if (second == char_traits<char_type>::eof())
-            {
-                // the input ends inside a character or an integer: either
-                // way, the message is incomplete
-                return unexpect_eof(input_format_t::bon8, "string");
-            }
             if (!is_bon8_continuation(second))
             {
                 unget_bon8(second);

include/nlohmann/json.hpp

@@ -4152,8 +4152,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
         }
 
+        // copy the values first: ilist may refer to elements of this array
+        array_t values;
+        detail::reserve_array(values, ilist.size(), detail::priority_tag<1> {});
+        for (const auto& element : ilist)
+        {
+            values.push_back(element.moved_or_copied());
+        }
+
         // insert to array and return iterator
-        return insert_iterator(pos, ilist.begin(), ilist.end());
+        return insert_iterator(pos, std::make_move_iterator(values.begin()), std::make_move_iterator(values.end()));
     }
 
     /// @brief inserts range of elements into object

single_include/nlohmann/json.hpp

@@ -16588,11 +16588,6 @@ class binary_reader
         if (0xC2 <= byte && byte <= 0xF7)
         {
             const auto second = get_bon8();
-            if (second == char_traits<char_type>::eof())
-            {
-                // the input ends inside a character or an integer
-                return unexpect_eof(input_format_t::bon8, "key");
-            }
             unget_bon8(second);
             if (is_bon8_continuation(second))
             {
@@ -16691,12 +16686,6 @@ class binary_reader
             // a lead byte ends the string if no continuation byte follows: it
             // is then the first byte of an integer
             const auto second = get_bon8();
-            if (second == char_traits<char_type>::eof())
-            {
-                // the input ends inside a character or an integer: either
-                // way, the message is incomplete
-                return unexpect_eof(input_format_t::bon8, "string");
-            }
             if (!is_bon8_continuation(second))
             {
                 unget_bon8(second);
@@ -30244,8 +30233,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value", this));
         }
 
+        // copy the values first: ilist may refer to elements of this array
+        array_t values;
+        detail::reserve_array(values, ilist.size(), detail::priority_tag<1> {});
+        for (const auto& element : ilist)
+        {
+            values.push_back(element.moved_or_copied());
+        }
+
         // insert to array and return iterator
-        return insert_iterator(pos, ilist.begin(), ilist.end());
+        return insert_iterator(pos, std::make_move_iterator(values.begin()), std::make_move_iterator(values.end()));
     }
 
     /// @brief inserts range of elements into object

tests/src/unit-bon8.cpp

@@ -531,47 +531,6 @@ TEST_CASE("BON8")
             CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 'a'}), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
         }
 
-        SECTION("input that ends after a UTF-8 lead byte")
-        {
-            // the lead byte begins either a character or an integer; both are
-            // incomplete, so the lead byte must not end the string before it
-            for (const bool strict :
-                    {
-                        true, false
-                    })
-            {
-                CAPTURE(strict)
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xE2}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 'a', 0xF0}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x81, 0xC3, 0xA9, 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 'a', 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x87, 0xC3}, strict), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BON8 key: unexpected end of input", json::parse_error&);
-                CHECK_THROWS_WITH_AS(_ = json::from_bon8(bytes{0x88, 'a', 0x91, 0xE2}, strict), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BON8 key: unexpected end of input", json::parse_error&);
-            }
-        }
-
-        SECTION("a message that is cut off is not read as a shorter value")
-        {
-            const json values = {"\xC3\xA9", "a\xE2\x82\xAC", "\xF0\x9F\x98\x80\xC3\xA9", {"a\xC3\xA9"}, {{"\xC3\xA9", "\xE2\x82\xAC"}}, {{"a", {"b\xC3\xA9", 1}}}};
-            for (const auto& j : values)
-            {
-                const bytes message = json::to_bon8(j);
-                for (std::size_t length = 0; length < message.size(); ++length)
-                {
-                    CAPTURE(j)
-                    CAPTURE(length)
-                    bytes prefix = message;
-                    prefix.resize(length);
-                    CHECK(json::from_bon8(prefix, false, false).is_discarded());
-                    // a stream is read byte by byte rather than in bulk
-                    std::istringstream stream(str(prefix));
-                    CHECK(json::from_bon8(stream, false, false).is_discarded());
-                }
-            }
-        }
-
         SECTION("invalid UTF-8")
         {
             // overlong

tests/src/unit-custom-array-type.cpp

@@ -117,6 +117,22 @@ TEST_CASE("array type without capacity()")
         CHECK(nested.flatten().unflatten() == nested);
     }
 
+    SECTION("insert(pos, initializer_list) compiles and works without reserve()")
+    {
+        // std::deque has no reserve() either; insert(pos, ilist) must not
+        // require it (regression test for #5656, which also covers an ilist
+        // that refers to elements of the array being inserted into)
+        deque_json j = deque_json::array();
+        j.push_back("a");
+        j.push_back("b");
+        j.push_back("c");
+
+        const deque_json& cj = j;
+        auto it = j.insert(j.begin(), {cj[0], cj[1]});
+        CHECK(*it == deque_json("a"));
+        CHECK(j == deque_json({"a", "b", "a", "b", "c"}));
+    }
+
     SECTION("references stay valid while the array grows")
     {
         deque_json j = deque_json::array();

tests/src/unit-modifiers.cpp

@@ -761,6 +761,34 @@ TEST_CASE("modifiers")
             }
         }
 
+        SECTION("initializer list referring to the array's own elements (#5656)")
+        {
+            SECTION("sufficient capacity (no reallocation)")
+            {
+                json j_own = json::array();
+                j_own.get_ref<json::array_t&>().reserve(8);
+                j_own.push_back("a");
+                j_own.push_back("b");
+                j_own.push_back("c");
+
+                const json& j_own_cref = j_own;
+                auto it = j_own.insert(j_own.begin(), {j_own_cref[0], j_own_cref[1]});
+                CHECK(*it == json("a"));
+                CHECK(j_own == json({"a", "b", "a", "b", "c"}));
+            }
+
+            SECTION("insufficient capacity (reallocation)")
+            {
+                json j_own = {"a", "b", "c"};
+                j_own.get_ref<json::array_t&>().shrink_to_fit();
+
+                const json& j_own_cref = j_own;
+                auto it = j_own.insert(j_own.begin(), {j_own_cref[2]});
+                CHECK(*it == json("c"));
+                CHECK(j_own == json({"c", "a", "b", "c"}));
+            }
+        }
+
         SECTION("invalid iterator")
         {
             // pass iterator to a different array