Merge branch 'develop' into claude/copy-without-assignable-base-5674

Conflicts:
- include/nlohmann/json.hpp: copy_array_level() keeps develop's #5721
  "set the array type only once the container exists" and then builds the
  elements with the PR's copy_construct_tag emplace_back loop
- single_include/nlohmann/json.hpp: regenerated with make amalgamate

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/operator_gtgt.md

@@ -18,6 +18,10 @@ Deserializes an input stream to a JSON value.
 
 the stream `i`
 
+## Exception safety
+
+Strong guarantee: if an exception is thrown, there are no changes in `j`.
+
 ## Exceptions
 
 - Throws [`parse_error.101`](../home/exceptions.md#jsonexceptionparse_error101) in case of an unexpected token, or if
@@ -125,3 +129,5 @@ being read.
   the stream; planned to become the default in version 4.0.0.
 - Fixed a null pointer dereference for an `std::istream` without a stream buffer (now throws `parse_error.101`), and a
   crash (`std::terminate`) when `i` has `eofbit` in its exception mask, in version 3.13.0.
+- Changed to the strong exception safety guarantee in version 3.13.0: `j` is no longer left with a partially parsed
+  value if parsing throws.

docs/mkdocs/docs/features/types/template_parameters.md

@@ -389,6 +389,7 @@ using array_t = ArrayType<basic_json, AllocatorType<basic_json>>;
 | Functionality                                                                                                                     | Additional requirement                                                                                                                                                                       |
 |-----------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | [`diff`](../../api/basic_json/diff.md), [`items`](../../api/basic_json/items.md), [`std::hash`](../../api/basic_json/std_hash.md) | conversion of a `#!cpp std::size_t` to `StringType`: either assignability from the result of `#!cpp std::to_string`, or an ADL overload `#!cpp void int_to_string(StringType&, std::size_t)` |
+| [`operator/(std::size_t)`](../../api/json_pointer/operator_slash.md)                                                              | the same conversion of a `#!cpp std::size_t` to `StringType` as `diff`, `items`, and `std::hash` above                                                                                       |
 | [`std::hash<basic_json>`](../../api/basic_json/std_hash.md)                                                                       | additionally a specialization of `#!cpp std::hash<StringType>`                                                                                                                               |
 | [`to_bson`](../../api/basic_json/to_bson.md)                                                                                      | `find(value_type)` and `npos`                                                                                                                                                                |
 | [`parse`](../../api/basic_json/parse.md) from a `string_t`                                                                        | the input adapters must accept it; otherwise pass a character range                                                                                                                          |

include/nlohmann/detail/json_pointer.hpp

@@ -26,6 +26,7 @@
 #include <nlohmann/detail/macro_scope.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 #include <nlohmann/detail/string_escape.hpp>
+#include <nlohmann/detail/string_utils.hpp>
 #include <nlohmann/detail/value_t.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -116,7 +117,7 @@ class json_pointer
     /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
     json_pointer& operator/=(std::size_t array_idx)
     {
-        return *this /= std::to_string(array_idx);
+        return *this /= detail::to_string<string_t>(array_idx);
     }
 
     /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer
@@ -752,7 +753,7 @@ class json_pointer
                         // would throw out_of_range.404 -- contains() must not throw (see #5395)
                         return false;
                     }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9")))
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9')))
                     {
                         // invalid char
                         return false;
@@ -780,7 +781,7 @@ class json_pointer
                     // not throw (see #5395), so such a reference token is treated as "not found"
                     errno = 0; // strtoull() does not reset errno on success
                     char* p_end = nullptr; // NOLINT(misc-const-correctness)
-                    const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
+                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
                     if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
                                              || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
                     {

include/nlohmann/json.hpp

@@ -1004,19 +1004,39 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     using copy_scratch_value_t = std::pair<typename object_t::key_type, basic_json>;
     using copy_scratch_t = std::vector<copy_scratch_value_t, AllocatorType<copy_scratch_value_t>>;
 
-    /// @brief copy everything of @a src into @a dst but its type and value
-    static void copy_metadata(const basic_json& src, basic_json& dst)
-    {
-        // a custom base class is only required to be copy-constructible and
-        // move-assignable, so the copy has to go through a temporary
-        static_cast<json_base_class_t&>(dst) = json_base_class_t(static_cast<const json_base_class_t&>(src));
+    /// @brief tag selecting the constructor below; used only to build the
+    /// elements of a deep copy (@ref copy_array_level, @ref copy_object_level)
+    struct copy_construct_tag {};
 
+  public:
+    /*!
+    @brief construct a null value whose base class - and, with @ref
+           JSON_DIAGNOSTIC_POSITIONS, positions - are copied from @a src
+
+    Copy-constructing @ref json_base_class_t here, rather than default-
+    constructing the element and assigning its base class afterwards, means
+    that copying a @ref basic_json only ever requires a copy-constructible
+    base class, and never a move-assignable one as well.
+
+    @note this constructor has to be public: @ref copy_array_level and
+          @ref copy_object_level reach it through @ref array_t's or @ref
+          object_t's own emplace_back(), which constructs the element from
+          outside @ref basic_json and so cannot call a private constructor.
+          @ref copy_construct_tag is private, though, and nothing in the
+          public interface hands out a value of it, so outside code can still
+          never name it to call this constructor itself.
+    */
+    basic_json(copy_construct_tag /*unused*/, const basic_json& src)
+        : json_base_class_t(src)
 #if JSON_DIAGNOSTIC_POSITIONS
-        dst.start_position = src.start_position;
-        dst.end_position = src.end_position;
+        , start_position(src.start_position)
+        , end_position(src.end_position)
 #endif
+    {
     }
 
+  private:
+
     /*!
     @brief copy the value of @a src into @a dst, which must not be structured
 
@@ -1079,8 +1099,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     }
 
     /*!
-    @brief copy everything of @a src into the null value @a dst but the children
+    @brief finish the copy @a dst of @a src that a @ref copy_construct_tag
+           constructor started, other than the children of an object or array
 
+    @a dst already has @a src's base class and, with @ref
+    JSON_DIAGNOSTIC_POSITIONS, positions; only its value is still missing.
     Objects and arrays are not copied here; they are appended to @a worklist to
     be created later by @ref copy_iteratively. Until that happens, @a dst remains
     a null value, so that a partially built copy can be destroyed at any point
@@ -1088,8 +1111,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     */
     static void copy_shallow(const basic_json& src, basic_json& dst, copy_worklist_t& worklist)
     {
-        copy_metadata(src, dst);
-
         if (src.m_data.m_type == value_t::object || src.m_data.m_type == value_t::array)
         {
             // defer: dst stays a null value until its container exists
@@ -1110,15 +1131,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     {
         const array_t& src_array = *src.m_data.m_value.array;
 
-        // create all elements up front: growing the array afterwards could
-        // invalidate the pointers that are handed to the worklist; resize()
-        // rather than the fill constructor, because not every array type
-        // provides the latter (e.g., ones without a matching allocator-aware
-        // fill constructor)
         dst.m_data.m_value.array = create<array_t>();
         // only now that the array exists may dst stop being a null value
         dst.m_data.m_type = value_t::array;
-        dst.m_data.m_value.array->resize(src_array.size());
+
+        // create every element - its base class already copy-constructed from
+        // its counterpart in src, via the copy_construct_tag constructor -
+        // before any of their addresses are handed to worklist below: growing
+        // the array while that is going on could reallocate it and invalidate
+        // addresses taken from an earlier iteration
+        for (const auto& src_element : src_array)
+        {
+            dst.m_data.m_value.array->emplace_back(copy_construct_tag{}, src_element);
+        }
 
         auto dst_it = dst.m_data.m_value.array->begin();
         for (auto src_it = src_array.cbegin(); src_it != src_array.cend(); ++src_it, ++dst_it)
@@ -1136,12 +1161,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
         // build the complete key skeleton and hand it to the object's range
         // constructor: adding the keys one by one would be quadratic for object
-        // types that are backed by a vector, such as nlohmann::ordered_map
+        // types that are backed by a vector, such as nlohmann::ordered_map; each
+        // value's base class is already copy-constructed from its counterpart
+        // in src, via the copy_construct_tag constructor
         scratch.clear();
         scratch.reserve(src_object.size());
         for (const auto& element : src_object)
         {
-            scratch.emplace_back(element.first, basic_json());
+            scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second));
         }
 
         dst.m_data.m_value.object = create<object_t>(std::make_move_iterator(scratch.begin()),
@@ -5092,7 +5119,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/
     friend std::istream& operator>>(std::istream& i, basic_json& j)
     {
-        parser(detail::input_adapter(i)).parse(false, j);
+        // parse into a temporary so that j is left unchanged if parsing fails
+        basic_json result;
+        parser(detail::input_adapter(i)).parse(false, result);
+        j = std::move(result);
         return i;
     }
 #endif  // JSON_NO_IO

single_include/nlohmann/json.hpp

@@ -19636,6 +19636,8 @@ NLOHMANN_JSON_NAMESPACE_END
 
 // #include <nlohmann/detail/string_escape.hpp>
 
+// #include <nlohmann/detail/string_utils.hpp>
+
 // #include <nlohmann/detail/value_t.hpp>
 
 
@@ -19727,7 +19729,7 @@ class json_pointer
     /// @sa https://json.nlohmann.me/api/json_pointer/operator_slasheq/
     json_pointer& operator/=(std::size_t array_idx)
     {
-        return *this /= std::to_string(array_idx);
+        return *this /= detail::to_string<string_t>(array_idx);
     }
 
     /// @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer
@@ -20363,7 +20365,7 @@ class json_pointer
                         // would throw out_of_range.404 -- contains() must not throw (see #5395)
                         return false;
                     }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !("0" <= reference_token && reference_token <= "9")))
+                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9')))
                     {
                         // invalid char
                         return false;
@@ -20391,7 +20393,7 @@ class json_pointer
                     // not throw (see #5395), so such a reference token is treated as "not found"
                     errno = 0; // strtoull() does not reset errno on success
                     char* p_end = nullptr; // NOLINT(misc-const-correctness)
-                    const unsigned long long magnitude = std::strtoull(reference_token.c_str(), &p_end, 10); // NOLINT(runtime/int)
+                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
                     if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
                                              || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
                     {
@@ -27929,19 +27931,39 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     using copy_scratch_value_t = std::pair<typename object_t::key_type, basic_json>;
     using copy_scratch_t = std::vector<copy_scratch_value_t, AllocatorType<copy_scratch_value_t>>;
 
-    /// @brief copy everything of @a src into @a dst but its type and value
-    static void copy_metadata(const basic_json& src, basic_json& dst)
-    {
-        // a custom base class is only required to be copy-constructible and
-        // move-assignable, so the copy has to go through a temporary
-        static_cast<json_base_class_t&>(dst) = json_base_class_t(static_cast<const json_base_class_t&>(src));
+    /// @brief tag selecting the constructor below; used only to build the
+    /// elements of a deep copy (@ref copy_array_level, @ref copy_object_level)
+    struct copy_construct_tag {};
 
+  public:
+    /*!
+    @brief construct a null value whose base class - and, with @ref
+           JSON_DIAGNOSTIC_POSITIONS, positions - are copied from @a src
+
+    Copy-constructing @ref json_base_class_t here, rather than default-
+    constructing the element and assigning its base class afterwards, means
+    that copying a @ref basic_json only ever requires a copy-constructible
+    base class, and never a move-assignable one as well.
+
+    @note this constructor has to be public: @ref copy_array_level and
+          @ref copy_object_level reach it through @ref array_t's or @ref
+          object_t's own emplace_back(), which constructs the element from
+          outside @ref basic_json and so cannot call a private constructor.
+          @ref copy_construct_tag is private, though, and nothing in the
+          public interface hands out a value of it, so outside code can still
+          never name it to call this constructor itself.
+    */
+    basic_json(copy_construct_tag /*unused*/, const basic_json& src)
+        : json_base_class_t(src)
 #if JSON_DIAGNOSTIC_POSITIONS
-        dst.start_position = src.start_position;
-        dst.end_position = src.end_position;
+        , start_position(src.start_position)
+        , end_position(src.end_position)
 #endif
+    {
     }
 
+  private:
+
     /*!
     @brief copy the value of @a src into @a dst, which must not be structured
 
@@ -28004,8 +28026,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     }
 
     /*!
-    @brief copy everything of @a src into the null value @a dst but the children
+    @brief finish the copy @a dst of @a src that a @ref copy_construct_tag
+           constructor started, other than the children of an object or array
 
+    @a dst already has @a src's base class and, with @ref
+    JSON_DIAGNOSTIC_POSITIONS, positions; only its value is still missing.
     Objects and arrays are not copied here; they are appended to @a worklist to
     be created later by @ref copy_iteratively. Until that happens, @a dst remains
     a null value, so that a partially built copy can be destroyed at any point
@@ -28013,8 +28038,6 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     */
     static void copy_shallow(const basic_json& src, basic_json& dst, copy_worklist_t& worklist)
     {
-        copy_metadata(src, dst);
-
         if (src.m_data.m_type == value_t::object || src.m_data.m_type == value_t::array)
         {
             // defer: dst stays a null value until its container exists
@@ -28035,15 +28058,19 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     {
         const array_t& src_array = *src.m_data.m_value.array;
 
-        // create all elements up front: growing the array afterwards could
-        // invalidate the pointers that are handed to the worklist; resize()
-        // rather than the fill constructor, because not every array type
-        // provides the latter (e.g., ones without a matching allocator-aware
-        // fill constructor)
         dst.m_data.m_value.array = create<array_t>();
         // only now that the array exists may dst stop being a null value
         dst.m_data.m_type = value_t::array;
-        dst.m_data.m_value.array->resize(src_array.size());
+
+        // create every element - its base class already copy-constructed from
+        // its counterpart in src, via the copy_construct_tag constructor -
+        // before any of their addresses are handed to worklist below: growing
+        // the array while that is going on could reallocate it and invalidate
+        // addresses taken from an earlier iteration
+        for (const auto& src_element : src_array)
+        {
+            dst.m_data.m_value.array->emplace_back(copy_construct_tag{}, src_element);
+        }
 
         auto dst_it = dst.m_data.m_value.array->begin();
         for (auto src_it = src_array.cbegin(); src_it != src_array.cend(); ++src_it, ++dst_it)
@@ -28061,12 +28088,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
         // build the complete key skeleton and hand it to the object's range
         // constructor: adding the keys one by one would be quadratic for object
-        // types that are backed by a vector, such as nlohmann::ordered_map
+        // types that are backed by a vector, such as nlohmann::ordered_map; each
+        // value's base class is already copy-constructed from its counterpart
+        // in src, via the copy_construct_tag constructor
         scratch.clear();
         scratch.reserve(src_object.size());
         for (const auto& element : src_object)
         {
-            scratch.emplace_back(element.first, basic_json());
+            scratch.emplace_back(element.first, basic_json(copy_construct_tag{}, element.second));
         }
 
         dst.m_data.m_value.object = create<object_t>(std::make_move_iterator(scratch.begin()),
@@ -32017,7 +32046,10 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     /// @sa https://json.nlohmann.me/api/basic_json/operator_gtgt/
     friend std::istream& operator>>(std::istream& i, basic_json& j)
     {
-        parser(detail::input_adapter(i)).parse(false, j);
+        // parse into a temporary so that j is left unchanged if parsing fails
+        basic_json result;
+        parser(detail::input_adapter(i)).parse(false, result);
+        j = std::move(result);
         return i;
     }
 #endif  // JSON_NO_IO

tests/src/unit-alt-string.cpp

@@ -352,6 +352,41 @@ TEST_CASE("alternative string type")
         CHECK(j2.flatten().unflatten() == j2);
     }
 
+    SECTION("contains(json_pointer)")
+    {
+        // contains(json_pointer) must compile and work with a string_t that has
+        // no c_str() and no comparison with const char* (see #5666)
+        auto j = alt_json::parse(R"({"foo": ["bar", "baz"]})");
+
+        // present: object key and array indices
+        CHECK(j.contains(alt_json::json_pointer("/foo")));
+        CHECK(j.contains(alt_json::json_pointer("/foo/0")));
+        CHECK(j.contains(alt_json::json_pointer("/foo/1")));
+
+        // missing: absent object key and out-of-range array index
+        CHECK_FALSE(j.contains(alt_json::json_pointer("/bar")));
+        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/2")));
+
+        // "-" always fails the range check
+        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/-")));
+
+        // an array index must not have a leading zero
+        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/01")));
+
+        // a reference token that is not a number
+        CHECK_FALSE(j.contains(alt_json::json_pointer("/foo/bar")));
+    }
+
+    SECTION("operator/(std::size_t)")
+    {
+        // json_pointer::operator/=(std::size_t) must compile without string_t
+        // being constructible from std::string (see #5666)
+        auto j = alt_json::parse(R"({"foo": ["bar", "baz"]})");
+
+        CHECK(j.at(alt_json::json_pointer("/foo") / std::size_t(0)) == j["foo"][0]);
+        CHECK(j.at(alt_json::json_pointer("/foo") / std::size_t(1)) == j["foo"][1]);
+    }
+
     SECTION("patch")
     {
         alt_json const patch1 = alt_json::parse(R"([{ "op": "add", "path": "/a/b", "value": [ "foo", "bar" ] }])");

tests/src/unit-custom-base-class.cpp

@@ -405,3 +405,73 @@ TEST_CASE("JSON Visit Node")
         );
     CHECK(expected.empty());
 }
+
+// A custom base class with a const member: copy-constructible (initializing a
+// const member works fine), but not copy-/move-assignable (assigning one does
+// not). Used to check that copy construction never requires more than that.
+struct const_member_base
+{
+    const int id = 7; // NOLINT(misc-non-private-member-variables-in-classes)
+};
+
+using json_with_const_base = nlohmann::basic_json <
+                             std::map,
+                             std::vector,
+                             std::string,
+                             bool,
+                             std::int64_t,
+                             std::uint64_t,
+                             double,
+                             std::allocator,
+                             nlohmann::adl_serializer,
+                             std::vector<std::uint8_t>,
+                             const_member_base
+                             >;
+
+// build an array nested @a depth levels deep, with the innermost value 1;
+// every level is constructed (never assigned), since const_member_base does
+// not support assignment
+static json_with_const_base make_nested_array(std::size_t depth)
+{
+    if (depth == 0)
+    {
+        return json_with_const_base(1);
+    }
+    return json_with_const_base::array({make_nested_array(depth - 1)});
+}
+
+TEST_CASE("Regression test for issue #5674 - copy construction must not require an assignable base class")
+{
+    SECTION("depth 0")
+    {
+        // as in the original bug report: copy construction only, no assignment
+        const json_with_const_base j = {1, 2};
+        const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization)
+
+        CHECK(copy.size() == 2);
+        CHECK(copy.id == 7);
+    }
+
+    SECTION("nested deeper than the copy constructor's descent bound")
+    {
+        // beyond nesting_depth_limit() (128) levels, the copy constructor
+        // copies without the call stack (copy_iteratively / copy_array_level),
+        // which used to assign the base class of every element it created
+        const std::size_t depth = 300;
+
+        const json_with_const_base j = make_nested_array(depth);
+        const json_with_const_base copy = j; // NOLINT(performance-unnecessary-copy-initialization)
+
+        const json_with_const_base* c = &copy;
+        for (std::size_t level = 0; level <= depth; ++level)
+        {
+            CAPTURE(level)
+            REQUIRE(c->id == 7);
+            if (level < depth)
+            {
+                c = &c->at(0);
+            }
+        }
+        CHECK(*c == 1);
+    }
+}

tests/src/unit-diagnostics.cpp

@@ -19,6 +19,7 @@ using nlohmann::json;
 
 #include <map>
 #include <unordered_map>
+#include <sstream>
 
 TEST_CASE("Better diagnostics")
 {
@@ -492,6 +493,21 @@ TEST_CASE("Regression tests for extended diagnostics")
             CHECK(copy == j);
         }
     }
+
+    SECTION("Regression test for issue #5652 - operator>> leaves a partial value in its target on a parse error")
+    {
+        json j = "old value";
+        std::istringstream is("[1, x");
+        CHECK_THROWS_WITH_AS(is >> j, "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: '1, x'", json::parse_error);
+
+        // j must be left unchanged, as json::parse() guarantees for its result
+        CHECK(j == "old value");
+
+        // copying j must not trigger assert_invariant(): a failed parse must
+        // not leave array/object elements without a parent pointer
+        json const copy = j; // NOLINT(performance-unnecessary-copy-initialization)
+        CHECK(copy == j);
+    }
 }
 
 TEST_CASE("Better diagnostics past the descent bound of update() and merge_patch()")