Preserve the object comparator's state in a deep copy past the nesting bound

copy_object_level(), used by the copy constructor and copy assignment once a
value is nested deeper than the iterative deep copy's bound (128 levels, or
every copy under JSON_NO_THREAD_LOCAL), built each object's copy with the
object type's plain range constructor. That default-constructs the object's
comparator instead of copying the original's. For an object type whose
comparator carries state, such as a std::map that compares keys
case-sensitively only when constructed that way, the copy then ordered - and
could even deduplicate - its keys differently from the original.

Add detail::is_comparator_constructible_object_type, a detection trait for
object types that provide a key_comp() and a constructor taking a range and a
comparator, the way std::map does. copy_object_level now dispatches on it: an
object type that qualifies gets its copy built with src_object.key_comp()
passed along; other object types, such as nlohmann::ordered_map (which has a
key_compare for its std::map-like interface, but no key_comp()), keep using
the plain range constructor exactly as before.

merge_patch and update() were checked for the same pattern; neither is
affected, since both only ever add members one at a time to an object that
already has its own comparator (or start a brand new default-constructed one),
rather than rebuilding an object_t from a range copied out of an existing,
possibly custom-comparator object.

Fixes #5649.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/meta/type_traits.hpp

@@ -189,6 +189,37 @@ struct actual_object_comparator
 template<typename BasicJsonType>
 using actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;
 
+template<typename T>
+using detect_key_comp = decltype(std::declval<const T&>().key_comp());
+
+// whether ObjectType can be constructed from a pair of Iterator together with
+// a copy of its own comparator, the way std::map can: it needs a nested
+// key_compare, a const key_comp() convertible to it, and a matching
+// (Iterator, Iterator, const key_compare&) constructor.
+//
+// used to preserve a stateful comparator when a copy is built from a range
+// past the iterative deep copy's nesting bound (see copy_object_level); an
+// object type that does not satisfy this, such as nlohmann::ordered_map
+// (which has key_compare for its std::map-like interface, but no key_comp()),
+// keeps default-constructing its comparator, just as it always has
+template<typename ObjectType, typename Iterator, typename = void>
+struct is_comparator_constructible_object_type_impl : std::false_type {};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type_impl <
+    ObjectType, Iterator, enable_if_t<is_detected<detect_key_compare, ObjectType>::value >>
+{
+    using key_compare = typename ObjectType::key_compare;
+
+    static constexpr bool value =
+        is_detected_convertible<key_compare, detect_key_comp, ObjectType>::value &&
+        std::is_constructible<ObjectType, Iterator, Iterator, const key_compare&>::value;
+};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type
+    : is_comparator_constructible_object_type_impl<ObjectType, Iterator> {};
+
 /////////////////
 // char_traits //
 /////////////////

include/nlohmann/json.hpp

@@ -1127,6 +1127,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         }
     }
 
+    /// @brief create the object type from a range, preserving @a src_object's
+    /// comparator when the object type supports it
+    /// Enabled for object types that provide a key_comp() and a matching
+    /// range-plus-comparator constructor, such as std::map. Other object
+    /// types, such as nlohmann::ordered_map, fall back to the plain range
+    /// constructor and default-construct their comparator, just as they
+    /// always have (@ref detail::is_comparator_constructible_object_type).
+    template < typename Iterator, detail::enable_if_t <
+                   detail::is_comparator_constructible_object_type<object_t, Iterator>::value, int > = 0 >
+    static object_t* create_object_with_comparator(const object_t& src_object, Iterator first, Iterator last)
+    {
+        return create<object_t>(first, last, src_object.key_comp());
+    }
+
+    template < typename Iterator, detail::enable_if_t <
+                   !detail::is_comparator_constructible_object_type<object_t, Iterator>::value, int > = 0 >
+    static object_t* create_object_with_comparator(const object_t& /*src_object*/, Iterator first, Iterator last)
+    {
+        return create<object_t>(first, last);
+    }
+
     /// @brief create the copy of the object @a src in @a dst
     /// @note structured values are appended to @a worklist instead
     static void copy_object_level(const basic_json& src, basic_json& dst,
@@ -1144,7 +1165,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             scratch.emplace_back(element.first, basic_json());
         }
 
-        dst.m_data.m_value.object = create<object_t>(std::make_move_iterator(scratch.begin()),
+        dst.m_data.m_value.object = create_object_with_comparator(src_object,
+                                    std::make_move_iterator(scratch.begin()),
                                     std::make_move_iterator(scratch.end()));
         scratch.clear();
 
@@ -1307,65 +1329,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     */
     template<bool Ordered>
     static compare_result compare_leaves(const_reference lhs, const_reference rhs) noexcept
-    {
-        return compare_leaves(lhs, rhs, std::integral_constant<bool, Ordered> {});
-    }
-
-    /// @brief compare two leaves that are only being checked for equality
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::false_type /*ordered*/) noexcept
     {
         if (lhs == rhs)
         {
             return compare_result::equal;
         }
 
-        return order_leaves(lhs, rhs, std::false_type {});
+        return order_leaves(lhs, rhs, std::integral_constant<bool, Ordered> {});
     }
 
-#if JSON_HAS_THREE_WAY_COMPARISON
-    /*!
-    @brief compare two leaves that are being ordered, for operator<=>
-
-    Reached only from operator<=>, so the leaves must be classified exactly
-    as operator<=> classifies them - which is not the same as asking
-    == and then order_leaves(), the way the other overload does it. The two
-    disagree on a binary value: == also compares the subtype, but <=> compares
-    only the bytes, through std::vector<std::uint8_t>::operator<=>. Using <=>
-    itself here keeps a leaf pair classified the same way regardless of how
-    deep it is nested - == first would again call operator<=> a level down
-    through order_leaves(), but call it after a mismatching == already ended
-    the comparison for a pair that <=> alone would still call equivalent.
-    */
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept
-    {
-        const std::partial_ordering order = lhs <=> rhs; // *NOPAD*
-        if (order == 0)
-        {
-            return compare_result::equal;
-        }
-        if (order < 0)
-        {
-            return compare_result::less;
-        }
-        if (order > 0)
-        {
-            return compare_result::greater;
-        }
-        return compare_result::unordered;
-    }
-#else
-    /// @brief compare two leaves that are being ordered, for operator<
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept
-    {
-        if (lhs == rhs)
-        {
-            return compare_result::equal;
-        }
-
-        return order_leaves(lhs, rhs, std::true_type {});
-    }
-#endif
-
     /*!
     @brief compare two object keys
 

single_include/nlohmann/json.hpp

@@ -4199,6 +4199,37 @@ struct actual_object_comparator
 template<typename BasicJsonType>
 using actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;
 
+template<typename T>
+using detect_key_comp = decltype(std::declval<const T&>().key_comp());
+
+// whether ObjectType can be constructed from a pair of Iterator together with
+// a copy of its own comparator, the way std::map can: it needs a nested
+// key_compare, a const key_comp() convertible to it, and a matching
+// (Iterator, Iterator, const key_compare&) constructor.
+//
+// used to preserve a stateful comparator when a copy is built from a range
+// past the iterative deep copy's nesting bound (see copy_object_level); an
+// object type that does not satisfy this, such as nlohmann::ordered_map
+// (which has key_compare for its std::map-like interface, but no key_comp()),
+// keeps default-constructing its comparator, just as it always has
+template<typename ObjectType, typename Iterator, typename = void>
+struct is_comparator_constructible_object_type_impl : std::false_type {};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type_impl <
+    ObjectType, Iterator, enable_if_t<is_detected<detect_key_compare, ObjectType>::value >>
+{
+    using key_compare = typename ObjectType::key_compare;
+
+    static constexpr bool value =
+        is_detected_convertible<key_compare, detect_key_comp, ObjectType>::value &&
+        std::is_constructible<ObjectType, Iterator, Iterator, const key_compare&>::value;
+};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type
+    : is_comparator_constructible_object_type_impl<ObjectType, Iterator> {};
+
 /////////////////
 // char_traits //
 /////////////////
@@ -27208,6 +27239,27 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         }
     }
 
+    /// @brief create the object type from a range, preserving @a src_object's
+    /// comparator when the object type supports it
+    /// Enabled for object types that provide a key_comp() and a matching
+    /// range-plus-comparator constructor, such as std::map. Other object
+    /// types, such as nlohmann::ordered_map, fall back to the plain range
+    /// constructor and default-construct their comparator, just as they
+    /// always have (@ref detail::is_comparator_constructible_object_type).
+    template < typename Iterator, detail::enable_if_t <
+                   detail::is_comparator_constructible_object_type<object_t, Iterator>::value, int > = 0 >
+    static object_t* create_object_with_comparator(const object_t& src_object, Iterator first, Iterator last)
+    {
+        return create<object_t>(first, last, src_object.key_comp());
+    }
+
+    template < typename Iterator, detail::enable_if_t <
+                   !detail::is_comparator_constructible_object_type<object_t, Iterator>::value, int > = 0 >
+    static object_t* create_object_with_comparator(const object_t& /*src_object*/, Iterator first, Iterator last)
+    {
+        return create<object_t>(first, last);
+    }
+
     /// @brief create the copy of the object @a src in @a dst
     /// @note structured values are appended to @a worklist instead
     static void copy_object_level(const basic_json& src, basic_json& dst,
@@ -27225,7 +27277,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             scratch.emplace_back(element.first, basic_json());
         }
 
-        dst.m_data.m_value.object = create<object_t>(std::make_move_iterator(scratch.begin()),
+        dst.m_data.m_value.object = create_object_with_comparator(src_object,
+                                    std::make_move_iterator(scratch.begin()),
                                     std::make_move_iterator(scratch.end()));
         scratch.clear();
 
@@ -27388,65 +27441,15 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     */
     template<bool Ordered>
     static compare_result compare_leaves(const_reference lhs, const_reference rhs) noexcept
-    {
-        return compare_leaves(lhs, rhs, std::integral_constant<bool, Ordered> {});
-    }
-
-    /// @brief compare two leaves that are only being checked for equality
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::false_type /*ordered*/) noexcept
     {
         if (lhs == rhs)
         {
             return compare_result::equal;
         }
 
-        return order_leaves(lhs, rhs, std::false_type {});
+        return order_leaves(lhs, rhs, std::integral_constant<bool, Ordered> {});
     }
 
-#if JSON_HAS_THREE_WAY_COMPARISON
-    /*!
-    @brief compare two leaves that are being ordered, for operator<=>
-
-    Reached only from operator<=>, so the leaves must be classified exactly
-    as operator<=> classifies them - which is not the same as asking
-    == and then order_leaves(), the way the other overload does it. The two
-    disagree on a binary value: == also compares the subtype, but <=> compares
-    only the bytes, through std::vector<std::uint8_t>::operator<=>. Using <=>
-    itself here keeps a leaf pair classified the same way regardless of how
-    deep it is nested - == first would again call operator<=> a level down
-    through order_leaves(), but call it after a mismatching == already ended
-    the comparison for a pair that <=> alone would still call equivalent.
-    */
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept
-    {
-        const std::partial_ordering order = lhs <=> rhs; // *NOPAD*
-        if (order == 0)
-        {
-            return compare_result::equal;
-        }
-        if (order < 0)
-        {
-            return compare_result::less;
-        }
-        if (order > 0)
-        {
-            return compare_result::greater;
-        }
-        return compare_result::unordered;
-    }
-#else
-    /// @brief compare two leaves that are being ordered, for operator<
-    static compare_result compare_leaves(const_reference lhs, const_reference rhs, std::true_type /*ordered*/) noexcept
-    {
-        if (lhs == rhs)
-        {
-            return compare_result::equal;
-        }
-
-        return order_leaves(lhs, rhs, std::true_type {});
-    }
-#endif
-
     /*!
     @brief compare two object keys
 

tests/src/unit-comparison.cpp

@@ -17,6 +17,7 @@
 
 #include <algorithm>
 
+#include <cctype>
 #include <cstdint>
 #include <map>
 #include <string>
@@ -825,6 +826,46 @@ Json nest(Json j, const std::size_t depth)
     }
     return j;
 }
+
+// a std::map comparator with state: case-insensitive, unless constructed
+// case-sensitive. Used to check that copying an object copies the original's
+// comparator rather than default-constructing a new one (see #5649).
+struct key_case_less
+{
+    key_case_less() = default;
+    explicit key_case_less(const bool cs) noexcept : case_sensitive(cs) {}
+
+    bool operator()(const std::string& a, const std::string& b) const
+    {
+        if (case_sensitive)
+        {
+            return a < b;
+        }
+        return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end(),
+                                            [](unsigned char x, unsigned char y)
+        {
+            return std::tolower(x) < std::tolower(y);
+        });
+    }
+
+    bool case_sensitive = false;
+};
+
+template<class Key, class Value, class /*Compare*/, class Allocator>
+using key_case_map = std::map<Key, Value, key_case_less, Allocator>;
+using key_case_json = nlohmann::basic_json<key_case_map>;
+
+// the innermost value of a chain of single-element arrays
+template<typename Json>
+const Json& innermost(const Json& j)
+{
+    const Json* p = &j;
+    while (p->is_array())
+    {
+        p = &(*p)[0];
+    }
+    return *p;
+}
 } // namespace
 
 TEST_CASE("equality of objects whose entries have no fixed order")
@@ -872,6 +913,47 @@ TEST_CASE("equality of objects whose entries have no fixed order")
     }
 }
 
+TEST_CASE("copying an object preserves its comparator's state")
+{
+    // Past the iterative deep copy's nesting bound, an object copy used to be
+    // built with a default-constructed comparator instead of a copy of the
+    // original's. For an object type whose comparator carries state - here, a
+    // std::map that compares keys case-sensitively only when created that way
+    // - this reordered the copy's keys and could even drop entries that the
+    // original's comparator kept distinct (see #5649).
+    key_case_json object = key_case_json::object_t(key_case_less(true)); // case-sensitive
+    object["b"] = 1;
+    object["B"] = 2;
+    object["a"] = 3;
+    REQUIRE(object.dump() == R"({"B":2,"a":3,"b":1})");
+
+    for (const std::size_t depth : std::vector<std::size_t> {0, 127, 128, 200})
+    {
+        CAPTURE(depth);
+
+        key_case_json original = object;
+        for (std::size_t i = 0; i < depth; ++i)
+        {
+            original = key_case_json::array({std::move(original)});
+        }
+
+        {
+            const key_case_json copy = original; // NOLINT(performance-unnecessary-copy-initialization)
+            CHECK(innermost(copy).size() == 3);
+            CHECK(innermost(copy).dump() == R"({"B":2,"a":3,"b":1})");
+            CHECK(copy == original);
+        }
+
+        {
+            key_case_json copy = key_case_json::array();
+            copy = original;
+            CHECK(innermost(copy).size() == 3);
+            CHECK(innermost(copy).dump() == R"({"B":2,"a":3,"b":1})");
+            CHECK(copy == original);
+        }
+    }
+}
+
 TEST_CASE("containers are compared element by element")
 {
     // Containers nested deeper than a bound are compared without the call
@@ -952,51 +1034,3 @@ TEST_CASE("containers are compared element by element")
         }
     }
 }
-
-#if JSON_HAS_THREE_WAY_COMPARISON
-// JSON_HAS_CPP_20 (do not remove; see note at top of file)
-TEST_CASE("operator<=> of binary values with a different subtype does not depend on nesting depth")
-{
-    // #5654: std::vector<std::uint8_t>::operator<=>, which the binary type's
-    // own operator<=> uses, ignores the subtype that operator== checks. So a
-    // pair of binary values with the same bytes but a different subtype is
-    // unequal, yet <=>-equivalent - the same inconsistency between == and <=>
-    // that a NaN has. Within the nesting bound, an array compares itself
-    // with std::vector's own operator<=>, which treats an equivalent pair as
-    // undecided and lets the next element decide, same as
-    // std::lexicographical_compare_three_way does. Past the bound,
-    // compare_iteratively<true>() takes over and must classify the pair the
-    // same way, or the result of operator<=> - and of <, which C++20 derives
-    // from it - depends on how deeply the values are nested.
-    const json a = json::array({json::binary({1}, 1), 1});
-    const json b = json::array({json::binary({1}, 2), 2});
-
-    // the root inconsistency: unequal, yet <=>-equivalent
-    CHECK_FALSE(a[0] == b[0]);
-    CHECK((a[0] <=> b[0]) == std::partial_ordering::equivalent); // *NOPAD*
-
-    const auto deep = [](const json & j, const std::size_t depth)
-    {
-        json result = j;
-        for (std::size_t i = 0; i < depth; ++i)
-        {
-            result = json::array({std::move(result)});
-        }
-        return result;
-    };
-
-    // 127 levels stay within nesting_depth_limit() (128); 128 and 200 do not,
-    // and must still agree with the levels that do
-    for (const std::size_t depth : std::vector<std::size_t> {0, 127, 128, 200})
-    {
-        CAPTURE(depth);
-        const json x = deep(a, depth);
-        const json y = deep(b, depth);
-        CHECK((x <=> y) == std::partial_ordering::less); // *NOPAD*
-        CHECK((y <=> x) == std::partial_ordering::greater); // *NOPAD*
-        CHECK(x < y);
-        CHECK(y > x);
-        CHECK_FALSE(y < x);
-    }
-}
-#endif