diff --git a/BUILD.bazel b/BUILD.bazel index ea8ffae21..b13e62c22 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -58,6 +58,7 @@ cc_library( "include/nlohmann/detail/output/binary_writer.hpp", "include/nlohmann/detail/output/output_adapters.hpp", "include/nlohmann/detail/output/serializer.hpp", + "include/nlohmann/detail/recursion_depth_limit.hpp", "include/nlohmann/detail/string_concat.hpp", "include/nlohmann/detail/string_escape.hpp", "include/nlohmann/detail/string_utils.hpp", diff --git a/include/nlohmann/detail/hash.hpp b/include/nlohmann/detail/hash.hpp index be8063f89..20a971886 100644 --- a/include/nlohmann/detail/hash.hpp +++ b/include/nlohmann/detail/hash.hpp @@ -11,8 +11,10 @@ #include // uint8_t #include // size_t #include // hash +#include // vector #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -26,6 +28,9 @@ inline std::size_t combine(std::size_t seed, std::size_t h) noexcept return seed; } +template +std::size_t hash_iteratively(const BasicJsonType& j); + /*! @brief hash a JSON value @@ -33,12 +38,21 @@ The hash function tries to rely on std::hash where possible. Furthermore, the type of the JSON value is taken into account to have different hash values for null, 0, 0U, and false, etc. +Hashing an array or an object hashes its elements, which used to call this +function again once per nesting level, so a value nested deeply enough +exhausted the call stack and terminated the process. The descent is bounded +here: once @ref recursion_depth_limit levels have been entered, @ref +hash_iteratively hashes what is left without the call stack. A value nested +less deeply than that - all but a vanishing minority - is hashed exactly as +before, without allocating. + @tparam BasicJsonType basic_json specialization @param j JSON value to hash +@param depth nesting level of @a j, counted from the value passed by the caller @return hash value of j */ template -std::size_t hash(const BasicJsonType& j) +std::size_t hash(const BasicJsonType& j, const std::size_t depth = 0) { using string_t = typename BasicJsonType::string_t; using number_integer_t = typename BasicJsonType::number_integer_t; @@ -56,22 +70,32 @@ std::size_t hash(const BasicJsonType& j) case BasicJsonType::value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + { + return hash_iteratively(j); + } + auto seed = combine(type, j.size()); for (const auto& element : j.items()) { const auto h = std::hash {}(element.key()); seed = combine(seed, h); - seed = combine(seed, hash(element.value())); + seed = combine(seed, hash(element.value(), depth + 1)); } return seed; } case BasicJsonType::value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + { + return hash_iteratively(j); + } + auto seed = combine(type, j.size()); for (const auto& element : j) { - seed = combine(seed, hash(element)); + seed = combine(seed, hash(element, depth + 1)); } return seed; } @@ -127,5 +151,77 @@ std::size_t hash(const BasicJsonType& j) } } +/// an array or object whose elements @ref hash_iteratively is hashing +template +struct hash_frame +{ + hash_frame(const BasicJsonType* value_, std::size_t seed_) noexcept + : value(value_), position(value_->cbegin()), seed(seed_) + {} + + const BasicJsonType* value; + typename BasicJsonType::const_iterator position; + std::size_t seed; +}; + +/*! +@brief hash the array or object @a j without the call stack + +Computes the same value as @ref hash, keeping the arrays and objects it has +entered on an explicit stack instead of descending into them. Only reached for +values nested deeper than @ref recursion_depth_limit. + +@tparam BasicJsonType basic_json specialization +@param j array or object to hash +@return hash value of j +*/ +template +std::size_t hash_iteratively(const BasicJsonType& j) +{ + using string_t = typename BasicJsonType::string_t; + + std::vector> stack; + stack.emplace_back(&j, combine(static_cast(j.type()), j.size())); + + while (true) + { + // a copy, as entering an element below can reallocate the stack; the + // frame itself is only changed through stack.back() + const hash_frame frame = stack.back(); + + if (frame.position == frame.value->cend()) + { + // all elements are hashed: fold this value's hash into its parent's + // seed, exactly where the recursive version returns it + const std::size_t h = frame.seed; + stack.pop_back(); + if (stack.empty()) + { + return h; + } + stack.back().seed = combine(stack.back().seed, h); + continue; + } + + if (frame.value->is_object()) + { + stack.back().seed = combine(stack.back().seed, std::hash {}(frame.position.key())); + } + + // advance before entering the element, which pushes onto the stack + const BasicJsonType& element = *frame.position; + ++stack.back().position; + + if (element.is_structured()) + { + stack.emplace_back(&element, combine(static_cast(element.type()), element.size())); + } + else + { + stack.back().seed = combine(stack.back().seed, hash(element)); + } + } +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index f9e7f7840..7c38276ce 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -133,7 +134,7 @@ class serializer Serializing a container descends into its elements, so a value nested deeply enough used to exhaust the call stack and terminate the process with no - exception to catch. The descent is bounded here: once @ref dump_depth_limit + exception to catch. The descent is bounded here: once @ref recursion_depth_limit levels have been entered, @ref dump_iteratively writes out what is left without the call stack. A value nested less deeply than that - all but a vanishing minority - is written by exactly the code that always wrote it. @@ -148,7 +149,7 @@ class serializer { case value_t::object: { - if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { dump_iteratively(val, current_indent); return; @@ -223,7 +224,7 @@ class serializer case value_t::array: { - if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { dump_iteratively(val, current_indent); return; @@ -408,19 +409,12 @@ class serializer } private: - /// the number of levels @ref dump_internal descends into before it hands - /// over to @ref dump_iteratively - static constexpr std::size_t dump_depth_limit() - { - return 128; - } - /*! @brief write out @a val and everything below it without the call stack Emits the same bytes as @ref dump_internal, keeping the containers it has entered on an explicit stack instead of descending into them. Only reached - for values nested deeper than @ref dump_depth_limit, which is why it is not + for values nested deeper than @ref recursion_depth_limit, which is why it is not written for speed: walking every value this way measured up to 20% slower on object-heavy documents than letting the compiler drive the descent. */ diff --git a/include/nlohmann/detail/recursion_depth_limit.hpp b/include/nlohmann/detail/recursion_depth_limit.hpp new file mode 100644 index 000000000..fe3bd8026 --- /dev/null +++ b/include/nlohmann/detail/recursion_depth_limit.hpp @@ -0,0 +1,35 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // size_t + +#include + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the number of nesting levels an operation recurses into + +Operations that walk a value (serializing, hashing, merging, ...) recurse once +per nesting level, which is fastest, but a value nested deeply enough would +exhaust the call stack. So they recurse only this many levels deep and finish +whatever lies below with an explicit stack. All of them share this limit. + +@sa https://github.com/nlohmann/json/issues/5387 +*/ +constexpr std::size_t recursion_depth_limit() noexcept +{ + return 128; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 9c1d821e1..b64feb4a9 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -68,6 +68,7 @@ #include #include #include +#include #include #include #include diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c11fa27e5..335a29289 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7028,9 +7028,48 @@ NLOHMANN_JSON_NAMESPACE_END #include // uint8_t #include // size_t #include // hash +#include // vector // #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ + +/*! +@brief the number of nesting levels an operation recurses into + +Operations that walk a value (serializing, hashing, merging, ...) recurse once +per nesting level, which is fastest, but a value nested deeply enough would +exhaust the call stack. So they recurse only this many levels deep and finish +whatever lies below with an explicit stack. All of them share this limit. + +@sa https://github.com/nlohmann/json/issues/5387 +*/ +constexpr std::size_t recursion_depth_limit() noexcept +{ + return 128; +} + +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + // #include @@ -7045,6 +7084,9 @@ inline std::size_t combine(std::size_t seed, std::size_t h) noexcept return seed; } +template +std::size_t hash_iteratively(const BasicJsonType& j); + /*! @brief hash a JSON value @@ -7052,12 +7094,21 @@ The hash function tries to rely on std::hash where possible. Furthermore, the type of the JSON value is taken into account to have different hash values for null, 0, 0U, and false, etc. +Hashing an array or an object hashes its elements, which used to call this +function again once per nesting level, so a value nested deeply enough +exhausted the call stack and terminated the process. The descent is bounded +here: once @ref recursion_depth_limit levels have been entered, @ref +hash_iteratively hashes what is left without the call stack. A value nested +less deeply than that - all but a vanishing minority - is hashed exactly as +before, without allocating. + @tparam BasicJsonType basic_json specialization @param j JSON value to hash +@param depth nesting level of @a j, counted from the value passed by the caller @return hash value of j */ template -std::size_t hash(const BasicJsonType& j) +std::size_t hash(const BasicJsonType& j, const std::size_t depth = 0) { using string_t = typename BasicJsonType::string_t; using number_integer_t = typename BasicJsonType::number_integer_t; @@ -7075,22 +7126,32 @@ std::size_t hash(const BasicJsonType& j) case BasicJsonType::value_t::object: { + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + { + return hash_iteratively(j); + } + auto seed = combine(type, j.size()); for (const auto& element : j.items()) { const auto h = std::hash {}(element.key()); seed = combine(seed, h); - seed = combine(seed, hash(element.value())); + seed = combine(seed, hash(element.value(), depth + 1)); } return seed; } case BasicJsonType::value_t::array: { + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) + { + return hash_iteratively(j); + } + auto seed = combine(type, j.size()); for (const auto& element : j) { - seed = combine(seed, hash(element)); + seed = combine(seed, hash(element, depth + 1)); } return seed; } @@ -7146,6 +7207,78 @@ std::size_t hash(const BasicJsonType& j) } } +/// an array or object whose elements @ref hash_iteratively is hashing +template +struct hash_frame +{ + hash_frame(const BasicJsonType* value_, std::size_t seed_) noexcept + : value(value_), position(value_->cbegin()), seed(seed_) + {} + + const BasicJsonType* value; + typename BasicJsonType::const_iterator position; + std::size_t seed; +}; + +/*! +@brief hash the array or object @a j without the call stack + +Computes the same value as @ref hash, keeping the arrays and objects it has +entered on an explicit stack instead of descending into them. Only reached for +values nested deeper than @ref recursion_depth_limit. + +@tparam BasicJsonType basic_json specialization +@param j array or object to hash +@return hash value of j +*/ +template +std::size_t hash_iteratively(const BasicJsonType& j) +{ + using string_t = typename BasicJsonType::string_t; + + std::vector> stack; + stack.emplace_back(&j, combine(static_cast(j.type()), j.size())); + + while (true) + { + // a copy, as entering an element below can reallocate the stack; the + // frame itself is only changed through stack.back() + const hash_frame frame = stack.back(); + + if (frame.position == frame.value->cend()) + { + // all elements are hashed: fold this value's hash into its parent's + // seed, exactly where the recursive version returns it + const std::size_t h = frame.seed; + stack.pop_back(); + if (stack.empty()) + { + return h; + } + stack.back().seed = combine(stack.back().seed, h); + continue; + } + + if (frame.value->is_object()) + { + stack.back().seed = combine(stack.back().seed, std::hash {}(frame.position.key())); + } + + // advance before entering the element, which pushes onto the stack + const BasicJsonType& element = *frame.position; + ++stack.back().position; + + if (element.is_structured()) + { + stack.emplace_back(&element, combine(static_cast(element.type()), element.size())); + } + else + { + stack.back().seed = combine(stack.back().seed, hash(element)); + } + } +} + } // namespace detail NLOHMANN_JSON_NAMESPACE_END @@ -22295,6 +22428,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -22400,7 +22535,7 @@ class serializer Serializing a container descends into its elements, so a value nested deeply enough used to exhaust the call stack and terminate the process with no - exception to catch. The descent is bounded here: once @ref dump_depth_limit + exception to catch. The descent is bounded here: once @ref recursion_depth_limit levels have been entered, @ref dump_iteratively writes out what is left without the call stack. A value nested less deeply than that - all but a vanishing minority - is written by exactly the code that always wrote it. @@ -22415,7 +22550,7 @@ class serializer { case value_t::object: { - if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { dump_iteratively(val, current_indent); return; @@ -22490,7 +22625,7 @@ class serializer case value_t::array: { - if (JSON_HEDLEY_UNLIKELY(depth >= dump_depth_limit())) + if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit())) { dump_iteratively(val, current_indent); return; @@ -22675,19 +22810,12 @@ class serializer } private: - /// the number of levels @ref dump_internal descends into before it hands - /// over to @ref dump_iteratively - static constexpr std::size_t dump_depth_limit() - { - return 128; - } - /*! @brief write out @a val and everything below it without the call stack Emits the same bytes as @ref dump_internal, keeping the containers it has entered on an explicit stack instead of descending into them. Only reached - for values nested deeper than @ref dump_depth_limit, which is why it is not + for values nested deeper than @ref recursion_depth_limit, which is why it is not written for speed: walking every value this way measured up to 20% slower on object-heavy documents than letting the compiler drive the descent. */ @@ -24000,6 +24128,8 @@ class serializer } // namespace detail NLOHMANN_JSON_NAMESPACE_END +// #include + // #include // #include diff --git a/tests/src/unit-hash.cpp b/tests/src/unit-hash.cpp index c161efa6e..eb843c291 100644 --- a/tests/src/unit-hash.cpp +++ b/tests/src/unit-hash.cpp @@ -13,6 +13,78 @@ using json = nlohmann::json; using ordered_json = nlohmann::ordered_json; #include +#include + +namespace +{ +// how detail::hash defines the hash of an array or object: the seeds of the +// elements, combined in order. Recursive, so only usable on values nested a +// few hundred levels deep - which is exactly what is needed to check that the +// iterative path taken below detail::recursion_depth_limit() computes the same. +template +std::size_t reference_hash(const BasicJsonType& j) +{ + using nlohmann::detail::combine; + using string_t = typename BasicJsonType::string_t; + + if (!j.is_structured()) + { + return std::hash {}(j); + } + + auto seed = combine(static_cast(j.type()), j.size()); + for (const auto& element : j.items()) + { + if (j.is_object()) + { + seed = combine(seed, std::hash {}(element.key())); + } + seed = combine(seed, reference_hash(element.value())); + } + return seed; +} + +// a value nested `depth` levels deep, with siblings on every level +template +BasicJsonType nested(const std::size_t depth, const bool objects) +{ + BasicJsonType value = "leaf"; + for (std::size_t i = 0; i < depth; ++i) + { + if (objects) + { + value = BasicJsonType{{"before", i}, {"nested", std::move(value)}, {"after", {i, "x"}}}; + } + else + { + value = BasicJsonType::array({i, std::move(value), BasicJsonType::object({{"k", i}})}); + } + } + return value; +} + +std::string nested_text(const std::size_t depth, const bool objects) +{ + std::string text; + if (objects) + { + text.reserve((6 * depth) + 1); + for (std::size_t i = 0; i < depth; ++i) + { + text += "{\"a\":"; + } + text += "1"; + text.append(depth, '}'); + } + else + { + text.assign(depth, '['); + text += "1"; + text.append(depth, ']'); + } + return text; +} +} // namespace TEST_CASE("hash") { @@ -111,3 +183,44 @@ TEST_CASE("hash") CHECK(hashes.size() == 21); } + +TEST_CASE("hash of deeply nested values") +{ + SECTION("hashing past the descent bound computes the same values") + { + // every depth on either side of where the iterative path takes over + for (std::size_t depth = 0; depth <= (2 * nlohmann::detail::recursion_depth_limit()) + 10; ++depth) + { + CAPTURE(depth); + const auto arrays = nested(depth, false); + const auto objects = nested(depth, true); + const auto ordered = nested(depth, true); + CHECK(std::hash {}(arrays) == reference_hash(arrays)); + CHECK(std::hash {}(objects) == reference_hash(objects)); + CHECK(std::hash {}(ordered) == reference_hash(ordered)); + } + } + + SECTION("values nested too deeply for the call stack (#5545)") + { + // recursing once per level used to exhaust the call stack here; the + // values are only parsed and hashed, never copied or compared, since + // those recurse as well + const std::size_t depth = 100000; + for (const bool objects : + { + false, true + }) + { + CAPTURE(objects); + const auto text = nested_text(depth, objects); + const auto a = json::parse(text); + const auto b = json::parse(text); + CHECK(std::hash {}(a) == std::hash {}(b)); + + const auto c = ordered_json::parse(text); + const auto d = ordered_json::parse(text); + CHECK(std::hash {}(c) == std::hash {}(d)); + } + } +}