diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index f26412a9e..da837e048 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -16,7 +16,6 @@ #include // ostream #endif // JSON_NO_IO #include // max -#include // map #include // string #include // move #include // vector @@ -499,21 +498,73 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} @@ -521,65 +572,67 @@ class json_pointer // reference_token, adding it if it is new std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } - - // shared prefixes are found without copying the token - const auto found = children[id].find(reference_token); - if (found != children[id].end()) - { - return found->second; - } - - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t child = children.size(); - children[id].emplace(reference_token, child); - children.emplace_back(); + parent.push_back(id); is_array.push_back(false); - return child; - } - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) + { + grow(); + } + return child; } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -623,8 +676,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -1257,25 +1308,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) - // (each key is parsed once and the pointer kept for the second pass) prefix_tree tree; - std::vector pointers; - pointers.reserve(value.m_data.m_value.object->size()); - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - pointers.emplace_back(element.first); - std::size_t id = 0; - for (const auto& reference_token : pointers.back().reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, reference_token); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values - std::size_t index = 0; + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -1283,11 +1362,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - pointers[index++].get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 5983d49c8..7a3b0a2f6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20642,7 +20642,6 @@ NLOHMANN_JSON_NAMESPACE_END #include // ostream #endif // JSON_NO_IO #include // max -#include // map #include // string #include // move #include // vector @@ -21131,21 +21130,73 @@ class json_pointer the result independent of the order in which the flattened object is iterated, which is unspecified for some object types. - The prefixes form a tree and are numbered, so each of them is stored only - once (as a node) rather than as a copy of all of its reference tokens. + The prefixes form a tree and are numbered (number 0 is the empty prefix), + so each of them is stored only once (as a node) rather than as a copy of + all of its reference tokens. The tree does not own the reference tokens: + it refers to the tokens of the parsed keys, which must outlive it. */ struct prefix_tree { - // children[id] maps a reference token to the number of the prefix - // extended by that token; number 0 is the empty prefix - std::vector> children; + // open-addressing hash table from (parent, last reference token) to + // the number of the node; empty slots have token == nullptr + struct slot + { + std::size_t hash; + std::size_t parent; + const string_t* token; + std::size_t id; + + slot() noexcept + : hash(0), parent(0), token(nullptr), id(0) + {} + + slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept + : hash(h), parent(p), token(t), id(i) + {} + }; + + std::vector slots; + std::size_t used = 0; + + static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept + { + std::size_t h = parent * static_cast(0x9E3779B97F4A7C15ULL); + for (std::size_t i = 0; i < token.size(); ++i) + { + h = (h ^ static_cast(token[i])) * static_cast(0x100000001B3ULL); + } + return h ^ (h >> 29u); + } + + void grow() + { + std::vector old(slots.size() * 2); + old.swap(slots); + const std::size_t mask = slots.size() - 1; + for (const auto& s : old) + { + if (s.token != nullptr) + { + std::size_t i = s.hash & mask; + while (slots[i].token != nullptr) + { + i = (i + 1) & mask; + } + slots[i] = s; + } + } + } + + // parent[id] is the number of the prefix without its last token + std::vector parent; // is_array[id] is true iff some flattened key has the reference token // 0 directly below the prefix with number id std::vector is_array; // start with the empty prefix only prefix_tree() - : children(1) + : slots(16) + , parent(1, 0) , is_array(1, false) {} @@ -21153,65 +21204,67 @@ class json_pointer // reference_token, adding it if it is new std::size_t add_child(std::size_t id, const string_t& reference_token) { + const std::size_t hash = hash_of(id, reference_token); + const std::size_t mask = slots.size() - 1; + std::size_t i = hash & mask; + while (slots[i].token != nullptr) + { + if (slots[i].hash == hash && slots[i].parent == id && *slots[i].token == reference_token) + { + return slots[i].id; + } + i = (i + 1) & mask; + } + + // new prefix + const std::size_t child = parent.size(); + slots[i] = slot(hash, id, &reference_token, child); if (reference_token == "0") { is_array[id] = true; } - - // shared prefixes are found without copying the token - const auto found = children[id].find(reference_token); - if (found != children[id].end()) - { - return found->second; - } - - // read the number before the emplace_back below, which may - // reallocate children and invalidate the iterator - const std::size_t child = children.size(); - children[id].emplace(reference_token, child); - children.emplace_back(); + parent.push_back(id); is_array.push_back(false); - return child; - } - // return the number of the prefix with number id extended by - // reference_token, which must have been added before - std::size_t find_child(std::size_t id, const string_t& reference_token) const - { - const auto it = children[id].find(reference_token); - JSON_ASSERT(it != children[id].end()); - return it->second; + // keep the load factor below 1/2 + if (++used * 2 > slots.size()) + { + grow(); + } + return child; } }; /*! @brief create and return a reference to the pointed to value - Complexity: Linear in the number of reference tokens (times the logarithm - of the number of siblings for the prefix lookup). + @param[in] j the value to create the pointed to value in + @param[in] tokens the reference tokens of the pointer + @param[in] prefixes prefixes[i] is the number in @a tree of the prefix + consisting of the first i reference tokens + + Complexity: Linear in the number of reference tokens. @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j, const prefix_tree& tree) const + static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector& tokens, + const std::vector& prefixes, const prefix_tree& tree) { auto* result = &j; - // the number of the prefix consumed so far; used to look up whether - // the value to be created below is an array or an object - std::size_t id = 0; - // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) + for (std::size_t i = 0; i < tokens.size(); ++i) { + const auto& reference_token = tokens[i]; switch (result->type()) { case detail::value_t::null: { - if (tree.is_array[id]) + if (tree.is_array[prefixes[i]]) { // some reference token below this position is 0, so the // value is an array @@ -21255,8 +21308,6 @@ class json_pointer default: JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j)); } - - id = tree.find_child(id, reference_token); } return *result; @@ -21889,25 +21940,53 @@ class json_pointer BasicJsonType result; + // parse every key once; the reference tokens are kept for the second + // pass (the vectors are never resized, so the tree may refer to them) + std::vector> keys; + keys.reserve(value.m_data.m_value.object->size()); + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + keys.push_back(std::move(ptr.reference_tokens)); + } + // collect the pointer prefixes that have a reference token 0 among // their children; the values below them are arrays, all others are // objects (see prefix_tree) - // (each key is parsed once and the pointer kept for the second pass) prefix_tree tree; - std::vector pointers; - pointers.reserve(value.m_data.m_value.object->size()); - for (const auto& element : *value.m_data.m_value.object) + // the number of the prefix with i + 1 tokens of the previous key; keys + // are often sorted, so most of the prefix is the same as for the + // previous key and need not be looked up again + std::vector path; + const std::vector* previous = nullptr; + // the number of the complete prefix (i.e., the key) of each key + std::vector leaves; + leaves.reserve(keys.size()); + for (const auto& tokens : keys) { - pointers.emplace_back(element.first); - std::size_t id = 0; - for (const auto& reference_token : pointers.back().reference_tokens) + std::size_t common = 0; + if (previous != nullptr) { - id = tree.add_child(id, reference_token); + const std::size_t limit = (std::min)(tokens.size(), previous->size()); + while (common < limit && tokens[common] == (*previous)[common]) + { + ++common; + } } + path.resize(common); + std::size_t id = common == 0 ? 0 : path[common - 1]; + for (std::size_t i = common; i < tokens.size(); ++i) + { + id = tree.add_child(id, tokens[i]); + path.push_back(id); + } + leaves.push_back(id); + previous = &tokens; } // iterate the JSON object values - std::size_t index = 0; + std::vector prefixes; + std::size_t key_index = 0; for (const auto& element : *value.m_data.m_value.object) { if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) @@ -21915,11 +21994,21 @@ class json_pointer JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); } + const auto& tokens = keys[key_index]; + // number the prefixes of this key from its last one to the first + prefixes.resize(tokens.size()); + for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) + { + id = tree.parent[id]; + prefixes[i - 1] = id; + } + ++key_index; + // Assign the value to the reference pointed to by JSON pointer. Note // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - pointers[index++].get_and_create(result, tree) = element.second; + get_and_create(result, tokens, prefixes, tree) = element.second; } return result;