amorfo77/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-21 05:28:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Unflatten without splitting keys or walking each path from the root

Browse Source 
unflatten() split every key into a vector of reference tokens, stored the
pointer prefixes in a tree to decide which values become arrays, and then
looked up every token of every key from the root again.

Keys are now checked by a scan that raises the same parse_error.107/108 as
split(), skipping the part a key shares with the previous one, which was
checked already. The prefixes that denote arrays are kept in a small hash
set of key slices; the array/object decision is unchanged. The walk keeps
the values along the previous key's path, so only the part of a key that
differs from the previous one is looked up. Keys of a flattened object are
usually sorted, so that part is short.

unflatten() of canada.json, citm_catalog.json and twitter.json is 1.9x to
2.3x as fast; the output and the exceptions are unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-20 07:47:34 +02:00
1 parent c106e60584
commit 4f8ac26088
2 files changed
+546 -378
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

include/nlohmann/detail/json_pointer.hpp
+273 -189
View File
@@ -12,7 +12,9 @@
#include <cctype> // isdigit #include <cctype> // isdigit
#include <cerrno> // errno, ERANGE #include <cerrno> // errno, ERANGE
#include <cstddef> // size_t #include <cstddef> // size_t
#include <cstdint> // uint64_t
#include <cstdlib> // strtoull #include <cstdlib> // strtoull
#include <cstring> // memcmp, memcpy
#ifndef JSON_NO_IO #ifndef JSON_NO_IO
#include <iosfwd> // ostream #include <iosfwd> // ostream
#endif // JSON_NO_IO #endif // JSON_NO_IO
@@ -492,50 +494,111 @@ class json_pointer
private: private:
/*! /*!
@brief the pointer prefixes of a flattened object, and which of them denote arrays @brief the pointer prefixes of a flattened object that denote arrays
@ref unflatten collects the pointer prefixes that have a reference token 0 @ref unflatten creates arrays exactly below the prefixes that have a
among their children; @ref get_and_create creates arrays exactly below reference token 0 among their children, and objects everywhere else.
those prefixes and objects everywhere else. Deciding this up front keeps Deciding this up front keeps the result independent of the order in which
the result independent of the order in which the flattened object is the flattened object is iterated, which is unspecified for some object
iterated, which is unspecified for some object types. types.
The prefixes form a tree and are numbered (number 0 is the empty prefix), A prefix is identified by the text of the key it is the beginning of:
so each of them is stored only once (as a node) rather than as a copy of escaping is unambiguous, so equal prefixes of valid keys are equal texts.
all of its reference tokens. The tree does not own the reference tokens: The set refers to the keys, which must outlive it.
it refers to the tokens of the parsed keys, which must outlive it.
*/ */
struct prefix_tree class array_prefix_set
{ {
// open-addressing hash table from (parent, last reference token) to public:
// the number of the node; empty slots have token == nullptr void insert(const char* data, std::size_t size)
{
// flattened objects are usually sorted, so most keys that have a
// 0 yield the prefix of the previous one
if (last_data != nullptr && last_size == size && std::memcmp(last_data, data, size) == 0)
{
return;
}
last_data = data;
last_size = size;
if (table.empty())
{
table.resize(16);
}
const std::size_t hash = hash_of(data, size);
const std::size_t mask = table.size() - 1;
std::size_t i = hash & mask;
while (table[i].data != nullptr)
{
if (table[i].hash == hash && table[i].size == size && std::memcmp(table[i].data, data, size) == 0)
{
return;
}
i = (i + 1) & mask;
}
table[i] = slot(hash, data, size);
// keep the load factor below 1/2
if (++used * 2 > table.size())
{
grow();
}
}
bool empty() const noexcept
{
return used == 0;
}
bool contains(const char* data, std::size_t size) const noexcept
{
if (used == 0)
{
return false;
}
const std::size_t hash = hash_of(data, size);
const std::size_t mask = table.size() - 1;
for (std::size_t i = hash & mask; table[i].data != nullptr; i = (i + 1) & mask)
{
if (table[i].hash == hash && table[i].size == size && std::memcmp(table[i].data, data, size) == 0)
{
return true;
}
}
return false;
}
private:
struct slot struct slot
{ {
std::size_t hash; slot() noexcept = default;
std::size_t parent;
const string_t* token;
std::size_t id;
slot() noexcept slot(std::size_t h, const char* d, std::size_t s) noexcept
: hash(0), parent(0), token(nullptr), id(0) : hash(h), data(d), size(s)
{} {}
slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept std::size_t hash = 0;
: hash(h), parent(p), token(t), id(i) const char* data = nullptr; // nullptr: the slot is empty
{} std::size_t size = 0;
}; };
std::vector<slot> table; static std::size_t hash_of(const char* data, std::size_t size) noexcept
std::size_t used = 0;
static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept
{ {
std::size_t h = parent * static_cast<std::size_t>(0x9E3779B97F4A7C15ULL & (std::numeric_limits<std::size_t>::max)()); std::uint64_t h = 0x9E3779B97F4A7C15ULL ^ size;
for (std::size_t i = 0; i < token.size(); ++i) std::size_t i = 0;
for (; i + 8 <= size; i += 8)
{ {
h = (h ^ static_cast<unsigned char>(token[i])) * static_cast<std::size_t>(0x100000001B3ULL & (std::numeric_limits<std::size_t>::max)()); std::uint64_t w = 0;
std::memcpy(&w, data + i, 8);
h = (h ^ w) * 0x100000001B3ULL;
h ^= h >> 32u;
} }
return h ^ (h >> 29u); std::uint64_t w = 0;
if (i < size)
{
std::memcpy(&w, data + i, size - i);
}
h = (h ^ w) * 0x9E3779B97F4A7C15ULL;
return static_cast<std::size_t>(h ^ (h >> 29u));
} }
void grow() void grow()
@@ -545,10 +608,10 @@ class json_pointer
const std::size_t mask = table.size() - 1; const std::size_t mask = table.size() - 1;
for (const auto& s : old) for (const auto& s : old)
{ {
if (s.token != nullptr) if (s.data != nullptr)
{ {
std::size_t i = s.hash & mask; std::size_t i = s.hash & mask;
while (table[i].token != nullptr) while (table[i].data != nullptr)
{ {
i = (i + 1) & mask; i = (i + 1) & mask;
} }
@@ -557,130 +620,95 @@ class json_pointer
} }
} }
// parent[id] is the number of the prefix without its last token std::vector<slot> table{};
std::vector<std::size_t> parent; std::size_t used = 0;
// is_array[id] is true iff some flattened key has the reference token const char* last_data = nullptr;
// 0 directly below the prefix with number id std::size_t last_size = 0;
std::vector<bool> is_array;
// start with the empty prefix only
prefix_tree()
: table(16)
, parent(1, 0)
, is_array(1, false)
{}
// return the number of the prefix with number id extended by
// reference_token, adding it if it is new
std::size_t add_child(std::size_t id, const string_t& reference_token)
{
const std::size_t hash = hash_of(id, reference_token);
const std::size_t mask = table.size() - 1;
std::size_t i = hash & mask;
while (table[i].token != nullptr)
{
if (table[i].hash == hash && table[i].parent == id && *table[i].token == reference_token)
{
return table[i].id;
}
i = (i + 1) & mask;
}
// new prefix
const std::size_t child = parent.size();
table[i] = slot(hash, id, &reference_token, child);
if (reference_token == "0")
{
is_array[id] = true;
}
parent.push_back(id);
is_array.push_back(false);
// keep the load factor below 1/2
if (++used * 2 > table.size())
{
grow();
}
return child;
}
}; };
/// @brief the length of the longest common prefix of the first @a limit bytes of @a a and @a b
static std::size_t common_prefix_length(const char* a, const char* b, const std::size_t limit) noexcept
{
std::size_t i = 0;
for (; i + 8 <= limit; i += 8)
{
std::uint64_t x = 0;
std::uint64_t y = 0;
std::memcpy(&x, a + i, 8);
std::memcpy(&y, b + i, 8);
if (x != y)
{
break;
}
}
while (i < limit && a[i] == b[i])
{
++i;
}
return i;
}
/*! /*!
@brief create and return a reference to the pointed to value @brief check that @a key is a valid JSON pointer, and add the prefixes of
it that denote arrays to @a arrays
@param[in] j the value to create the pointed to value in @throw parse_error.107 if the key is not empty or begins with '/'
@param[in] tokens the reference tokens of the pointer @throw parse_error.108 if character '~' is not followed by '0' or '1'
@param[in] prefixes prefixes[i] is the number in @a tree of the prefix
consisting of the first i reference tokens
Complexity: Linear in the number of reference tokens. The same checks as @ref split, which throws the same exceptions, but
without building the reference tokens.
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if array index is not a number
@throw type_error.313 if value cannot be unflattened
*/ */
template<typename BasicJsonType> static void scan_key(const string_t& key, const string_t* previous, array_prefix_set& arrays)
static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector<string_t>& tokens,
const std::vector<std::size_t>& prefixes, const prefix_tree& tree)
{ {
auto* result = &j; const std::size_t size = key.size();
if (size == 0)
// in case no reference tokens exist, return a reference to the JSON value
// j which will be overwritten by a primitive value
for (std::size_t i = 0; i < tokens.size(); ++i)
{ {
const auto& reference_token = tokens[i]; return;
switch (result->type())
{
case detail::value_t::null:
{
if (tree.is_array[prefixes[i]])
{
// some reference token below this position is 0, so the
// value is an array
result = &result->operator[](array_index<BasicJsonType>(reference_token));
}
else
{
// start a new object otherwise
result = &result->operator[](reference_token);
}
break;
} }
case detail::value_t::object: const char* const data = key.data();
if (JSON_HEDLEY_UNLIKELY(data[0] != '/'))
{ {
// create an entry in the object JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", key, "'"), nullptr));
result = &object_member(*result, reference_token);
break;
} }
case detail::value_t::array: // The beginning of the key that it has in common with the previous
// one, which was checked already, is skipped: the scan starts with
// the token after the last slash in it.
std::size_t begin = 1;
if (previous != nullptr)
{ {
// create an entry in the array std::size_t same = common_prefix_length(data, previous->data(), (std::min)(size, previous->size()));
result = &result->operator[](array_index<BasicJsonType>(reference_token)); while (same > 1 && data[same - 1] != '/')
break; {
--same;
} }
if (same > 1)
/* {
The following code is only reached if there exists a reference begin = same;
token _and_ the current value is primitive. In this case, we have
an error situation, because primitive values may only occur as
a single value; that is, with an empty list of reference tokens.
*/
case detail::value_t::string:
case detail::value_t::boolean:
case detail::value_t::number_integer:
case detail::value_t::number_unsigned:
case detail::value_t::number_float:
case detail::value_t::binary:
case detail::value_t::discarded:
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j));
} }
} }
return *result; for (std::size_t i = begin; i <= size; ++i)
{
if (i == size || data[i] == '/')
{
// a token 0 makes the pointer before it an array
if (i - begin == 1 && data[begin] == '0')
{
arrays.insert(data, begin - 1);
}
begin = i + 1;
}
else if (data[i] == '~')
{
// ~ must be followed by 0 or 1
if (JSON_HEDLEY_UNLIKELY(i + 1 == size || (data[i + 1] != '0' && data[i + 1] != '1')))
{
JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr));
}
}
}
} }
/*! /*!
@@ -1338,53 +1366,30 @@ class json_pointer
BasicJsonType result; BasicJsonType result;
// parse every key once; the reference tokens are kept for the second // check every key, and collect the prefixes that have a reference
// pass (the vectors are never resized, so the tree may refer to them) // token 0 among their children; the values below them are arrays,
std::vector<std::vector<string_t>> keys; // all others are objects (see array_prefix_set)
keys.reserve(value.m_data.m_value.object->size()); array_prefix_set arrays;
const string_t* previous_key = nullptr;
for (const auto& element : *value.m_data.m_value.object) for (const auto& element : *value.m_data.m_value.object)
{ {
json_pointer ptr(element.first); scan_key(element.first, previous_key, arrays);
keys.push_back(std::move(ptr.reference_tokens)); previous_key = &element.first;
} }
// collect the pointer prefixes that have a reference token 0 among // values[i] is the value after the first i reference tokens of the
// their children; the values below them are arrays, all others are // previous key, ends[i] the length of that prefix of the key. Keys are
// objects (see prefix_tree) // often sorted, so most of the path to the value of a key is the
prefix_tree tree; // same as for the previous key and need not be looked up again. Only
// the number of the prefix with i + 1 tokens of the previous key; keys // the values below the common prefix are dropped, and only they can
// are often sorted, so most of the prefix is the same as for the // move when a value is added: the ones above are not inside the
// previous key and need not be looked up again // container that grows.
std::vector<std::size_t> path; std::vector<BasicJsonType*> values(1, &result);
const std::vector<string_t>* previous = nullptr; std::vector<std::size_t> ends(1, 0);
// the number of the complete prefix (i.e., the key) of each key const string_t* previous = nullptr;
std::vector<std::size_t> leaves; string_t token;
leaves.reserve(keys.size());
for (const auto& tokens : keys)
{
std::size_t common = 0;
if (previous != nullptr)
{
const std::size_t limit = (std::min)(tokens.size(), previous->size());
while (common < limit && tokens[common] == (*previous)[common])
{
++common;
}
}
path.resize(common);
std::size_t id = common == 0 ? 0 : path[common - 1];
for (std::size_t i = common; i < tokens.size(); ++i)
{
id = tree.add_child(id, tokens[i]);
path.push_back(id);
}
leaves.push_back(id);
previous = &tokens;
}
// iterate the JSON object values // iterate the JSON object values
std::vector<std::size_t> prefixes;
std::size_t key_index = 0;
for (const auto& element : *value.m_data.m_value.object) for (const auto& element : *value.m_data.m_value.object)
{ {
if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))
@@ -1392,21 +1397,100 @@ class json_pointer
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second));
} }
const auto& tokens = keys[key_index]; const string_t& key = element.first;
// number the prefixes of this key from its last one to the first const char* const data = key.data();
prefixes.resize(tokens.size());
for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) // the number of reference tokens the key has in common with the
// previous one
std::size_t depth = 0;
if (previous != nullptr)
{ {
id = tree.parent[id]; const std::size_t same = common_prefix_length(data, previous->data(), (std::min)(key.size(), previous->size()));
prefixes[i - 1] = id; depth = ends.size() - 1;
while (depth > 0 && (ends[depth] > same || (ends[depth] == same && same < key.size() && data[same] != '/')))
{
--depth;
} }
++key_index; }
values.resize(depth + 1);
ends.resize(depth + 1);
previous = &key;
// Assign the value to the reference pointed to by JSON pointer. Note // Assign the value to the reference pointed to by JSON pointer. Note
// that if the JSON pointer is "" (i.e., points to the whole value), // that if the JSON pointer is "" (i.e., points to the whole value),
// function get_and_create returns a reference to the result itself. // the value is the result itself. An assignment will then create a
// An assignment will then create a primitive value. // primitive value.
get_and_create(result, tokens, prefixes, tree) = element.second; BasicJsonType* current = values.back();
for (std::size_t pos = ends.back(); pos < key.size();)
{
// the reference token between the slash at pos and the next
std::size_t end = key.find_first_of('/', pos + 1);
if (end == string_t::npos)
{
end = key.size();
}
token.clear();
token.append(data + pos + 1, end - pos - 1);
if (token.find_first_of('~') != string_t::npos)
{
detail::unescape(token);
}
switch (current->type())
{
case detail::value_t::null:
{
if (arrays.contains(data, pos))
{
// some reference token below this position is 0, so the
// value is an array
current = &current->operator[](array_index<BasicJsonType>(token));
}
else
{
// start a new object otherwise
current = &current->operator[](token);
}
break;
}
case detail::value_t::object:
{
// create an entry in the object
current = &object_member(*current, token);
break;
}
case detail::value_t::array:
{
// create an entry in the array
current = &current->operator[](array_index<BasicJsonType>(token));
break;
}
/*
The following code is only reached if there exists a reference
token _and_ the current value is primitive. In this case, we have
an error situation, because primitive values may only occur as
a single value; that is, with an empty list of reference tokens.
*/
case detail::value_t::string:
case detail::value_t::boolean:
case detail::value_t::number_integer:
case detail::value_t::number_unsigned:
case detail::value_t::number_float:
case detail::value_t::binary:
case detail::value_t::discarded:
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &result));
}
values.push_back(current);
ends.push_back(end);
pos = end;
}
*current = element.second;
} }
return result; return result;
single_include/nlohmann/json.hpp
Generated
+273 -189
View File
@@ -21430,7 +21430,9 @@ NLOHMANN_JSON_NAMESPACE_END
#include <cctype> // isdigit #include <cctype> // isdigit
#include <cerrno> // errno, ERANGE #include <cerrno> // errno, ERANGE
#include <cstddef> // size_t #include <cstddef> // size_t
#include <cstdint> // uint64_t
#include <cstdlib> // strtoull #include <cstdlib> // strtoull
#include <cstring> // memcmp, memcpy
#ifndef JSON_NO_IO #ifndef JSON_NO_IO
#include <iosfwd> // ostream #include <iosfwd> // ostream
#endif // JSON_NO_IO #endif // JSON_NO_IO
@@ -21917,50 +21919,111 @@ class json_pointer
private: private:
/*! /*!
@brief the pointer prefixes of a flattened object, and which of them denote arrays @brief the pointer prefixes of a flattened object that denote arrays
@ref unflatten collects the pointer prefixes that have a reference token 0 @ref unflatten creates arrays exactly below the prefixes that have a
among their children; @ref get_and_create creates arrays exactly below reference token 0 among their children, and objects everywhere else.
those prefixes and objects everywhere else. Deciding this up front keeps Deciding this up front keeps the result independent of the order in which
the result independent of the order in which the flattened object is the flattened object is iterated, which is unspecified for some object
iterated, which is unspecified for some object types. types.
The prefixes form a tree and are numbered (number 0 is the empty prefix), A prefix is identified by the text of the key it is the beginning of:
so each of them is stored only once (as a node) rather than as a copy of escaping is unambiguous, so equal prefixes of valid keys are equal texts.
all of its reference tokens. The tree does not own the reference tokens: The set refers to the keys, which must outlive it.
it refers to the tokens of the parsed keys, which must outlive it.
*/ */
struct prefix_tree class array_prefix_set
{ {
// open-addressing hash table from (parent, last reference token) to public:
// the number of the node; empty slots have token == nullptr void insert(const char* data, std::size_t size)
{
// flattened objects are usually sorted, so most keys that have a
// 0 yield the prefix of the previous one
if (last_data != nullptr && last_size == size && std::memcmp(last_data, data, size) == 0)
{
return;
}
last_data = data;
last_size = size;
if (table.empty())
{
table.resize(16);
}
const std::size_t hash = hash_of(data, size);
const std::size_t mask = table.size() - 1;
std::size_t i = hash & mask;
while (table[i].data != nullptr)
{
if (table[i].hash == hash && table[i].size == size && std::memcmp(table[i].data, data, size) == 0)
{
return;
}
i = (i + 1) & mask;
}
table[i] = slot(hash, data, size);
// keep the load factor below 1/2
if (++used * 2 > table.size())
{
grow();
}
}
bool empty() const noexcept
{
return used == 0;
}
bool contains(const char* data, std::size_t size) const noexcept
{
if (used == 0)
{
return false;
}
const std::size_t hash = hash_of(data, size);
const std::size_t mask = table.size() - 1;
for (std::size_t i = hash & mask; table[i].data != nullptr; i = (i + 1) & mask)
{
if (table[i].hash == hash && table[i].size == size && std::memcmp(table[i].data, data, size) == 0)
{
return true;
}
}
return false;
}
private:
struct slot struct slot
{ {
std::size_t hash; slot() noexcept = default;
std::size_t parent;
const string_t* token;
std::size_t id;
slot() noexcept slot(std::size_t h, const char* d, std::size_t s) noexcept
: hash(0), parent(0), token(nullptr), id(0) : hash(h), data(d), size(s)
{} {}
slot(std::size_t h, std::size_t p, const string_t* t, std::size_t i) noexcept std::size_t hash = 0;
: hash(h), parent(p), token(t), id(i) const char* data = nullptr; // nullptr: the slot is empty
{} std::size_t size = 0;
}; };
std::vector<slot> table; static std::size_t hash_of(const char* data, std::size_t size) noexcept
std::size_t used = 0;
static std::size_t hash_of(std::size_t parent, const string_t& token) noexcept
{ {
std::size_t h = parent * static_cast<std::size_t>(0x9E3779B97F4A7C15ULL & (std::numeric_limits<std::size_t>::max)()); std::uint64_t h = 0x9E3779B97F4A7C15ULL ^ size;
for (std::size_t i = 0; i < token.size(); ++i) std::size_t i = 0;
for (; i + 8 <= size; i += 8)
{ {
h = (h ^ static_cast<unsigned char>(token[i])) * static_cast<std::size_t>(0x100000001B3ULL & (std::numeric_limits<std::size_t>::max)()); std::uint64_t w = 0;
std::memcpy(&w, data + i, 8);
h = (h ^ w) * 0x100000001B3ULL;
h ^= h >> 32u;
} }
return h ^ (h >> 29u); std::uint64_t w = 0;
if (i < size)
{
std::memcpy(&w, data + i, size - i);
}
h = (h ^ w) * 0x9E3779B97F4A7C15ULL;
return static_cast<std::size_t>(h ^ (h >> 29u));
} }
void grow() void grow()
@@ -21970,10 +22033,10 @@ class json_pointer
const std::size_t mask = table.size() - 1; const std::size_t mask = table.size() - 1;
for (const auto& s : old) for (const auto& s : old)
{ {
if (s.token != nullptr) if (s.data != nullptr)
{ {
std::size_t i = s.hash & mask; std::size_t i = s.hash & mask;
while (table[i].token != nullptr) while (table[i].data != nullptr)
{ {
i = (i + 1) & mask; i = (i + 1) & mask;
} }
@@ -21982,130 +22045,95 @@ class json_pointer
} }
} }
// parent[id] is the number of the prefix without its last token std::vector<slot> table{};
std::vector<std::size_t> parent; std::size_t used = 0;
// is_array[id] is true iff some flattened key has the reference token const char* last_data = nullptr;
// 0 directly below the prefix with number id std::size_t last_size = 0;
std::vector<bool> is_array;
// start with the empty prefix only
prefix_tree()
: table(16)
, parent(1, 0)
, is_array(1, false)
{}
// return the number of the prefix with number id extended by
// reference_token, adding it if it is new
std::size_t add_child(std::size_t id, const string_t& reference_token)
{
const std::size_t hash = hash_of(id, reference_token);
const std::size_t mask = table.size() - 1;
std::size_t i = hash & mask;
while (table[i].token != nullptr)
{
if (table[i].hash == hash && table[i].parent == id && *table[i].token == reference_token)
{
return table[i].id;
}
i = (i + 1) & mask;
}
// new prefix
const std::size_t child = parent.size();
table[i] = slot(hash, id, &reference_token, child);
if (reference_token == "0")
{
is_array[id] = true;
}
parent.push_back(id);
is_array.push_back(false);
// keep the load factor below 1/2
if (++used * 2 > table.size())
{
grow();
}
return child;
}
}; };
/// @brief the length of the longest common prefix of the first @a limit bytes of @a a and @a b
static std::size_t common_prefix_length(const char* a, const char* b, const std::size_t limit) noexcept
{
std::size_t i = 0;
for (; i + 8 <= limit; i += 8)
{
std::uint64_t x = 0;
std::uint64_t y = 0;
std::memcpy(&x, a + i, 8);
std::memcpy(&y, b + i, 8);
if (x != y)
{
break;
}
}
while (i < limit && a[i] == b[i])
{
++i;
}
return i;
}
/*! /*!
@brief create and return a reference to the pointed to value @brief check that @a key is a valid JSON pointer, and add the prefixes of
it that denote arrays to @a arrays
@param[in] j the value to create the pointed to value in @throw parse_error.107 if the key is not empty or begins with '/'
@param[in] tokens the reference tokens of the pointer @throw parse_error.108 if character '~' is not followed by '0' or '1'
@param[in] prefixes prefixes[i] is the number in @a tree of the prefix
consisting of the first i reference tokens
Complexity: Linear in the number of reference tokens. The same checks as @ref split, which throws the same exceptions, but
without building the reference tokens.
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if array index is not a number
@throw type_error.313 if value cannot be unflattened
*/ */
template<typename BasicJsonType> static void scan_key(const string_t& key, const string_t* previous, array_prefix_set& arrays)
static BasicJsonType& get_and_create(BasicJsonType& j, const std::vector<string_t>& tokens,
const std::vector<std::size_t>& prefixes, const prefix_tree& tree)
{ {
auto* result = &j; const std::size_t size = key.size();
if (size == 0)
// in case no reference tokens exist, return a reference to the JSON value
// j which will be overwritten by a primitive value
for (std::size_t i = 0; i < tokens.size(); ++i)
{ {
const auto& reference_token = tokens[i]; return;
switch (result->type())
{
case detail::value_t::null:
{
if (tree.is_array[prefixes[i]])
{
// some reference token below this position is 0, so the
// value is an array
result = &result->operator[](array_index<BasicJsonType>(reference_token));
}
else
{
// start a new object otherwise
result = &result->operator[](reference_token);
}
break;
} }
case detail::value_t::object: const char* const data = key.data();
if (JSON_HEDLEY_UNLIKELY(data[0] != '/'))
{ {
// create an entry in the object JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_missing_slash, 1, detail::concat("JSON pointer must be empty or begin with '/' - was: '", key, "'"), nullptr));
result = &object_member(*result, reference_token);
break;
} }
case detail::value_t::array: // The beginning of the key that it has in common with the previous
// one, which was checked already, is skipped: the scan starts with
// the token after the last slash in it.
std::size_t begin = 1;
if (previous != nullptr)
{ {
// create an entry in the array std::size_t same = common_prefix_length(data, previous->data(), (std::min)(size, previous->size()));
result = &result->operator[](array_index<BasicJsonType>(reference_token)); while (same > 1 && data[same - 1] != '/')
break; {
--same;
} }
if (same > 1)
/* {
The following code is only reached if there exists a reference begin = same;
token _and_ the current value is primitive. In this case, we have
an error situation, because primitive values may only occur as
a single value; that is, with an empty list of reference tokens.
*/
case detail::value_t::string:
case detail::value_t::boolean:
case detail::value_t::number_integer:
case detail::value_t::number_unsigned:
case detail::value_t::number_float:
case detail::value_t::binary:
case detail::value_t::discarded:
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &j));
} }
} }
return *result; for (std::size_t i = begin; i <= size; ++i)
{
if (i == size || data[i] == '/')
{
// a token 0 makes the pointer before it an array
if (i - begin == 1 && data[begin] == '0')
{
arrays.insert(data, begin - 1);
}
begin = i + 1;
}
else if (data[i] == '~')
{
// ~ must be followed by 0 or 1
if (JSON_HEDLEY_UNLIKELY(i + 1 == size || (data[i + 1] != '0' && data[i + 1] != '1')))
{
JSON_THROW(detail::parse_error::create(detail::exception_id::pointer_invalid_escape, 0, "escape character '~' must be followed with '0' or '1'", nullptr));
}
}
}
} }
/*! /*!
@@ -22763,53 +22791,30 @@ class json_pointer
BasicJsonType result; BasicJsonType result;
// parse every key once; the reference tokens are kept for the second // check every key, and collect the prefixes that have a reference
// pass (the vectors are never resized, so the tree may refer to them) // token 0 among their children; the values below them are arrays,
std::vector<std::vector<string_t>> keys; // all others are objects (see array_prefix_set)
keys.reserve(value.m_data.m_value.object->size()); array_prefix_set arrays;
const string_t* previous_key = nullptr;
for (const auto& element : *value.m_data.m_value.object) for (const auto& element : *value.m_data.m_value.object)
{ {
json_pointer ptr(element.first); scan_key(element.first, previous_key, arrays);
keys.push_back(std::move(ptr.reference_tokens)); previous_key = &element.first;
} }
// collect the pointer prefixes that have a reference token 0 among // values[i] is the value after the first i reference tokens of the
// their children; the values below them are arrays, all others are // previous key, ends[i] the length of that prefix of the key. Keys are
// objects (see prefix_tree) // often sorted, so most of the path to the value of a key is the
prefix_tree tree; // same as for the previous key and need not be looked up again. Only
// the number of the prefix with i + 1 tokens of the previous key; keys // the values below the common prefix are dropped, and only they can
// are often sorted, so most of the prefix is the same as for the // move when a value is added: the ones above are not inside the
// previous key and need not be looked up again // container that grows.
std::vector<std::size_t> path; std::vector<BasicJsonType*> values(1, &result);
const std::vector<string_t>* previous = nullptr; std::vector<std::size_t> ends(1, 0);
// the number of the complete prefix (i.e., the key) of each key const string_t* previous = nullptr;
std::vector<std::size_t> leaves; string_t token;
leaves.reserve(keys.size());
for (const auto& tokens : keys)
{
std::size_t common = 0;
if (previous != nullptr)
{
const std::size_t limit = (std::min)(tokens.size(), previous->size());
while (common < limit && tokens[common] == (*previous)[common])
{
++common;
}
}
path.resize(common);
std::size_t id = common == 0 ? 0 : path[common - 1];
for (std::size_t i = common; i < tokens.size(); ++i)
{
id = tree.add_child(id, tokens[i]);
path.push_back(id);
}
leaves.push_back(id);
previous = &tokens;
}
// iterate the JSON object values // iterate the JSON object values
std::vector<std::size_t> prefixes;
std::size_t key_index = 0;
for (const auto& element : *value.m_data.m_value.object) for (const auto& element : *value.m_data.m_value.object)
{ {
if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive())) if (JSON_HEDLEY_UNLIKELY(!element.second.is_primitive()))
@@ -22817,21 +22822,100 @@ class json_pointer
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second)); JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_value_not_primitive, "values in object must be primitive", &element.second));
} }
const auto& tokens = keys[key_index]; const string_t& key = element.first;
// number the prefixes of this key from its last one to the first const char* const data = key.data();
prefixes.resize(tokens.size());
for (std::size_t i = tokens.size(), id = leaves[key_index]; i > 0; --i) // the number of reference tokens the key has in common with the
// previous one
std::size_t depth = 0;
if (previous != nullptr)
{ {
id = tree.parent[id]; const std::size_t same = common_prefix_length(data, previous->data(), (std::min)(key.size(), previous->size()));
prefixes[i - 1] = id; depth = ends.size() - 1;
while (depth > 0 && (ends[depth] > same || (ends[depth] == same && same < key.size() && data[same] != '/')))
{
--depth;
} }
++key_index; }
values.resize(depth + 1);
ends.resize(depth + 1);
previous = &key;
// Assign the value to the reference pointed to by JSON pointer. Note // Assign the value to the reference pointed to by JSON pointer. Note
// that if the JSON pointer is "" (i.e., points to the whole value), // that if the JSON pointer is "" (i.e., points to the whole value),
// function get_and_create returns a reference to the result itself. // the value is the result itself. An assignment will then create a
// An assignment will then create a primitive value. // primitive value.
get_and_create(result, tokens, prefixes, tree) = element.second; BasicJsonType* current = values.back();
for (std::size_t pos = ends.back(); pos < key.size();)
{
// the reference token between the slash at pos and the next
std::size_t end = key.find_first_of('/', pos + 1);
if (end == string_t::npos)
{
end = key.size();
}
token.clear();
token.append(data + pos + 1, end - pos - 1);
if (token.find_first_of('~') != string_t::npos)
{
detail::unescape(token);
}
switch (current->type())
{
case detail::value_t::null:
{
if (arrays.contains(data, pos))
{
// some reference token below this position is 0, so the
// value is an array
current = &current->operator[](array_index<BasicJsonType>(token));
}
else
{
// start a new object otherwise
current = &current->operator[](token);
}
break;
}
case detail::value_t::object:
{
// create an entry in the object
current = &object_member(*current, token);
break;
}
case detail::value_t::array:
{
// create an entry in the array
current = &current->operator[](array_index<BasicJsonType>(token));
break;
}
/*
The following code is only reached if there exists a reference
token _and_ the current value is primitive. In this case, we have
an error situation, because primitive values may only occur as
a single value; that is, with an empty list of reference tokens.
*/
case detail::value_t::string:
case detail::value_t::boolean:
case detail::value_t::number_integer:
case detail::value_t::number_unsigned:
case detail::value_t::number_float:
case detail::value_t::binary:
case detail::value_t::discarded:
default:
JSON_THROW(detail::type_error::create(detail::exception_id::unflatten_invalid_value, "invalid value to unflatten", &result));
}
values.push_back(current);
ends.push_back(end);
pos = end;
}
*current = element.second;
} }
return result; return result;