From 7a37a27a678e5484f92db78b697e18f96d320bac Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 28 Aug 2026 14:36:50 +0000 Subject: [PATCH] Fix unflatten and binary dumping for non-default configurations unflatten() decided between array and object by looking at the first reference token it happened to see for a node: it started an array only when that token was 0. With a sorted object type the token 0 always arrives first, so the result was correct by accident; with an object type whose iteration order is unspecified, {"/c/2":3,"/c/1":2,"/c/0":1} unflattened to an object with the keys "0", "1", and "2" instead of an array. Collect the pointer prefixes that have a reference token 0 among their children before building the result, and let get_and_create() consult that set. The outcome is now independent of the iteration order and matches, for every input, what a sorted object type produced before: a value is restored as an array if and only if one of its keys is 0. Iterating the flattened object in a different order would have been simpler, but it would have changed the key order of the result for insertion-ordered object types. The serializer, std::hash, and the UBJSON writer converted the elements of a binary value to an integer implicitly, which does not compile for a BinaryType whose value type is std::byte, and which made dump() write the bytes of a signed value type as negative numbers. Convert to std::uint8_t explicitly in all three places, so every byte type dumps as 0..255. The default std::vector configuration is unaffected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018hxZxz8svM54c6ATEvXp5E Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/binary_t.md | 9 ++- docs/mkdocs/docs/api/basic_json/unflatten.md | 10 ++- .../features/types/template_parameters.md | 14 ++-- include/nlohmann/detail/hash.hpp | 4 +- include/nlohmann/detail/json_pointer.hpp | 48 +++++++++-- .../nlohmann/detail/output/binary_writer.hpp | 4 +- include/nlohmann/detail/output/serializer.hpp | 24 ++++-- single_include/nlohmann/json.hpp | 80 ++++++++++++++++--- tests/src/unit-custom-object-type.cpp | 10 +++ tests/src/unit-json_pointer.cpp | 10 +++ tests/src/unit-regression2.cpp | 33 ++++++++ 11 files changed, 207 insertions(+), 39 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/binary_t.md b/docs/mkdocs/docs/api/basic_json/binary_t.md index e19cb4d05..36600748a 100644 --- a/docs/mkdocs/docs/api/basic_json/binary_t.md +++ b/docs/mkdocs/docs/api/basic_json/binary_t.md @@ -54,10 +54,8 @@ The default values for `BinaryType` is `#!cpp std::vector`. #### Supported byte types -`#!cpp std::vector` and `#!cpp std::vector` are fully supported. With -`#!cpp std::vector`, assignment, [`get`](get.md), and the [binary formats](../../features/binary_formats/index.md) -work, but [`dump`](dump.md) and [`std::hash`](std_hash.md) do not compile, because `#!cpp std::byte` neither -converts to an integer nor is hashable as one. +`#!cpp std::vector`, `#!cpp std::vector`, and `#!cpp std::vector` are supported. +Regardless of which of them is configured, [`dump`](dump.md) writes the bytes as the numbers 0..255. #### Custom BinaryType behavior @@ -135,3 +133,6 @@ type `#!cpp binary_t*` must be dereferenced. ## Version history - Added in version 3.8.0. Changed the type of subtype to `std::uint64_t` in version 3.10.0. +- Fixed [`dump`](dump.md), [`std::hash`](std_hash.md), and [`to_ubjson`](to_ubjson.md) for byte types that are not + integers (e.g., `#!cpp std::byte`) in version 3.13.0. `dump` now writes the bytes of a signed byte type (e.g., + `#!cpp char`) as 0..255 rather than as negative numbers. diff --git a/docs/mkdocs/docs/api/basic_json/unflatten.md b/docs/mkdocs/docs/api/basic_json/unflatten.md index 1af243b58..ac88cd73e 100644 --- a/docs/mkdocs/docs/api/basic_json/unflatten.md +++ b/docs/mkdocs/docs/api/basic_json/unflatten.md @@ -37,7 +37,14 @@ Linear in the size of the JSON value. ## Notes Empty objects and arrays are flattened by [`flatten()`](flatten.md) to `#!json null` values and cannot unflattened to -their original type. Apart from this example, for a JSON value `j`, the following is always true: +their original type. + +A flattened array and a flattened object whose keys are array indices are indistinguishable, because both are +described by the same JSON pointers. A value is therefore restored as an array if and only if one of its keys is the +reference token `0`, and as an object otherwise: `#!json {"2": 1}` is restored unchanged, whereas `#!json {"0": 1}` is +restored as `#!json [1]`. This decision does not depend on the order in which the flattened object is iterated. + +Apart from these two cases, for a JSON value `j`, the following is always true: `#!cpp j == j.flatten().unflatten()`. ## Examples @@ -63,3 +70,4 @@ their original type. Apart from this example, for a JSON value `j`, the followin ## Version history - Added in version 2.0.0. +- Made the array/object decision independent of the object's iteration order in version 3.13.0. diff --git a/docs/mkdocs/docs/features/types/template_parameters.md b/docs/mkdocs/docs/features/types/template_parameters.md index 41229144f..cf8e687bf 100644 --- a/docs/mkdocs/docs/features/types/template_parameters.md +++ b/docs/mkdocs/docs/features/types/template_parameters.md @@ -112,13 +112,11 @@ using unordered_json = nlohmann::basic_json; The same pattern (ignoring the third argument) is how [`tsl::ordered_map`](https://github.com/Tessil/ordered-map) and similar containers are integrated; see [Object Order](../object_order.md). -#### Hash-ordered containers and `unflatten` +#### Iteration order -[`unflatten`](../../api/basic_json/unflatten.md) rebuilds an array only if it encounters the reference token `0` -before the other indices of that array. Sorted containers (`#!cpp std::map`) and insertion-ordered containers -(`nlohmann::ordered_map`) both iterate the flattened object in an order that satisfies this. A container with an -unspecified iteration order does not, and `#!cpp j.flatten().unflatten()` may then return objects with the keys -`#!json "0"`, `#!json "1"`, ... where the original had arrays. +The library never relies on the container's iteration order for correctness; it does determine the order in which +object keys are serialized by [`dump`](../../api/basic_json/dump.md) and visited by +[`items`](../../api/basic_json/items.md). See [Object Order](../object_order.md). #### `capacity()` marks a container as insertion-ordered @@ -138,7 +136,7 @@ The library does not sort or de-duplicate keys itself; the behavior described in |--------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------| | `#!cpp std::map` (default) | full | | [`nlohmann::ordered_map`](../../api/ordered_map.md) | full; used by [`ordered_json`](../../api/ordered_json.md) | -| `#!cpp std::unordered_map`, through the adapter shown above | full except [`unflatten`](../../api/basic_json/unflatten.md) | +| `#!cpp std::unordered_map`, through the adapter shown above | full | | [`tsl::ordered_map`](https://github.com/Tessil/ordered-map), [`nlohmann::fifo_map`](https://github.com/nlohmann/fifo_map) | through the same adapter pattern; see [Object Order](../object_order.md) | | `#!cpp std::multimap`, `#!cpp std::unordered_multimap` | not usable; `emplace` does not return `#!cpp std::pair` | @@ -437,7 +435,7 @@ such a container to a `basic_json` value. |------------------------------------------|-----------------------------------------------------------------------------------------------| | `#!cpp std::vector` (default) | full | | `#!cpp std::vector` | full | -| `#!cpp std::vector` | assignment, [`get`](../../api/basic_json/get.md), and the binary formats work, but [`dump`](../../api/basic_json/dump.md) and [`std::hash`](../../api/basic_json/std_hash.md) do not compile | +| `#!cpp std::vector` | full | | `#!cpp std::string` | not usable; `binary_t::container_type` and `string_t` would be the same type, which makes the [`swap`](../../api/basic_json/swap.md) overloads ambiguous | | containers whose `value_type` is wider than one byte | not usable | diff --git a/include/nlohmann/detail/hash.hpp b/include/nlohmann/detail/hash.hpp index 61b3469f1..be8063f89 100644 --- a/include/nlohmann/detail/hash.hpp +++ b/include/nlohmann/detail/hash.hpp @@ -114,7 +114,9 @@ std::size_t hash(const BasicJsonType& j) seed = combine(seed, static_cast(j.get_binary().subtype())); for (const auto byte : j.get_binary()) { - seed = combine(seed, std::hash {}(byte)); + // the cast is needed for binary types whose value type is not + // an integer (e.g., std::byte) + seed = combine(seed, std::hash {}(static_cast(byte))); } return seed; } diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 576ba62cc..5341d897d 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -17,6 +17,7 @@ #endif // JSON_NO_IO #include // max #include // accumulate +#include // set #include // string #include // move #include // vector @@ -300,19 +301,35 @@ class json_pointer } private: + /*! + @brief the reference token sequences that denote arrays + + @ref unflatten collects the pointer prefixes that have a reference token 0 + among their children; @ref get_and_create creates arrays exactly below + those prefixes and objects everywhere else. Deciding this up front keeps + the result independent of the order in which the flattened object is + iterated, which is unspecified for some object types. + */ + using array_parents_t = std::set>; + /*! @brief create and return a reference to the pointed to value @complexity Linear in the number of reference tokens. + @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j) const + BasicJsonType& get_and_create(BasicJsonType& j, const array_parents_t& array_parents) const { auto* result = &j; + // the reference tokens that have been consumed so far; used to look up + // whether the value to be created below is an array or an object + std::vector prefix; + // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value for (const auto& reference_token : reference_tokens) @@ -321,10 +338,11 @@ class json_pointer { case detail::value_t::null: { - if (reference_token == "0") + if (array_parents.find(prefix) != array_parents.end()) { - // start a new array if the reference token is 0 - result = &result->operator[](0); + // some reference token below this position is 0, so the + // value is an array + result = &result->operator[](array_index(reference_token)); } else { @@ -364,6 +382,8 @@ class json_pointer default: JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j)); } + + prefix.push_back(reference_token); } return *result; @@ -939,6 +959,24 @@ class json_pointer BasicJsonType result; + // collect the pointer prefixes that have a reference token 0 among + // their children; the values below them are arrays, all others are + // objects (see array_parents_t) + array_parents_t array_parents; + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + std::vector prefix; + for (auto& reference_token : ptr.reference_tokens) + { + if (reference_token == "0") + { + array_parents.insert(prefix); + } + prefix.push_back(std::move(reference_token)); + } + } + // iterate the JSON object values for (const auto& element : *value.m_data.m_value.object) { @@ -951,7 +989,7 @@ class json_pointer // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; + json_pointer(element.first).get_and_create(result, array_parents) = element.second; } return result; diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index e636c6d84..ea4a83020 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -887,7 +887,9 @@ class binary_writer for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); - oa->write_character(to_char_type(j.m_data.m_value.binary->data()[i])); + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 0b608f8e2..1f4c9f80b 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -274,10 +274,10 @@ class serializer for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); + dump_integer(to_byte_value(*i)); o->write_characters(", ", 2); } - dump_integer(val.m_data.m_value.binary->back()); + dump_integer(to_byte_value(val.m_data.m_value.binary->back())); } o->write_characters("],\n", 3); @@ -305,10 +305,10 @@ class serializer for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); + dump_integer(to_byte_value(*i)); o->write_character(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_integer(to_byte_value(val.m_data.m_value.binary->back())); } o->write_characters("],\"subtype\":", 12); @@ -703,6 +703,19 @@ class serializer pos += 6; } + /*! + @brief convert a single element of a binary value to its byte value + + The elements of a binary value are dumped as the numbers 0..255, regardless + of the value type of the configured BinaryType: that type may be signed + (`char`), unsigned (`std::uint8_t`), or not an integer at all + (`std::byte`), none of which @ref dump_integer can handle uniformly. + */ + static std::uint8_t to_byte_value(binary_char_t x) noexcept + { + return static_cast(x); + } + // templates to avoid warnings about useless casts template ::value, int> = 0> bool is_negative_number(NumberType x) @@ -728,8 +741,7 @@ class serializer template < typename NumberType, detail::enable_if_t < std::is_integral::value || std::is_same::value || - std::is_same::value || - std::is_same::value, + std::is_same::value, int > = 0 > void dump_integer(NumberType x) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 977c61414..ebf3153a3 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6954,7 +6954,9 @@ std::size_t hash(const BasicJsonType& j) seed = combine(seed, static_cast(j.get_binary().subtype())); for (const auto byte : j.get_binary()) { - seed = combine(seed, std::hash {}(byte)); + // the cast is needed for binary types whose value type is not + // an integer (e.g., std::byte) + seed = combine(seed, std::hash {}(static_cast(byte))); } return seed; } @@ -15594,6 +15596,7 @@ NLOHMANN_JSON_NAMESPACE_END #endif // JSON_NO_IO #include // max #include // accumulate +#include // set #include // string #include // move #include // vector @@ -15882,19 +15885,35 @@ class json_pointer } private: + /*! + @brief the reference token sequences that denote arrays + + @ref unflatten collects the pointer prefixes that have a reference token 0 + among their children; @ref get_and_create creates arrays exactly below + those prefixes and objects everywhere else. Deciding this up front keeps + the result independent of the order in which the flattened object is + iterated, which is unspecified for some object types. + */ + using array_parents_t = std::set>; + /*! @brief create and return a reference to the pointed to value @complexity Linear in the number of reference tokens. + @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ template - BasicJsonType& get_and_create(BasicJsonType& j) const + BasicJsonType& get_and_create(BasicJsonType& j, const array_parents_t& array_parents) const { auto* result = &j; + // the reference tokens that have been consumed so far; used to look up + // whether the value to be created below is an array or an object + std::vector prefix; + // in case no reference tokens exist, return a reference to the JSON value // j which will be overwritten by a primitive value for (const auto& reference_token : reference_tokens) @@ -15903,10 +15922,11 @@ class json_pointer { case detail::value_t::null: { - if (reference_token == "0") + if (array_parents.find(prefix) != array_parents.end()) { - // start a new array if the reference token is 0 - result = &result->operator[](0); + // some reference token below this position is 0, so the + // value is an array + result = &result->operator[](array_index(reference_token)); } else { @@ -15946,6 +15966,8 @@ class json_pointer default: JSON_THROW(detail::type_error::create(313, "invalid value to unflatten", &j)); } + + prefix.push_back(reference_token); } return *result; @@ -16521,6 +16543,24 @@ class json_pointer BasicJsonType result; + // collect the pointer prefixes that have a reference token 0 among + // their children; the values below them are arrays, all others are + // objects (see array_parents_t) + array_parents_t array_parents; + for (const auto& element : *value.m_data.m_value.object) + { + json_pointer ptr(element.first); + std::vector prefix; + for (auto& reference_token : ptr.reference_tokens) + { + if (reference_token == "0") + { + array_parents.insert(prefix); + } + prefix.push_back(std::move(reference_token)); + } + } + // iterate the JSON object values for (const auto& element : *value.m_data.m_value.object) { @@ -16533,7 +16573,7 @@ class json_pointer // that if the JSON pointer is "" (i.e., points to the whole value), // function get_and_create returns a reference to the result itself. // An assignment will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; + json_pointer(element.first).get_and_create(result, array_parents) = element.second; } return result; @@ -17831,7 +17871,9 @@ class binary_writer for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); - oa->write_character(to_char_type(j.m_data.m_value.binary->data()[i])); + // the cast is needed for binary types whose value type + // is not an integer (e.g., std::byte) + oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } @@ -20280,10 +20322,10 @@ class serializer for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); + dump_integer(to_byte_value(*i)); o->write_characters(", ", 2); } - dump_integer(val.m_data.m_value.binary->back()); + dump_integer(to_byte_value(val.m_data.m_value.binary->back())); } o->write_characters("],\n", 3); @@ -20311,10 +20353,10 @@ class serializer for (auto i = val.m_data.m_value.binary->cbegin(); i != val.m_data.m_value.binary->cend() - 1; ++i) { - dump_integer(*i); + dump_integer(to_byte_value(*i)); o->write_character(','); } - dump_integer(val.m_data.m_value.binary->back()); + dump_integer(to_byte_value(val.m_data.m_value.binary->back())); } o->write_characters("],\"subtype\":", 12); @@ -20709,6 +20751,19 @@ class serializer pos += 6; } + /*! + @brief convert a single element of a binary value to its byte value + + The elements of a binary value are dumped as the numbers 0..255, regardless + of the value type of the configured BinaryType: that type may be signed + (`char`), unsigned (`std::uint8_t`), or not an integer at all + (`std::byte`), none of which @ref dump_integer can handle uniformly. + */ + static std::uint8_t to_byte_value(binary_char_t x) noexcept + { + return static_cast(x); + } + // templates to avoid warnings about useless casts template ::value, int> = 0> bool is_negative_number(NumberType x) @@ -20734,8 +20789,7 @@ class serializer template < typename NumberType, detail::enable_if_t < std::is_integral::value || std::is_same::value || - std::is_same::value || - std::is_same::value, + std::is_same::value, int > = 0 > void dump_integer(NumberType x) { diff --git a/tests/src/unit-custom-object-type.cpp b/tests/src/unit-custom-object-type.cpp index 2349c0ecd..56b853d91 100644 --- a/tests/src/unit-custom-object-type.cpp +++ b/tests/src/unit-custom-object-type.cpp @@ -88,6 +88,16 @@ TEST_CASE("object type without key_compare") CHECK(unordered_json::from_msgpack(unordered_json::to_msgpack(j)) == j); } + SECTION("flatten and unflatten do not depend on the iteration order") + { + // the flattened object is iterated in an unspecified order, so + // unflatten() must not decide between array and object based on + // whichever reference token it happens to see first + const auto j = unordered_json::parse( + R"({"c":[1,2,3],"d":{"e":"s"},"n":[[0,1],[2]],"o":{"2":"x"}})"); + CHECK(j.flatten().unflatten() == j); + } + SECTION("conversion to and from nlohmann::json") { const auto j = unordered_json::parse(R"({"a":1,"b":[true,null]})"); diff --git a/tests/src/unit-json_pointer.cpp b/tests/src/unit-json_pointer.cpp index a8ed4a89e..aa5c5dc89 100644 --- a/tests/src/unit-json_pointer.cpp +++ b/tests/src/unit-json_pointer.cpp @@ -465,6 +465,16 @@ TEST_CASE("JSON pointers") // explicit roundtrip check CHECK(j.flatten().unflatten() == j); + // an object is only unflattened to an array if one of its keys is the + // reference token 0; this must not depend on which key is seen first + CHECK(json({{"/2", "x"}}).unflatten() == json({{"2", "x"}})); + CHECK(json({{"/10", "y"}, {"/2", "z"}}).unflatten() == json({{"10", "y"}, {"2", "z"}})); + CHECK(json({{"/0", 1}, {"/1", 2}}).unflatten() == json({1, 2})); + CHECK(json({{"/1", 2}, {"/0", 1}}).unflatten() == json({1, 2})); + CHECK(json({{"/0", 1}, {"/2", 3}}).unflatten() == json({1, nullptr, 3})); + CHECK(json({{"/a/1", 2}, {"/a/0", 1}}).unflatten() == json({{"a", {1, 2}}})); + CHECK(json({{"/a/1", 2}, {"/a/x", 1}}).unflatten() == json({{"a", {{"1", 2}, {"x", 1}}}})); + // roundtrip for primitive values json j_null; CHECK(j_null.flatten().unflatten() == j_null); diff --git a/tests/src/unit-regression2.cpp b/tests/src/unit-regression2.cpp index 2e7450e2e..7397f7534 100644 --- a/tests/src/unit-regression2.cpp +++ b/tests/src/unit-regression2.cpp @@ -1154,6 +1154,39 @@ TEST_CASE("regression tests 2") CHECK(!default_json.is_binary()); } + SECTION("dumping a binary value with a custom BinaryType") + { + // the elements of a binary value are dumped as the numbers 0..255, + // whatever the value type of the configured BinaryType is + const std::vector bytes{std::byte{0}, std::byte{1}, std::byte{0xFF}}; + CHECK(json_4804::binary(bytes).dump() == R"({"bytes":[0,1,255],"subtype":null})"); + CHECK(json_4804::binary(bytes, 42).dump() == R"({"bytes":[0,1,255],"subtype":42})"); + CHECK(json_4804::binary({}).dump() == R"({"bytes":[],"subtype":null})"); + + // a signed byte type must not dump negative numbers + using json_char_binary = nlohmann::basic_json < + std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, + double, std::allocator, nlohmann::adl_serializer, std::vector, void >; + const std::vector chars{char(0), char(1), char(0xFF)}; + CHECK(json_char_binary::binary(chars).dump() == R"({"bytes":[0,1,255],"subtype":null})"); + + // the default binary type is unchanged + CHECK(json::binary({0, 1, 255}, 42).dump() == R"({"bytes":[0,1,255],"subtype":42})"); + } + + SECTION("hashing and UBJSON with a custom BinaryType") + { + const std::vector bytes{std::byte{0}, std::byte{1}, std::byte{0xFF}}; + const auto j = json_4804::binary(bytes); + + CHECK(std::hash {}(j) == std::hash {}(j)); + CHECK(json_4804::from_cbor(json_4804::to_cbor(j)) == j); + CHECK(json_4804::from_msgpack(json_4804::to_msgpack(j)) == j); + + // UBJSON has no binary type, so binary values are written as arrays + CHECK(json_4804::from_ubjson(json_4804::to_ubjson(j)) == json_4804({0, 1, 255})); + } + SECTION("discussion #4209 - custom BinaryType extraction from parsed array") { // Test that extracting a custom BinaryType from a parsed JSON array still works