amorfo77/json
1
0 Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-17 19:48:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix silent wrong results for custom number types

Browse Source 
Squashed onto develop cd6b428ef (CI fixes from #5860). Original commits:

- Fix silent wrong results for custom number types
- Fix CI: build the out_of_range.407 message from string_t and fix a link
- Fix CI: diagnostics message, clang-tidy, and MinGW section limit
- Format the out_of_range.407 number without dump()
- Move the std::deque array test out of unit-regression3.cpp
- Fix clang-tidy and clang 3.4/3.5 failures in the custom number type changes
- Fix CI: cast the 128-bit unsigned value explicitly in the integer fast path

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann committed 2026-10-16 19:26:47 +02:00
1 parent cd6b428ef0
commit 419e1678ee
15 files changed
+1127 -358
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/src/unit-custom-array-type.cpp
+46
View File
@@ -9,7 +9,9 @@
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <cstdint>
#include <deque>
#include <map>
#include <memory>
@@ -164,3 +166,47 @@ TEST_CASE("array type without at()")
CHECK(j.at(no_at_json::json_pointer("/1")) == 2);
CHECK_THROWS_AS(j.at(no_at_json::json_pointer("/3")), no_at_json::out_of_range);
}
TEST_CASE("regression test #5476 - array type without reserve()")
{
// the capacity reserved for definite-length arrays must not require the
// array type to have a reserve() member function
SECTION("std::deque")
{
const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})");
CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})");
// the binary formats pass a definite length to start_array()
CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j);
CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j);
CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j);
// parse() instantiates the callback parser as well, which reserves too
const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept
{
return true;
});
CHECK(with_callback == deque_json({1, 2, 3}));
}
SECTION("std::vector still reserves")
{
json array = json::array();
for (int i = 0; i < 100; ++i)
{
array.push_back(i);
}
const auto j = json::from_cbor(json::to_cbor(array));
CHECK(j == array);
CHECK(j.get_ref<const json::array_t&>().capacity() >= 100);
}
SECTION("the reservation stays capped")
{
// CBOR array announcing 2^32-1 elements, but truncated right after the
// header: the input must be rejected without reserving that capacity
const std::vector<std::uint8_t> truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
CHECK(json::from_cbor(truncated, true, false).is_discarded());
}
}
tests/src/unit-custom-number-types.cpp
+367
View File
@@ -0,0 +1,367 @@
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++ (supporting code)
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
#include <cstdint>
#include <limits>
#include <map>
#include <string>
#include <type_traits>
#include <vector>
#if JSON_HAS_THREE_WAY_COMPARISON
#include <compare>
#endif
namespace custom_number_types
{
// a signed integer of class type: std::is_signed is only true for arithmetic
// types, so the library has to take the signedness from std::numeric_limits,
// as it does for 128-bit and multiprecision integer classes such as
// absl::int128 or boost::multiprecision::cpp_int
class class_int
{
public:
// trivial, like the 128-bit integer classes: the value is stored in a union
class_int() = default;
// implicit from built-in integers and explicit from floats, like absl::int128;
// non-template overloads (as absl::int128 has) because clang 3.4 and 3.5
// reject basic_json's defaulted data() for a number type with template
// constructors
class_int(int v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
class_int(unsigned v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
class_int(long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
class_int(unsigned long v) : value(static_cast<std::int64_t>(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
class_int(long long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
class_int(unsigned long long v) : value(static_cast<std::int64_t>(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
explicit class_int(double v) : value(static_cast<std::int64_t>(v)) {}
template<typename T, typename std::enable_if<std::is_arithmetic<T>::value, int>::type = 0>
explicit operator T() const
{
return static_cast<T>(value);
}
friend bool operator==(class_int lhs, class_int rhs)
{
return lhs.value == rhs.value;
}
friend bool operator!=(class_int lhs, class_int rhs)
{
return lhs.value != rhs.value;
}
friend bool operator<(class_int lhs, class_int rhs)
{
return lhs.value < rhs.value;
}
#if JSON_HAS_THREE_WAY_COMPARISON
friend std::strong_ordering operator<=>(class_int lhs, class_int rhs) // *NOPAD*
{
return lhs.value <=> rhs.value; // *NOPAD*
}
#endif
private:
std::int64_t value;
};
} // namespace custom_number_types
using custom_number_types::class_int;
namespace std
{
// only the members the library uses
template<>
class numeric_limits<class_int>
{
public:
static constexpr bool is_signed = true;
static constexpr int digits = std::numeric_limits<std::int64_t>::digits;
};
} // namespace std
namespace
{
using class_int_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, class_int, std::uint64_t, double>;
class_int_json make_class_int(std::int64_t v)
{
class_int_json j(class_int_json::value_t::number_integer);
j.get_ref<class_int&>() = class_int(v);
return j;
}
} // namespace
// the serializer cannot print an integer of class type, so doctest must not
// try when an assertion fails
namespace doctest
{
template<>
struct StringMaker<class_int_json>
{
static String convert(const class_int_json& j)
{
return j.type_name();
}
};
} // namespace doctest
// __int128 as number type needs the std::numeric_limits (for the range checks)
// and std::is_integral (for the serializer) specializations, which libc++
// always provides and libstdc++ only outside strict ISO modes; the MSVC
// standard library has none
#if defined(__SIZEOF_INT128__) && (defined(_LIBCPP_VERSION) || defined(__GLIBCXX_TYPE_INT_N_0))
#define JSON_TEST_INT128_NUMBER_TYPES 1
#else
#define JSON_TEST_INT128_NUMBER_TYPES 0
#endif
#if JSON_TEST_INT128_NUMBER_TYPES
namespace
{
// __extension__ keeps -Wpedantic from flagging the non-standard type
__extension__ typedef __int128 int128; // NOLINT(modernize-use-using)
__extension__ typedef unsigned __int128 uint128; // NOLINT(modernize-use-using)
static_assert(std::numeric_limits<int128>::digits == 127 && std::numeric_limits<uint128>::digits == 128 &&
std::is_integral<int128>::value && std::is_integral<uint128>::value, // NOLINT(misc-redundant-expression)
"__int128 is not fully supported by the standard library");
using wide_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, int128, uint128, double>;
wide_json make_int(int128 v)
{
wide_json j(wide_json::value_t::number_integer);
j.get_ref<int128&>() = v;
return j;
}
wide_json make_uint(uint128 v)
{
wide_json j(wide_json::value_t::number_unsigned);
j.get_ref<uint128&>() = v;
return j;
}
wide_json wrap(const wide_json& value)
{
wide_json o(wide_json::value_t::object);
o["v"] = value;
return o;
}
} // namespace
TEST_CASE("custom number types: integers beyond 64 bits")
{
using out_of_range = wide_json::out_of_range;
const int128 two_64 = static_cast<int128>(1) << 64;
const int128 two_100 = static_cast<int128>(1) << 100;
const int128 max_int64 = (std::numeric_limits<std::int64_t>::max)();
const int128 min_int64 = (std::numeric_limits<std::int64_t>::min)();
const uint128 max_uint64 = (std::numeric_limits<std::uint64_t>::max)();
// before the range checks, these values were silently truncated, e.g.,
// 2^100 was written as 0
const wide_json int_big = make_int(two_100);
const wide_json int_big_negative = make_int(-two_100);
const wide_json uint_big = make_uint(static_cast<uint128>(two_100));
std::vector<std::uint8_t> _;
SECTION("CBOR")
{
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit uint64", out_of_range&);
// the extremes of the 128-bit types, whose digits are produced without
// negating the minimum as a whole
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits<int128>::min)())), "[json.exception.out_of_range.407] integer number -170141183460469231731687303715884105728 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits<int128>::max)())), "[json.exception.out_of_range.407] integer number 170141183460469231731687303715884105727 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_uint((std::numeric_limits<uint128>::max)())), "[json.exception.out_of_range.407] integer number 340282366920938463463374607431768211455 cannot be represented by CBOR as it does not fit uint64", out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(two_64)), out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(-two_64 - 1)), out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_cbor(make_uint(static_cast<uint128>(max_uint64) + 1)), out_of_range&);
// CBOR's integers cover [-2^64, 2^64-1]
CHECK(wide_json::to_cbor(make_int(two_64 - 1)) == std::vector<std::uint8_t>({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
CHECK(wide_json::to_cbor(make_int(-two_64)) == std::vector<std::uint8_t>({0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
CHECK(wide_json::to_cbor(make_uint(max_uint64)) == std::vector<std::uint8_t>({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
}
SECTION("MessagePack")
{
CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit uint64", out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(two_64)), out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(min_int64 - 1)), out_of_range&);
// MessagePack's integers cover [-2^63, 2^64-1]
CHECK(wide_json::to_msgpack(make_int(two_64 - 1)) == std::vector<std::uint8_t>({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
CHECK(wide_json::to_msgpack(make_int(min_int64)) == std::vector<std::uint8_t>({0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}));
CHECK(wide_json::to_msgpack(make_uint(max_uint64)) == std::vector<std::uint8_t>({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
}
SECTION("BSON")
{
// wrap() stores the value at /v, which JSON_DIAGNOSTICS adds to the message
#if JSON_DIAGNOSTICS
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] (/v) integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&);
#else
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&);
#endif
CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(max_int64 + 1))), out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(min_int64 - 1))), out_of_range&);
// nested values are checked as well, before anything is written
wide_json nested(wide_json::value_t::object);
nested["a"] = wide_json::array({wrap(int_big)});
std::vector<std::uint8_t> out;
CHECK_THROWS_AS(wide_json::to_bson(nested, out), out_of_range&);
CHECK(out.empty());
CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(max_int64)))) == wrap(make_int(max_int64)));
CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(min_int64)))) == wrap(make_int(min_int64)));
CHECK_NOTHROW(wide_json::to_bson(wrap(make_uint(max_uint64))));
}
SECTION("BON8")
{
CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&);
CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(max_int64 + 1)), out_of_range&);
CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(min_int64 - 1)), out_of_range&);
CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(max_int64))) == make_int(max_int64));
CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(min_int64))) == make_int(min_int64));
}
SECTION("UBJSON and BJData")
{
// integers beyond 64 bits are written exactly as high-precision numbers
const std::string digits = "1267650600228229401496703205376";
std::vector<std::uint8_t> expected = {'H', 'i', static_cast<std::uint8_t>(digits.size())};
expected.insert(expected.end(), digits.begin(), digits.end());
CHECK(wide_json::to_ubjson(int_big) == expected);
CHECK(wide_json::to_ubjson(uint_big) == expected);
CHECK(wide_json::to_bjdata(int_big) == expected);
// BJData's uint64 marker 'M' was used for any unsigned value beyond
// int64, which truncated the ones beyond 64 bits
CHECK(wide_json::to_bjdata(uint_big) == expected);
// the uint64 marker is still used where the value fits
CHECK(wide_json::to_bjdata(make_uint(max_uint64)) == std::vector<std::uint8_t>({'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}));
// the elements of an optimized container are written the same way;
// BJData does not allow 'H' as the type of an optimized container
std::vector<std::uint8_t> expected_ubjson_array = {'[', '$', 'H', '#', 'i', 2};
std::vector<std::uint8_t> expected_bjdata_array = {'[', '#', 'i', 2};
for (int i = 0; i < 2; ++i)
{
expected_ubjson_array.insert(expected_ubjson_array.end(), expected.begin() + 1, expected.end());
expected_bjdata_array.insert(expected_bjdata_array.end(), expected.begin(), expected.end());
}
CHECK(wide_json::to_ubjson(wide_json::array({uint_big, uint_big}), true, true) == expected_ubjson_array);
CHECK(wide_json::to_bjdata(wide_json::array({uint_big, uint_big}), true, true) == expected_bjdata_array);
}
SECTION("BJData ND-array")
{
// an ND-array element beyond 64 bits does not fit any dtype, so the
// annotated object is written as a plain object instead of truncating
// the element into range
wide_json element(wide_json::value_t::object);
element["_ArrayType_"] = "uint8";
element["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(1)});
element["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(static_cast<uint128>(two_64) + 1)});
const auto element_bytes = wide_json::to_bjdata(element, true, true);
REQUIRE(!element_bytes.empty());
CHECK(element_bytes[0] == '{');
wide_json signed_element = element;
signed_element["_ArrayType_"] = "int8";
signed_element["_ArrayData_"] = wide_json::array({make_int(1), make_int(-two_64 + 1)});
const auto signed_element_bytes = wide_json::to_bjdata(signed_element, true, true);
REQUIRE(!signed_element_bytes.empty());
CHECK(signed_element_bytes[0] == '{');
// the same for a dimension beyond 64 bits, which wrapped into a
// dimension matching the size of _ArrayData_ (here: 1)
wide_json dimension = element;
dimension["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(static_cast<uint128>(two_64) + 1)});
dimension["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)});
const auto dimension_bytes = wide_json::to_bjdata(dimension, true, true);
REQUIRE(!dimension_bytes.empty());
CHECK(dimension_bytes[0] == '{');
// in range, the ND-array is still written
wide_json fits = element;
fits["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)});
const auto fits_bytes = wide_json::to_bjdata(fits, true, true);
REQUIRE(!fits_bytes.empty());
CHECK(fits_bytes[0] == '[');
}
}
#endif
TEST_CASE("custom number types")
{
SECTION("signed integer of class type compared with a float")
{
// std::is_signed is false for a class type, which made the comparison
// treat any float below zero as less than every integer
const class_int_json minus_five = make_class_int(-5);
const class_int_json minus_two = make_class_int(-2);
const class_int_json five = make_class_int(5);
const class_int_json minus_two_and_a_half = -2.5;
const class_int_json two_and_a_half = 2.5;
const class_int_json huge_negative = -1e30;
const class_int_json huge_positive = 1e30;
CHECK(minus_five < minus_two_and_a_half);
CHECK_FALSE(minus_two_and_a_half < minus_five);
CHECK(minus_two_and_a_half > minus_five);
CHECK(minus_five <= minus_two_and_a_half);
CHECK(minus_two_and_a_half >= minus_five);
CHECK(minus_five != minus_two_and_a_half);
CHECK(minus_two_and_a_half < minus_two);
CHECK_FALSE(minus_two < minus_two_and_a_half);
CHECK(two_and_a_half < five);
CHECK(minus_five < two_and_a_half);
// floats beyond the integer's range
CHECK(huge_negative < minus_five);
CHECK_FALSE(minus_five < huge_negative);
CHECK(five < huge_positive);
CHECK_FALSE(huge_positive < five);
// equality
const class_int_json minus_two_float = -2.0;
CHECK(minus_two == minus_two_float);
CHECK(minus_two_float == minus_two);
}
}
tests/src/unit-msgpack-custom-types.cpp
+296
View File
@@ -0,0 +1,296 @@
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++ (supporting code)
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-License-Identifier: MIT
// MessagePack tests with basic_json specializations other than nlohmann::json;
// they are kept apart from unit-msgpack.cpp because every specialization adds
// many sections to the object file, and MinGW cannot link objects with more
// than 65535 sections (see the clang job in .github/workflows/windows.yml)
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
using nlohmann::json;
#include <cstdint> // SIZE_MAX, UINT32_MAX
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
// the fake sizes below do not fit into a 32-bit std::size_t
// with clang and libstdc++ 10, the std::filesystem::path conversion that
// C++17 builds consider for every string type is ambiguous for a class
// derived from std::string, so the string case is not tested there
#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11)
#define JSON_TEST_BEYOND_UINT32_STRING 1
#endif
#if SIZE_MAX > UINT32_MAX
template<typename T, typename A = std::allocator<T>>
struct huge_array : std::vector<T, A>
{
using base = std::vector<T, A>;
using base::base;
bool fake_size = false;
std::size_t size() const noexcept
{
if (fake_size)
{
return (std::numeric_limits<std::uint32_t>::max)() + 1ULL;
}
return base::size();
}
};
using huge_array_json = nlohmann::json::with_array_t<huge_array>;
TEST_CASE("MessagePack Size above uint32 for array")
{
huge_array_json j = huge_array_json::array();
j.push_back(1);
j.push_back(2);
j.push_back(3);
auto& array = j.get_ref<huge_array_json::array_t&>();
array.fake_size = true;
// write into a caller-owned vector: to_msgpack(j) reserves space based on
// the (faked) element count, which fails with bad_alloc on Windows
std::vector<std::uint8_t> result;
CHECK_THROWS_WITH_AS(
huge_array_json::to_msgpack(j, result),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
array.fake_size = false;
}
template<typename K, typename V,
typename C = std::less<K>,
typename A = std::allocator<std::pair<const K, V>>>
struct huge_map : std::map<K, V, C, A>
{
using base = std::map<K, V, C, A>;
using base::base;
bool fake_size = false;
std::size_t size() const noexcept
{
if (fake_size)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
return base::size();
}
};
using huge_object_json = nlohmann::json::with_object_t<huge_map>;
TEST_CASE("MessagePack Size above uint32 for object")
{
huge_object_json j = huge_object_json::object();
j["one"] = 1;
j["two"] = 2;
auto& object = j.get_ref<huge_object_json::object_t&>();
object.fake_size = true;
// write into a caller-owned vector: to_msgpack(j) reserves space based on
// the (faked) element count, which fails with bad_alloc on Windows
std::vector<std::uint8_t> result;
CHECK_THROWS_WITH_AS(
huge_object_json::to_msgpack(j, result),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
object.fake_size = false;
}
#ifdef JSON_TEST_BEYOND_UINT32_STRING
struct huge_string : std::string
{
using std::string::string;
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
};
using huge_string_json = nlohmann::json::with_string_t<huge_string>;
TEST_CASE("MessagePack Size above uint32 for string")
{
const huge_string_json j = "hello";
CHECK_THROWS_WITH_AS(
huge_string_json::to_msgpack(j),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
}
#endif
struct huge_binary : std::vector<std::uint8_t>
{
using std::vector<std::uint8_t>::vector;
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
};
using huge_binary_json = nlohmann::json::with_binary_t<huge_binary>;
TEST_CASE("MessagePack Size above uint32 for binary")
{
huge_binary_json j = huge_binary_json::binary(huge_binary{});
j.get_binary().push_back(0x01);
j.get_binary().push_back(0x02);
CHECK_THROWS_WITH_AS(
huge_binary_json::to_msgpack(j),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
}
#endif
namespace
{
// types that report a size beyond UINT32_MAX without allocating that much
// memory, so the MessagePack length limit can be tested cheaply; see the
// similar types in unit-bson.cpp
std::size_t beyond_uint32_size()
{
return static_cast<std::size_t>((std::numeric_limits<std::uint32_t>::max)()) + 1;
}
class beyond_uint32_binary_t : public std::vector<std::uint8_t>
{
public:
using std::vector<std::uint8_t>::vector;
size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return beyond_uint32_size();
}
};
#ifdef JSON_TEST_BEYOND_UINT32_STRING
class beyond_uint32_string_t : public std::string
{
public:
using std::string::string;
size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return beyond_uint32_size();
}
};
using beyond_uint32_string_json = nlohmann::json::with_string_t<beyond_uint32_string_t>;
#endif
using beyond_uint32_binary_json = nlohmann::json::with_binary_t<beyond_uint32_binary_t>;
} // namespace
TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized")
{
// MessagePack stores the length of a string, binary value, array, or
// object in at most 32 bits; a larger one used to be written without any
// length at all
#if SIZE_MAX > UINT32_MAX
{
const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295";
const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{});
CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&);
const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42);
CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&);
#ifdef JSON_TEST_BEYOND_UINT32_STRING
// created from its type rather than from a beyond_uint32_string_t:
// that would consider the std::filesystem::path conversion, which
// libstdc++ 10 cannot decide for a class derived from std::string
const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string);
CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&);
#endif
}
#endif
}
TEST_CASE("MessagePack numbers use the active union member (see #5644)")
{
// when number_integer_t is narrower than number_unsigned_t, to_msgpack()
// used to read the union member that was not the active one, writing
// wrong bytes for some values; std::int64_t/std::uint64_t (the default
// types, where both members have the same width) were not affected
using int32_json = nlohmann::json::with_integers_t<std::int32_t, std::uint64_t>;
using int16_json = nlohmann::json::with_integers_t<std::int16_t, std::uint64_t>;
SECTION("number_integer_t = std::int32_t")
{
SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)")
{
const int32_json j = 6442450944ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00};
const auto result = int32_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int32_json::from_msgpack(result) == j);
}
SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)")
{
const int32_json j = 4294967496ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
const auto result = int32_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int32_json::from_msgpack(result) == j);
}
}
SECTION("number_integer_t = std::int16_t, 98304 (uint 32)")
{
const int16_json j = 98304ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xce, 0x00, 0x01, 0x80, 0x00};
const auto result = int16_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int16_json::from_msgpack(result) == j);
}
SECTION("default types (std::int64_t/std::uint64_t) are unaffected")
{
const json j = 4294967496ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
const auto result = json::to_msgpack(j);
CHECK(result == expected);
CHECK(json::from_msgpack(result) == j);
}
}
tests/src/unit-msgpack.cpp
-270
View File
@@ -2173,276 +2173,6 @@ TEST_CASE("MessagePack roundtrips" * doctest::skip())
}
}
// the fake sizes below do not fit into a 32-bit std::size_t
// with clang and libstdc++ 10, the std::filesystem::path conversion that
// C++17 builds consider for every string type is ambiguous for a class
// derived from std::string, so the string case is not tested there
#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11)
#define JSON_TEST_BEYOND_UINT32_STRING 1
#endif
#if SIZE_MAX > UINT32_MAX
template<typename T, typename A = std::allocator<T>>
struct huge_array : std::vector<T, A>
{
using base = std::vector<T, A>;
using base::base;
bool fake_size = false;
std::size_t size() const noexcept
{
if (fake_size)
{
return (std::numeric_limits<std::uint32_t>::max)() + 1ULL;
}
return base::size();
}
};
using huge_array_json = nlohmann::json::with_array_t<huge_array>;
TEST_CASE("MessagePack Size above uint32 for array")
{
huge_array_json j = huge_array_json::array();
j.push_back(1);
j.push_back(2);
j.push_back(3);
auto& array = j.get_ref<huge_array_json::array_t&>();
array.fake_size = true;
// write into a caller-owned vector: to_msgpack(j) reserves space based on
// the (faked) element count, which fails with bad_alloc on Windows
std::vector<std::uint8_t> result;
CHECK_THROWS_WITH_AS(
huge_array_json::to_msgpack(j, result),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
array.fake_size = false;
}
template<typename K, typename V,
typename C = std::less<K>,
typename A = std::allocator<std::pair<const K, V>>>
struct huge_map : std::map<K, V, C, A>
{
using base = std::map<K, V, C, A>;
using base::base;
bool fake_size = false;
std::size_t size() const noexcept
{
if (fake_size)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
return base::size();
}
};
using huge_object_json = nlohmann::json::with_object_t<huge_map>;
TEST_CASE("MessagePack Size above uint32 for object")
{
huge_object_json j = huge_object_json::object();
j["one"] = 1;
j["two"] = 2;
auto& object = j.get_ref<huge_object_json::object_t&>();
object.fake_size = true;
// write into a caller-owned vector: to_msgpack(j) reserves space based on
// the (faked) element count, which fails with bad_alloc on Windows
std::vector<std::uint8_t> result;
CHECK_THROWS_WITH_AS(
huge_object_json::to_msgpack(j, result),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
object.fake_size = false;
}
#ifdef JSON_TEST_BEYOND_UINT32_STRING
struct huge_string : std::string
{
using std::string::string;
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
};
using huge_string_json = nlohmann::json::with_string_t<huge_string>;
TEST_CASE("MessagePack Size above uint32 for string")
{
const huge_string_json j = "hello";
CHECK_THROWS_WITH_AS(
huge_string_json::to_msgpack(j),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
}
#endif
struct huge_binary : std::vector<std::uint8_t>
{
using std::vector<std::uint8_t>::vector;
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
}
};
using huge_binary_json = nlohmann::json::with_binary_t<huge_binary>;
TEST_CASE("MessagePack Size above uint32 for binary")
{
huge_binary_json j = huge_binary_json::binary(huge_binary{});
j.get_binary().push_back(0x01);
j.get_binary().push_back(0x02);
CHECK_THROWS_WITH_AS(
huge_binary_json::to_msgpack(j),
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
json::out_of_range&);
}
#endif
namespace
{
// types that report a size beyond UINT32_MAX without allocating that much
// memory, so the MessagePack length limit can be tested cheaply; see the
// similar types in unit-bson.cpp
std::size_t beyond_uint32_size()
{
return static_cast<std::size_t>((std::numeric_limits<std::uint32_t>::max)()) + 1;
}
class beyond_uint32_binary_t : public std::vector<std::uint8_t>
{
public:
using std::vector<std::uint8_t>::vector;
size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return beyond_uint32_size();
}
};
#ifdef JSON_TEST_BEYOND_UINT32_STRING
class beyond_uint32_string_t : public std::string
{
public:
using std::string::string;
size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
{
return beyond_uint32_size();
}
};
using beyond_uint32_string_json = nlohmann::json::with_string_t<beyond_uint32_string_t>;
#endif
using beyond_uint32_binary_json = nlohmann::json::with_binary_t<beyond_uint32_binary_t>;
} // namespace
TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized")
{
// MessagePack stores the length of a string, binary value, array, or
// object in at most 32 bits; a larger one used to be written without any
// length at all
#if SIZE_MAX > UINT32_MAX
{
const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295";
const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{});
CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&);
const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42);
CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&);
#ifdef JSON_TEST_BEYOND_UINT32_STRING
// created from its type rather than from a beyond_uint32_string_t:
// that would consider the std::filesystem::path conversion, which
// libstdc++ 10 cannot decide for a class derived from std::string
const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string);
CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&);
#endif
}
#endif
}
TEST_CASE("MessagePack numbers use the active union member (see #5644)")
{
// when number_integer_t is narrower than number_unsigned_t, to_msgpack()
// used to read the union member that was not the active one, writing
// wrong bytes for some values; std::int64_t/std::uint64_t (the default
// types, where both members have the same width) were not affected
using int32_json = nlohmann::json::with_integers_t<std::int32_t, std::uint64_t>;
using int16_json = nlohmann::json::with_integers_t<std::int16_t, std::uint64_t>;
SECTION("number_integer_t = std::int32_t")
{
SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)")
{
const int32_json j = 6442450944ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00};
const auto result = int32_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int32_json::from_msgpack(result) == j);
}
SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)")
{
const int32_json j = 4294967496ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
const auto result = int32_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int32_json::from_msgpack(result) == j);
}
}
SECTION("number_integer_t = std::int16_t, 98304 (uint 32)")
{
const int16_json j = 98304ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xce, 0x00, 0x01, 0x80, 0x00};
const auto result = int16_json::to_msgpack(j);
CHECK(result == expected);
CHECK(int16_json::from_msgpack(result) == j);
}
SECTION("default types (std::int64_t/std::uint64_t) are unaffected")
{
const json j = 4294967496ULL;
CHECK(j.is_number_unsigned());
std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
const auto result = json::to_msgpack(j);
CHECK(result == expected);
CHECK(json::from_msgpack(result) == j);
}
}
TEST_CASE("MessagePack large strings and binaries (chunked reader)")
{
// get_msgpack_string()/get_msgpack_binary() both read through get_binary(),
tests/src/unit-regression3.cpp
-47
View File
@@ -43,7 +43,6 @@ using ordered_json = nlohmann::ordered_json;
#endif
#include <cstdio>
#include <deque>
#include <list>
#include <type_traits>
#include <utility>
@@ -867,52 +866,6 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with
}
TEST_CASE("regression test #5476 - array type without reserve()")
{
// the capacity reserved for definite-length arrays must not require the
// array type to have a reserve() member function
using deque_json = nlohmann::json::with_array_t<std::deque>;
SECTION("std::deque")
{
const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})");
CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})");
// the binary formats pass a definite length to start_array()
CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j);
CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j);
CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j);
// parse() instantiates the callback parser as well, which reserves too
const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept
{
return true;
});
CHECK(with_callback == deque_json({1, 2, 3}));
}
SECTION("std::vector still reserves")
{
json array = json::array();
for (int i = 0; i < 100; ++i)
{
array.push_back(i);
}
const auto j = json::from_cbor(json::to_cbor(array));
CHECK(j == array);
CHECK(j.get_ref<const json::array_t&>().capacity() >= 100);
}
SECTION("the reservation stays capped")
{
// CBOR array announcing 2^32-1 elements, but truncated right after the
// header: the input must be rejected without reserving that capacity
const std::vector<std::uint8_t> truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF};
CHECK(json::from_cbor(truncated, true, false).is_discarded());
}
}
TEST_CASE("issue #5317 - nested indefinite-length CBOR string chunks are rejected")
{
json _;