diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 583139068..aa0240a73 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -123,4 +123,6 @@ Linear in the size of the JSON value `j`. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an - array or object was silently skipped, producing invalid BJData. \ No newline at end of file + array or object was silently skipped, producing invalid BJData. +- Writes unsigned integers wider than 64 bits as high-precision numbers since version 3.13.0; previously, they were + silently truncated to 64 bits. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bon8.md b/docs/mkdocs/docs/api/basic_json/to_bon8.md index 14cc085f6..debbb6ef0 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bon8.md +++ b/docs/mkdocs/docs/api/basic_json/to_bon8.md @@ -37,8 +37,9 @@ With (2), the bytes written before the exception remain in the output adapter. ## Exceptions -- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an unsigned integer - above 9223372036854775807, which BON8 cannot represent +- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer outside + the range of int64 (an unsigned integer above 9223372036854775807, or, with a number type wider than 64 bits, any + integer beyond int64), which BON8 cannot represent - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if `j` contains a string that is not valid UTF-8 - Throws [type_error.321](../../home/exceptions.md#jsonexceptiontype_error321) if `j` or a value nested in it is diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index 05f8b8548..315477e19 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -47,6 +47,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`type_error.317`](../../home/exceptions.md#jsonexceptiontype_error317) if the top-level type of the JSON value is not an object; example: `"to serialize to BSON, top-level type must be object, but is string"` +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains a signed integer + outside the range of int64 or an unsigned integer outside the range of uint64, which is only possible with a number + type wider than 64 bits; example: + `"integer number 9223372036854775808 cannot be represented by BSON as it does not fit int64"` - Throws [`out_of_range.409`](../../home/exceptions.md#jsonexceptionout_of_range409) if a key in the JSON object contains a null byte (code point U+0000); example: `"BSON key cannot contain code point U+0000 (at byte 2)"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a document, array, @@ -119,3 +123,5 @@ pass before anything is written. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316` before anything is written. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_cbor.md b/docs/mkdocs/docs/api/basic_json/to_cbor.md index f5b6a07bb..6508864e1 100644 --- a/docs/mkdocs/docs/api/basic_json/to_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/to_cbor.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^64, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]"` - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not valid UTF-8 and `error_handler` is `strict` (the default only if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) @@ -90,3 +93,5 @@ Linear in the size of the JSON value `j`. [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid CBOR. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_msgpack.md b/docs/mkdocs/docs/api/basic_json/to_msgpack.md index 2e32208bf..dca66271d 100644 --- a/docs/mkdocs/docs/api/basic_json/to_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/to_msgpack.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^63, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a string, binary value, array, or object exceeds 4294967295, the maximum MessagePack can store; example: `"MessagePack length 4294967296 exceeds maximum of 4294967295"` @@ -112,3 +115,5 @@ Linear in the size of the JSON value `j`. `number_unsigned_t`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid MessagePack. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 8f1016214..7658fb3cb 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -972,9 +972,19 @@ double-precision number when `number_float_t` is `#!cpp float`. ### json.exception.out_of_range.407 -An unsigned integer number greater than 9223372036854775807 cannot be represented in the binary format. This exception -is only thrown by [`to_bon8`](../api/basic_json/to_bon8.md), as [BON8](../features/binary_formats/bon8.md) has no -unsigned 64-bit integer type. +An integer number cannot be represented by the binary format it is serialized to: + +- [BON8](../features/binary_formats/bon8.md) only stores integers that fit into int64, so + [`to_bon8`](../api/basic_json/to_bon8.md) throws for unsigned integers greater than 9223372036854775807. +- [CBOR](../features/binary_formats/cbor.md), [MessagePack](../features/binary_formats/messagepack.md), and + [BSON](../features/binary_formats/bson.md) store integers in at most 64 bits. With the default number types, every + integer fits, but a [`number_integer_t`](../api/basic_json/number_integer_t.md) or + [`number_unsigned_t`](../api/basic_json/number_unsigned_t.md) wider than 64 bits (e.g., `__int128`) can hold values + outside the range of the format: [-2^64, 2^64-1] for CBOR, [-2^63, 2^64-1] for MessagePack, and the range of int64 + (signed integers) or uint64 (unsigned integers) for BSON. + +[UBJSON](../features/binary_formats/ubjson.md) and [BJData](../features/binary_formats/bjdata.md) never throw this +exception, because they serialize integers beyond 64 bits as high-precision numbers. !!! note "Former occurrences" @@ -983,11 +993,17 @@ unsigned 64-bit integer type. - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers. - Since version 3.12.0, integer numbers beyond int64 are serialized as uint64 BSON numbers. -!!! failure "Example message" + Before version 3.13.0, CBOR, MessagePack, and BSON silently truncated integers wider than 64 bits, and BJData + truncated unsigned integers wider than 64 bits. + +!!! failure "Example messages" ``` integer number 18446744073709551615 cannot be represented by BON8 as it does not fit int64 ``` + ``` + integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1] + ``` ### json.exception.out_of_range.408 diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 0fe8b51f9..981035165 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -210,7 +210,7 @@ class binary_writer // one case for both, so that an array mixing negative // (number_integer) and non-negative (number_unsigned) values // does not mispredict the switch - write_cbor_integer(j); + write_cbor_integer(j, std::integral_constant()); break; } @@ -408,7 +408,7 @@ class binary_writer // one case for both, so that an array mixing negative // (number_integer) and non-negative (number_unsigned) values // does not mispredict the switch - write_msgpack_integer(j, std::integral_constant()); + write_msgpack_integer(j, std::integral_constant()); break; } @@ -759,6 +759,142 @@ class binary_writer JSON_THROW(type_error::create(exception_id::discarded_value_used, concat("cannot serialize discarded value to ", format_name), &j)); } + /// how the range of integer type @a NumberType relates to that of @a TargetType + template + struct integer_range + { + /// whether NumberType can hold values above the maximum of TargetType + static constexpr bool may_exceed_max = std::numeric_limits::digits > std::numeric_limits::digits; + /// whether NumberType can hold values below the minimum of TargetType + static constexpr bool may_fall_below_min = std::numeric_limits::is_signed + && (!std::numeric_limits::is_signed || may_exceed_max); + }; + + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant::may_exceed_max>()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant::may_fall_below_min>()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief the decimal digits of the non-negative integer @a n + */ + template + static std::string integer_to_string(NumberType n, std::false_type /*is_signed*/) + { + std::string result; + do + { + result.push_back(static_cast('0' + (n % 10))); + n = static_cast(n / 10); + } + while (n != 0); + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief the decimal digits of the integer @a n, with a leading '-' if it + is negative + */ + template + static std::string integer_to_string(NumberType n, std::true_type /*is_signed*/) + { + // the remainder of a negative n is not positive; each digit is + // negated on its own, because negating the minimum of NumberType + // as a whole would overflow + const bool negative = n < 0; + std::string result; + do + { + const auto remainder = n % 10; + result.push_back(static_cast('0' + (remainder < 0 ? -remainder : remainder))); + n = static_cast(n / 10); + } + while (n != 0); + if (negative) + { + result.push_back('-'); + } + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // neither std::to_string(), which has no overload for integer types + // wider than 64 bits, nor dump(), which would also instantiate the + // serializer for objects and fail for key types it cannot write + using integer_limits = std::numeric_limits; + using unsigned_limits = std::numeric_limits; + const std::string number = j.is_number_unsigned() + ? integer_to_string(j.m_data.m_value.number_unsigned, std::integral_constant()) + : integer_to_string(j.m_data.m_value.number_integer, std::integral_constant()); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -1515,6 +1651,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -1533,9 +1671,17 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: @@ -1574,8 +1720,9 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); @@ -1993,6 +2140,11 @@ class binary_writer void write_cbor_array(const BasicJsonType& j, const std::size_t depth) { write_cbor_head(0x80, j.m_data.m_value.array->size()); + write_cbor_array_elements(j, depth, std::integral_constant()); + } + + void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) + { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_cbor_integer(out, el); @@ -2002,6 +2154,14 @@ class binary_writer }); } + void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::false_type /*fits_64_bits*/) + { + for (const auto& el : *j.m_data.m_value.array) + { + write_cbor(el, depth + 1); + } + } + /*! @brief write the elements of the array @a j, runs of integers with @a encode_integer and all other values with @a write_value @@ -2045,11 +2205,46 @@ class binary_writer small (runs of integers in an array are encoded inline instead). */ JSON_HEDLEY_NEVER_INLINE - void write_cbor_integer(const BasicJsonType& j) + void write_cbor_integer(const BasicJsonType& j, std::true_type /*fits_64_bits*/) { oa.commit(encode_cbor_integer(oa.reserve(integer_encoding_space), j)); } + /// write a number_integer or number_unsigned value of a number type wider + /// than 64 bits as a CBOR integer, with range checks on the whole value + void write_cbor_integer(const BasicJsonType& j, std::false_type /*fits_64_bits*/) + { + if (j.m_data.m_type == value_t::number_unsigned) + { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); + return; + } + + if (j.m_data.m_value.number_integer >= 0) + { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } + // CBOR does not differentiate between positive signed + // integers and unsigned integers + write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); + } + else + { + // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } + write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); + } + } + /// encode a number_integer or number_unsigned value to @a out (see /// @ref write_cbor_integer) and return the number of characters std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept @@ -2061,13 +2256,13 @@ class binary_writer } /*! - @brief whether the MessagePack integers of BasicJsonType are written by - @ref encode_msgpack_integer + @brief whether the CBOR and MessagePack integers of BasicJsonType are + written by @ref encode_cbor_integer and @ref encode_msgpack_integer - It picks the format from the low 64 bits; a wider number type keeps the - range checks of @ref write_msgpack on the whole value. + These pick the format from the low 64 bits; a wider number type is written + with range checks on the whole value instead. */ - static constexpr bool msgpack_integers_fit_64_bits() noexcept + static constexpr bool integers_fit_64_bits() noexcept { return sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) && sizeof(typename BasicJsonType::number_unsigned_t) <= sizeof(std::uint64_t); @@ -2118,18 +2313,30 @@ class binary_writer { if (j.m_data.m_type == value_t::number_unsigned) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); return; } if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -2174,7 +2381,7 @@ class binary_writer void write_msgpack_array(const BasicJsonType& j, const std::size_t depth) { write_msgpack_array_prefix(j.m_data.m_value.array->size(), j); - write_msgpack_array_elements(j, depth, std::integral_constant()); + write_msgpack_array_elements(j, depth, std::integral_constant()); } void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) @@ -2392,7 +2599,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -2517,14 +2724,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -2756,10 +2965,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -2891,9 +3101,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -2902,6 +3112,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 3621a78d9..1dd5e41e3 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1525,7 +1525,7 @@ class serializer // without leading zeros if (abs_value < 10000000000000000u) { - const std::uint64_t value = abs_value; + const auto value = static_cast(abs_value); // fits: abs_value < 10^16 const std::uint64_t upper = value / 100000000u; const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) diff --git a/include/nlohmann/detail/value_t.hpp b/include/nlohmann/detail/value_t.hpp index 70d600365..9f64cb90b 100644 --- a/include/nlohmann/detail/value_t.hpp +++ b/include/nlohmann/detail/value_t.hpp @@ -14,7 +14,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed #include #if JSON_HAS_THREE_WAY_COMPARISON @@ -208,13 +207,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 2f22640b5..9fc04635c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -326,7 +326,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed // #include // __ _____ _____ _____ @@ -3466,13 +3465,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } @@ -23853,7 +23855,7 @@ class binary_writer // one case for both, so that an array mixing negative // (number_integer) and non-negative (number_unsigned) values // does not mispredict the switch - write_cbor_integer(j); + write_cbor_integer(j, std::integral_constant()); break; } @@ -24051,7 +24053,7 @@ class binary_writer // one case for both, so that an array mixing negative // (number_integer) and non-negative (number_unsigned) values // does not mispredict the switch - write_msgpack_integer(j, std::integral_constant()); + write_msgpack_integer(j, std::integral_constant()); break; } @@ -24402,6 +24404,142 @@ class binary_writer JSON_THROW(type_error::create(exception_id::discarded_value_used, concat("cannot serialize discarded value to ", format_name), &j)); } + /// how the range of integer type @a NumberType relates to that of @a TargetType + template + struct integer_range + { + /// whether NumberType can hold values above the maximum of TargetType + static constexpr bool may_exceed_max = std::numeric_limits::digits > std::numeric_limits::digits; + /// whether NumberType can hold values below the minimum of TargetType + static constexpr bool may_fall_below_min = std::numeric_limits::is_signed + && (!std::numeric_limits::is_signed || may_exceed_max); + }; + + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant::may_exceed_max>()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant::may_fall_below_min>()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief the decimal digits of the non-negative integer @a n + */ + template + static std::string integer_to_string(NumberType n, std::false_type /*is_signed*/) + { + std::string result; + do + { + result.push_back(static_cast('0' + (n % 10))); + n = static_cast(n / 10); + } + while (n != 0); + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief the decimal digits of the integer @a n, with a leading '-' if it + is negative + */ + template + static std::string integer_to_string(NumberType n, std::true_type /*is_signed*/) + { + // the remainder of a negative n is not positive; each digit is + // negated on its own, because negating the minimum of NumberType + // as a whole would overflow + const bool negative = n < 0; + std::string result; + do + { + const auto remainder = n % 10; + result.push_back(static_cast('0' + (remainder < 0 ? -remainder : remainder))); + n = static_cast(n / 10); + } + while (n != 0); + if (negative) + { + result.push_back('-'); + } + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // neither std::to_string(), which has no overload for integer types + // wider than 64 bits, nor dump(), which would also instantiate the + // serializer for objects and fail for key types it cannot write + using integer_limits = std::numeric_limits; + using unsigned_limits = std::numeric_limits; + const std::string number = j.is_number_unsigned() + ? integer_to_string(j.m_data.m_value.number_unsigned, std::integral_constant()) + : integer_to_string(j.m_data.m_value.number_integer, std::integral_constant()); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -25158,6 +25296,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -25176,9 +25316,17 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: @@ -25217,8 +25365,9 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); @@ -25636,6 +25785,11 @@ class binary_writer void write_cbor_array(const BasicJsonType& j, const std::size_t depth) { write_cbor_head(0x80, j.m_data.m_value.array->size()); + write_cbor_array_elements(j, depth, std::integral_constant()); + } + + void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) + { write_array_elements(j, [this](CharType * out, const BasicJsonType & el) { return encode_cbor_integer(out, el); @@ -25645,6 +25799,14 @@ class binary_writer }); } + void write_cbor_array_elements(const BasicJsonType& j, const std::size_t depth, std::false_type /*fits_64_bits*/) + { + for (const auto& el : *j.m_data.m_value.array) + { + write_cbor(el, depth + 1); + } + } + /*! @brief write the elements of the array @a j, runs of integers with @a encode_integer and all other values with @a write_value @@ -25688,11 +25850,46 @@ class binary_writer small (runs of integers in an array are encoded inline instead). */ JSON_HEDLEY_NEVER_INLINE - void write_cbor_integer(const BasicJsonType& j) + void write_cbor_integer(const BasicJsonType& j, std::true_type /*fits_64_bits*/) { oa.commit(encode_cbor_integer(oa.reserve(integer_encoding_space), j)); } + /// write a number_integer or number_unsigned value of a number type wider + /// than 64 bits as a CBOR integer, with range checks on the whole value + void write_cbor_integer(const BasicJsonType& j, std::false_type /*fits_64_bits*/) + { + if (j.m_data.m_type == value_t::number_unsigned) + { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } + write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); + return; + } + + if (j.m_data.m_value.number_integer >= 0) + { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } + // CBOR does not differentiate between positive signed + // integers and unsigned integers + write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); + } + else + { + // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } + write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); + } + } + /// encode a number_integer or number_unsigned value to @a out (see /// @ref write_cbor_integer) and return the number of characters std::size_t encode_cbor_integer(CharType* out, const BasicJsonType& j) const noexcept @@ -25704,13 +25901,13 @@ class binary_writer } /*! - @brief whether the MessagePack integers of BasicJsonType are written by - @ref encode_msgpack_integer + @brief whether the CBOR and MessagePack integers of BasicJsonType are + written by @ref encode_cbor_integer and @ref encode_msgpack_integer - It picks the format from the low 64 bits; a wider number type keeps the - range checks of @ref write_msgpack on the whole value. + These pick the format from the low 64 bits; a wider number type is written + with range checks on the whole value instead. */ - static constexpr bool msgpack_integers_fit_64_bits() noexcept + static constexpr bool integers_fit_64_bits() noexcept { return sizeof(typename BasicJsonType::number_integer_t) <= sizeof(std::uint64_t) && sizeof(typename BasicJsonType::number_unsigned_t) <= sizeof(std::uint64_t); @@ -25761,18 +25958,30 @@ class binary_writer { if (j.m_data.m_type == value_t::number_unsigned) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); return; } if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -25817,7 +26026,7 @@ class binary_writer void write_msgpack_array(const BasicJsonType& j, const std::size_t depth) { write_msgpack_array_prefix(j.m_data.m_value.array->size(), j); - write_msgpack_array_elements(j, depth, std::integral_constant()); + write_msgpack_array_elements(j, depth, std::integral_constant()); } void write_msgpack_array_elements(const BasicJsonType& j, const std::size_t depth, std::true_type /*fits_64_bits*/) @@ -26035,7 +26244,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -26160,14 +26369,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -26399,10 +26610,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -26534,9 +26746,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -26545,6 +26757,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; @@ -30489,7 +30705,7 @@ class serializer // without leading zeros if (abs_value < 10000000000000000u) { - const std::uint64_t value = abs_value; + const auto value = static_cast(abs_value); // fits: abs_value < 10^16 const std::uint64_t upper = value / 100000000u; const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index 6f015d234..e602b4716 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -9,7 +9,9 @@ #include "doctest_compatibility.h" #include +using nlohmann::json; +#include #include #include #include @@ -164,3 +166,47 @@ TEST_CASE("array type without at()") CHECK(j.at(no_at_json::json_pointer("/1")) == 2); CHECK_THROWS_AS(j.at(no_at_json::json_pointer("/3")), no_at_json::out_of_range); } + +TEST_CASE("regression test #5476 - array type without reserve()") +{ + // the capacity reserved for definite-length arrays must not require the + // array type to have a reserve() member function + SECTION("std::deque") + { + const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); + CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); + + // the binary formats pass a definite length to start_array() + CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); + CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); + CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j); + + // parse() instantiates the callback parser as well, which reserves too + const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept + { + return true; + }); + CHECK(with_callback == deque_json({1, 2, 3})); + } + + SECTION("std::vector still reserves") + { + json array = json::array(); + for (int i = 0; i < 100; ++i) + { + array.push_back(i); + } + + const auto j = json::from_cbor(json::to_cbor(array)); + CHECK(j == array); + CHECK(j.get_ref().capacity() >= 100); + } + + SECTION("the reservation stays capped") + { + // CBOR array announcing 2^32-1 elements, but truncated right after the + // header: the input must be rejected without reserving that capacity + const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_cbor(truncated, true, false).is_discarded()); + } +} diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp new file mode 100644 index 000000000..f9c197574 --- /dev/null +++ b/tests/src/unit-custom-number-types.cpp @@ -0,0 +1,367 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if JSON_HAS_THREE_WAY_COMPARISON + #include +#endif + +namespace custom_number_types +{ + +// a signed integer of class type: std::is_signed is only true for arithmetic +// types, so the library has to take the signedness from std::numeric_limits, +// as it does for 128-bit and multiprecision integer classes such as +// absl::int128 or boost::multiprecision::cpp_int +class class_int +{ + public: + // trivial, like the 128-bit integer classes: the value is stored in a union + class_int() = default; + + // implicit from built-in integers and explicit from floats, like absl::int128; + // non-template overloads (as absl::int128 has) because clang 3.4 and 3.5 + // reject basic_json's defaulted data() for a number type with template + // constructors + class_int(int v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned long v) : value(static_cast(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(long long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned long long v) : value(static_cast(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + explicit class_int(double v) : value(static_cast(v)) {} + + template::value, int>::type = 0> + explicit operator T() const + { + return static_cast(value); + } + + friend bool operator==(class_int lhs, class_int rhs) + { + return lhs.value == rhs.value; + } + friend bool operator!=(class_int lhs, class_int rhs) + { + return lhs.value != rhs.value; + } + friend bool operator<(class_int lhs, class_int rhs) + { + return lhs.value < rhs.value; + } +#if JSON_HAS_THREE_WAY_COMPARISON + friend std::strong_ordering operator<=>(class_int lhs, class_int rhs) // *NOPAD* + { + return lhs.value <=> rhs.value; // *NOPAD* + } +#endif + + private: + std::int64_t value; +}; + +} // namespace custom_number_types + +using custom_number_types::class_int; + +namespace std +{ +// only the members the library uses +template<> +class numeric_limits +{ + public: + static constexpr bool is_signed = true; + static constexpr int digits = std::numeric_limits::digits; +}; +} // namespace std + +namespace +{ + +using class_int_json = nlohmann::basic_json; + +class_int_json make_class_int(std::int64_t v) +{ + class_int_json j(class_int_json::value_t::number_integer); + j.get_ref() = class_int(v); + return j; +} + +} // namespace + +// the serializer cannot print an integer of class type, so doctest must not +// try when an assertion fails +namespace doctest +{ +template<> +struct StringMaker +{ + static String convert(const class_int_json& j) + { + return j.type_name(); + } +}; +} // namespace doctest + +// __int128 as number type needs the std::numeric_limits (for the range checks) +// and std::is_integral (for the serializer) specializations, which libc++ +// always provides and libstdc++ only outside strict ISO modes; the MSVC +// standard library has none +#if defined(__SIZEOF_INT128__) && (defined(_LIBCPP_VERSION) || defined(__GLIBCXX_TYPE_INT_N_0)) + #define JSON_TEST_INT128_NUMBER_TYPES 1 +#else + #define JSON_TEST_INT128_NUMBER_TYPES 0 +#endif + +#if JSON_TEST_INT128_NUMBER_TYPES +namespace +{ + +// __extension__ keeps -Wpedantic from flagging the non-standard type +__extension__ typedef __int128 int128; // NOLINT(modernize-use-using) +__extension__ typedef unsigned __int128 uint128; // NOLINT(modernize-use-using) + +static_assert(std::numeric_limits::digits == 127 && std::numeric_limits::digits == 128 && + std::is_integral::value && std::is_integral::value, // NOLINT(misc-redundant-expression) + "__int128 is not fully supported by the standard library"); + +using wide_json = nlohmann::basic_json; + +wide_json make_int(int128 v) +{ + wide_json j(wide_json::value_t::number_integer); + j.get_ref() = v; + return j; +} + +wide_json make_uint(uint128 v) +{ + wide_json j(wide_json::value_t::number_unsigned); + j.get_ref() = v; + return j; +} + +wide_json wrap(const wide_json& value) +{ + wide_json o(wide_json::value_t::object); + o["v"] = value; + return o; +} + +} // namespace + +TEST_CASE("custom number types: integers beyond 64 bits") +{ + using out_of_range = wide_json::out_of_range; + + const int128 two_64 = static_cast(1) << 64; + const int128 two_100 = static_cast(1) << 100; + const int128 max_int64 = (std::numeric_limits::max)(); + const int128 min_int64 = (std::numeric_limits::min)(); + const uint128 max_uint64 = (std::numeric_limits::max)(); + + // before the range checks, these values were silently truncated, e.g., + // 2^100 was written as 0 + const wide_json int_big = make_int(two_100); + const wide_json int_big_negative = make_int(-two_100); + const wide_json uint_big = make_uint(static_cast(two_100)); + std::vector _; + + SECTION("CBOR") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit uint64", out_of_range&); + // the extremes of the 128-bit types, whose digits are produced without + // negating the minimum as a whole + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits::min)())), "[json.exception.out_of_range.407] integer number -170141183460469231731687303715884105728 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits::max)())), "[json.exception.out_of_range.407] integer number 170141183460469231731687303715884105727 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_uint((std::numeric_limits::max)())), "[json.exception.out_of_range.407] integer number 340282366920938463463374607431768211455 cannot be represented by CBOR as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(-two_64 - 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_uint(static_cast(max_uint64) + 1)), out_of_range&); + + // CBOR's integers cover [-2^64, 2^64-1] + CHECK(wide_json::to_cbor(make_int(two_64 - 1)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_int(-two_64)) == std::vector({0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_uint(max_uint64)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("MessagePack") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(min_int64 - 1)), out_of_range&); + + // MessagePack's integers cover [-2^63, 2^64-1] + CHECK(wide_json::to_msgpack(make_int(two_64 - 1)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_msgpack(make_int(min_int64)) == std::vector({0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); + CHECK(wide_json::to_msgpack(make_uint(max_uint64)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("BSON") + { + // wrap() stores the value at /v, which JSON_DIAGNOSTICS adds to the message +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] (/v) integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&); +#else + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&); +#endif + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(max_int64 + 1))), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(min_int64 - 1))), out_of_range&); + + // nested values are checked as well, before anything is written + wide_json nested(wide_json::value_t::object); + nested["a"] = wide_json::array({wrap(int_big)}); + std::vector out; + CHECK_THROWS_AS(wide_json::to_bson(nested, out), out_of_range&); + CHECK(out.empty()); + + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(max_int64)))) == wrap(make_int(max_int64))); + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(min_int64)))) == wrap(make_int(min_int64))); + CHECK_NOTHROW(wide_json::to_bson(wrap(make_uint(max_uint64)))); + } + + SECTION("BON8") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(max_int64 + 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(min_int64 - 1)), out_of_range&); + + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(max_int64))) == make_int(max_int64)); + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(min_int64))) == make_int(min_int64)); + } + + SECTION("UBJSON and BJData") + { + // integers beyond 64 bits are written exactly as high-precision numbers + const std::string digits = "1267650600228229401496703205376"; + std::vector expected = {'H', 'i', static_cast(digits.size())}; + expected.insert(expected.end(), digits.begin(), digits.end()); + CHECK(wide_json::to_ubjson(int_big) == expected); + CHECK(wide_json::to_ubjson(uint_big) == expected); + CHECK(wide_json::to_bjdata(int_big) == expected); + // BJData's uint64 marker 'M' was used for any unsigned value beyond + // int64, which truncated the ones beyond 64 bits + CHECK(wide_json::to_bjdata(uint_big) == expected); + + // the uint64 marker is still used where the value fits + CHECK(wide_json::to_bjdata(make_uint(max_uint64)) == std::vector({'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + + // the elements of an optimized container are written the same way; + // BJData does not allow 'H' as the type of an optimized container + std::vector expected_ubjson_array = {'[', '$', 'H', '#', 'i', 2}; + std::vector expected_bjdata_array = {'[', '#', 'i', 2}; + for (int i = 0; i < 2; ++i) + { + expected_ubjson_array.insert(expected_ubjson_array.end(), expected.begin() + 1, expected.end()); + expected_bjdata_array.insert(expected_bjdata_array.end(), expected.begin(), expected.end()); + } + CHECK(wide_json::to_ubjson(wide_json::array({uint_big, uint_big}), true, true) == expected_ubjson_array); + CHECK(wide_json::to_bjdata(wide_json::array({uint_big, uint_big}), true, true) == expected_bjdata_array); + } + + SECTION("BJData ND-array") + { + // an ND-array element beyond 64 bits does not fit any dtype, so the + // annotated object is written as a plain object instead of truncating + // the element into range + wide_json element(wide_json::value_t::object); + element["_ArrayType_"] = "uint8"; + element["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(1)}); + element["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(static_cast(two_64) + 1)}); + const auto element_bytes = wide_json::to_bjdata(element, true, true); + REQUIRE(!element_bytes.empty()); + CHECK(element_bytes[0] == '{'); + + wide_json signed_element = element; + signed_element["_ArrayType_"] = "int8"; + signed_element["_ArrayData_"] = wide_json::array({make_int(1), make_int(-two_64 + 1)}); + const auto signed_element_bytes = wide_json::to_bjdata(signed_element, true, true); + REQUIRE(!signed_element_bytes.empty()); + CHECK(signed_element_bytes[0] == '{'); + + // the same for a dimension beyond 64 bits, which wrapped into a + // dimension matching the size of _ArrayData_ (here: 1) + wide_json dimension = element; + dimension["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(static_cast(two_64) + 1)}); + dimension["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto dimension_bytes = wide_json::to_bjdata(dimension, true, true); + REQUIRE(!dimension_bytes.empty()); + CHECK(dimension_bytes[0] == '{'); + + // in range, the ND-array is still written + wide_json fits = element; + fits["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto fits_bytes = wide_json::to_bjdata(fits, true, true); + REQUIRE(!fits_bytes.empty()); + CHECK(fits_bytes[0] == '['); + } +} + +#endif + +TEST_CASE("custom number types") +{ + SECTION("signed integer of class type compared with a float") + { + // std::is_signed is false for a class type, which made the comparison + // treat any float below zero as less than every integer + const class_int_json minus_five = make_class_int(-5); + const class_int_json minus_two = make_class_int(-2); + const class_int_json five = make_class_int(5); + const class_int_json minus_two_and_a_half = -2.5; + const class_int_json two_and_a_half = 2.5; + const class_int_json huge_negative = -1e30; + const class_int_json huge_positive = 1e30; + + CHECK(minus_five < minus_two_and_a_half); + CHECK_FALSE(minus_two_and_a_half < minus_five); + CHECK(minus_two_and_a_half > minus_five); + CHECK(minus_five <= minus_two_and_a_half); + CHECK(minus_two_and_a_half >= minus_five); + CHECK(minus_five != minus_two_and_a_half); + + CHECK(minus_two_and_a_half < minus_two); + CHECK_FALSE(minus_two < minus_two_and_a_half); + + CHECK(two_and_a_half < five); + CHECK(minus_five < two_and_a_half); + + // floats beyond the integer's range + CHECK(huge_negative < minus_five); + CHECK_FALSE(minus_five < huge_negative); + CHECK(five < huge_positive); + CHECK_FALSE(huge_positive < five); + + // equality + const class_int_json minus_two_float = -2.0; + CHECK(minus_two == minus_two_float); + CHECK(minus_two_float == minus_two); + } + +} diff --git a/tests/src/unit-msgpack-custom-types.cpp b/tests/src/unit-msgpack-custom-types.cpp new file mode 100644 index 000000000..5c4ab6173 --- /dev/null +++ b/tests/src/unit-msgpack-custom-types.cpp @@ -0,0 +1,296 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// MessagePack tests with basic_json specializations other than nlohmann::json; +// they are kept apart from unit-msgpack.cpp because every specialization adds +// many sections to the object file, and MinGW cannot link objects with more +// than 65535 sections (see the clang job in .github/workflows/windows.yml) + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include // SIZE_MAX, UINT32_MAX +#include +#include +#include +#include +#include +#include +#include + +// the fake sizes below do not fit into a 32-bit std::size_t +// with clang and libstdc++ 10, the std::filesystem::path conversion that +// C++17 builds consider for every string type is ambiguous for a class +// derived from std::string, so the string case is not tested there +#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11) + #define JSON_TEST_BEYOND_UINT32_STRING 1 +#endif + +#if SIZE_MAX > UINT32_MAX +template> +struct huge_array : std::vector +{ + using base = std::vector; + using base::base; + + bool fake_size = false; + + std::size_t size() const noexcept + { + if (fake_size) + { + return (std::numeric_limits::max)() + 1ULL; + } + + return base::size(); + } +}; + +using huge_array_json = nlohmann::json::with_array_t; + +TEST_CASE("MessagePack Size above uint32 for array") +{ + huge_array_json j = huge_array_json::array(); + + j.push_back(1); + j.push_back(2); + j.push_back(3); + + auto& array = j.get_ref(); + array.fake_size = true; + + // write into a caller-owned vector: to_msgpack(j) reserves space based on + // the (faked) element count, which fails with bad_alloc on Windows + std::vector result; + CHECK_THROWS_WITH_AS( + huge_array_json::to_msgpack(j, result), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); + + array.fake_size = false; +} + +template, + typename A = std::allocator>> + struct huge_map : std::map +{ + using base = std::map; + using base::base; + + bool fake_size = false; + + std::size_t size() const noexcept + { + if (fake_size) + { + return static_cast(UINT32_MAX) + 1ULL; + } + + return base::size(); + } +}; + +using huge_object_json = nlohmann::json::with_object_t; + +TEST_CASE("MessagePack Size above uint32 for object") +{ + + huge_object_json j = huge_object_json::object(); + + j["one"] = 1; + j["two"] = 2; + + auto& object = j.get_ref(); + object.fake_size = true; + + // write into a caller-owned vector: to_msgpack(j) reserves space based on + // the (faked) element count, which fails with bad_alloc on Windows + std::vector result; + CHECK_THROWS_WITH_AS( + huge_object_json::to_msgpack(j, result), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); + + object.fake_size = false; +} + +#ifdef JSON_TEST_BEYOND_UINT32_STRING +struct huge_string : std::string +{ + using std::string::string; + + std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return static_cast(UINT32_MAX) + 1ULL; + } +}; + +using huge_string_json = nlohmann::json::with_string_t; + +TEST_CASE("MessagePack Size above uint32 for string") +{ + const huge_string_json j = "hello"; + + CHECK_THROWS_WITH_AS( + huge_string_json::to_msgpack(j), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); +} +#endif + +struct huge_binary : std::vector +{ + using std::vector::vector; + + std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return static_cast(UINT32_MAX) + 1ULL; + } +}; + +using huge_binary_json = nlohmann::json::with_binary_t; + +TEST_CASE("MessagePack Size above uint32 for binary") +{ + + huge_binary_json j = huge_binary_json::binary(huge_binary{}); + + j.get_binary().push_back(0x01); + j.get_binary().push_back(0x02); + + CHECK_THROWS_WITH_AS( + huge_binary_json::to_msgpack(j), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); +} +#endif + +namespace +{ +// types that report a size beyond UINT32_MAX without allocating that much +// memory, so the MessagePack length limit can be tested cheaply; see the +// similar types in unit-bson.cpp +std::size_t beyond_uint32_size() +{ + return static_cast((std::numeric_limits::max)()) + 1; +} + +class beyond_uint32_binary_t : public std::vector +{ + public: + using std::vector::vector; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +#ifdef JSON_TEST_BEYOND_UINT32_STRING +class beyond_uint32_string_t : public std::string +{ + public: + using std::string::string; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +using beyond_uint32_string_json = nlohmann::json::with_string_t; +#endif + +using beyond_uint32_binary_json = nlohmann::json::with_binary_t; +} // namespace + +TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized") +{ + // MessagePack stores the length of a string, binary value, array, or + // object in at most 32 bits; a larger one used to be written without any + // length at all +#if SIZE_MAX > UINT32_MAX + { + const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295"; + + const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&); + + const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&); + +#ifdef JSON_TEST_BEYOND_UINT32_STRING + // created from its type rather than from a beyond_uint32_string_t: + // that would consider the std::filesystem::path conversion, which + // libstdc++ 10 cannot decide for a class derived from std::string + const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string); + CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&); +#endif + } +#endif +} + +TEST_CASE("MessagePack numbers use the active union member (see #5644)") +{ + // when number_integer_t is narrower than number_unsigned_t, to_msgpack() + // used to read the union member that was not the active one, writing + // wrong bytes for some values; std::int64_t/std::uint64_t (the default + // types, where both members have the same width) were not affected + using int32_json = nlohmann::json::with_integers_t; + using int16_json = nlohmann::json::with_integers_t; + + SECTION("number_integer_t = std::int32_t") + { + SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)") + { + const int32_json j = 6442450944ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00}; + const auto result = int32_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int32_json::from_msgpack(result) == j); + } + + SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)") + { + const int32_json j = 4294967496ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; + const auto result = int32_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int32_json::from_msgpack(result) == j); + } + } + + SECTION("number_integer_t = std::int16_t, 98304 (uint 32)") + { + const int16_json j = 98304ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xce, 0x00, 0x01, 0x80, 0x00}; + const auto result = int16_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int16_json::from_msgpack(result) == j); + } + + SECTION("default types (std::int64_t/std::uint64_t) are unaffected") + { + const json j = 4294967496ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + CHECK(json::from_msgpack(result) == j); + } +} diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index aeb0116d3..2890a240c 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -2173,276 +2173,6 @@ TEST_CASE("MessagePack roundtrips" * doctest::skip()) } } -// the fake sizes below do not fit into a 32-bit std::size_t -// with clang and libstdc++ 10, the std::filesystem::path conversion that -// C++17 builds consider for every string type is ambiguous for a class -// derived from std::string, so the string case is not tested there -#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11) - #define JSON_TEST_BEYOND_UINT32_STRING 1 -#endif - -#if SIZE_MAX > UINT32_MAX -template> -struct huge_array : std::vector -{ - using base = std::vector; - using base::base; - - bool fake_size = false; - - std::size_t size() const noexcept - { - if (fake_size) - { - return (std::numeric_limits::max)() + 1ULL; - } - - return base::size(); - } -}; - -using huge_array_json = nlohmann::json::with_array_t; - -TEST_CASE("MessagePack Size above uint32 for array") -{ - huge_array_json j = huge_array_json::array(); - - j.push_back(1); - j.push_back(2); - j.push_back(3); - - auto& array = j.get_ref(); - array.fake_size = true; - - // write into a caller-owned vector: to_msgpack(j) reserves space based on - // the (faked) element count, which fails with bad_alloc on Windows - std::vector result; - CHECK_THROWS_WITH_AS( - huge_array_json::to_msgpack(j, result), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); - - array.fake_size = false; -} - -template, - typename A = std::allocator>> - struct huge_map : std::map -{ - using base = std::map; - using base::base; - - bool fake_size = false; - - std::size_t size() const noexcept - { - if (fake_size) - { - return static_cast(UINT32_MAX) + 1ULL; - } - - return base::size(); - } -}; - -using huge_object_json = nlohmann::json::with_object_t; - -TEST_CASE("MessagePack Size above uint32 for object") -{ - - huge_object_json j = huge_object_json::object(); - - j["one"] = 1; - j["two"] = 2; - - auto& object = j.get_ref(); - object.fake_size = true; - - // write into a caller-owned vector: to_msgpack(j) reserves space based on - // the (faked) element count, which fails with bad_alloc on Windows - std::vector result; - CHECK_THROWS_WITH_AS( - huge_object_json::to_msgpack(j, result), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); - - object.fake_size = false; -} - -#ifdef JSON_TEST_BEYOND_UINT32_STRING -struct huge_string : std::string -{ - using std::string::string; - - std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return static_cast(UINT32_MAX) + 1ULL; - } -}; - -using huge_string_json = nlohmann::json::with_string_t; - -TEST_CASE("MessagePack Size above uint32 for string") -{ - const huge_string_json j = "hello"; - - CHECK_THROWS_WITH_AS( - huge_string_json::to_msgpack(j), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); -} -#endif - -struct huge_binary : std::vector -{ - using std::vector::vector; - - std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return static_cast(UINT32_MAX) + 1ULL; - } -}; - -using huge_binary_json = nlohmann::json::with_binary_t; - -TEST_CASE("MessagePack Size above uint32 for binary") -{ - - huge_binary_json j = huge_binary_json::binary(huge_binary{}); - - j.get_binary().push_back(0x01); - j.get_binary().push_back(0x02); - - CHECK_THROWS_WITH_AS( - huge_binary_json::to_msgpack(j), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); -} -#endif - -namespace -{ -// types that report a size beyond UINT32_MAX without allocating that much -// memory, so the MessagePack length limit can be tested cheaply; see the -// similar types in unit-bson.cpp -std::size_t beyond_uint32_size() -{ - return static_cast((std::numeric_limits::max)()) + 1; -} - -class beyond_uint32_binary_t : public std::vector -{ - public: - using std::vector::vector; - - size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return beyond_uint32_size(); - } -}; - -#ifdef JSON_TEST_BEYOND_UINT32_STRING -class beyond_uint32_string_t : public std::string -{ - public: - using std::string::string; - - size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return beyond_uint32_size(); - } -}; - -using beyond_uint32_string_json = nlohmann::json::with_string_t; -#endif - -using beyond_uint32_binary_json = nlohmann::json::with_binary_t; -} // namespace - -TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized") -{ - // MessagePack stores the length of a string, binary value, array, or - // object in at most 32 bits; a larger one used to be written without any - // length at all -#if SIZE_MAX > UINT32_MAX - { - const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295"; - - const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}); - CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&); - - const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42); - CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&); - -#ifdef JSON_TEST_BEYOND_UINT32_STRING - // created from its type rather than from a beyond_uint32_string_t: - // that would consider the std::filesystem::path conversion, which - // libstdc++ 10 cannot decide for a class derived from std::string - const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string); - CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&); -#endif - } -#endif -} - -TEST_CASE("MessagePack numbers use the active union member (see #5644)") -{ - // when number_integer_t is narrower than number_unsigned_t, to_msgpack() - // used to read the union member that was not the active one, writing - // wrong bytes for some values; std::int64_t/std::uint64_t (the default - // types, where both members have the same width) were not affected - using int32_json = nlohmann::json::with_integers_t; - using int16_json = nlohmann::json::with_integers_t; - - SECTION("number_integer_t = std::int32_t") - { - SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)") - { - const int32_json j = 6442450944ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00}; - const auto result = int32_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int32_json::from_msgpack(result) == j); - } - - SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)") - { - const int32_json j = 4294967496ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; - const auto result = int32_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int32_json::from_msgpack(result) == j); - } - } - - SECTION("number_integer_t = std::int16_t, 98304 (uint 32)") - { - const int16_json j = 98304ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xce, 0x00, 0x01, 0x80, 0x00}; - const auto result = int16_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int16_json::from_msgpack(result) == j); - } - - SECTION("default types (std::int64_t/std::uint64_t) are unaffected") - { - const json j = 4294967496ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; - const auto result = json::to_msgpack(j); - CHECK(result == expected); - CHECK(json::from_msgpack(result) == j); - } -} - TEST_CASE("MessagePack large strings and binaries (chunked reader)") { // get_msgpack_string()/get_msgpack_binary() both read through get_binary(), diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index c44b1bc92..b31cfd054 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -43,7 +43,6 @@ using ordered_json = nlohmann::ordered_json; #endif #include -#include #include #include #include @@ -867,52 +866,6 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with } -TEST_CASE("regression test #5476 - array type without reserve()") -{ - // the capacity reserved for definite-length arrays must not require the - // array type to have a reserve() member function - using deque_json = nlohmann::json::with_array_t; - - SECTION("std::deque") - { - const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); - CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); - - // the binary formats pass a definite length to start_array() - CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); - CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); - CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j); - - // parse() instantiates the callback parser as well, which reserves too - const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept - { - return true; - }); - CHECK(with_callback == deque_json({1, 2, 3})); - } - - SECTION("std::vector still reserves") - { - json array = json::array(); - for (int i = 0; i < 100; ++i) - { - array.push_back(i); - } - - const auto j = json::from_cbor(json::to_cbor(array)); - CHECK(j == array); - CHECK(j.get_ref().capacity() >= 100); - } - - SECTION("the reservation stays capped") - { - // CBOR array announcing 2^32-1 elements, but truncated right after the - // header: the input must be rejected without reserving that capacity - const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; - CHECK(json::from_cbor(truncated, true, false).is_discarded()); - } -} - TEST_CASE("issue #5317 - nested indefinite-length CBOR string chunks are rejected") { json _;