From 419e1678ee105210daf62bada32ebe91d4ed3733 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 16 Oct 2026 19:26:47 +0200 Subject: [PATCH] Fix silent wrong results for custom number types Squashed onto develop cd6b428ef (CI fixes from #5860). Original commits: - Fix silent wrong results for custom number types - Fix CI: build the out_of_range.407 message from string_t and fix a link - Fix CI: diagnostics message, clang-tidy, and MinGW section limit - Format the out_of_range.407 number without dump() - Move the std::deque array test out of unit-regression3.cpp - Fix clang-tidy and clang 3.4/3.5 failures in the custom number type changes - Fix CI: cast the 128-bit unsigned value explicitly in the integer fast path Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/to_bjdata.md | 4 +- docs/mkdocs/docs/api/basic_json/to_bon8.md | 5 +- docs/mkdocs/docs/api/basic_json/to_bson.md | 6 + docs/mkdocs/docs/api/basic_json/to_cbor.md | 5 + docs/mkdocs/docs/api/basic_json/to_msgpack.md | 5 + docs/mkdocs/docs/home/exceptions.md | 24 +- .../nlohmann/detail/output/binary_writer.hpp | 193 ++++++++- include/nlohmann/detail/output/serializer.hpp | 2 +- include/nlohmann/detail/value_t.hpp | 10 +- single_include/nlohmann/json.hpp | 205 +++++++++- tests/src/unit-custom-array-type.cpp | 46 +++ tests/src/unit-custom-number-types.cpp | 367 ++++++++++++++++++ tests/src/unit-msgpack-custom-types.cpp | 296 ++++++++++++++ tests/src/unit-msgpack.cpp | 270 ------------- tests/src/unit-regression3.cpp | 47 --- 15 files changed, 1127 insertions(+), 358 deletions(-) create mode 100644 tests/src/unit-custom-number-types.cpp create mode 100644 tests/src/unit-msgpack-custom-types.cpp diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 583139068..aa0240a73 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -123,4 +123,6 @@ Linear in the size of the JSON value `j`. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an - array or object was silently skipped, producing invalid BJData. \ No newline at end of file + array or object was silently skipped, producing invalid BJData. +- Writes unsigned integers wider than 64 bits as high-precision numbers since version 3.13.0; previously, they were + silently truncated to 64 bits. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bon8.md b/docs/mkdocs/docs/api/basic_json/to_bon8.md index 14cc085f6..debbb6ef0 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bon8.md +++ b/docs/mkdocs/docs/api/basic_json/to_bon8.md @@ -37,8 +37,9 @@ With (2), the bytes written before the exception remain in the output adapter. ## Exceptions -- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an unsigned integer - above 9223372036854775807, which BON8 cannot represent +- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer outside + the range of int64 (an unsigned integer above 9223372036854775807, or, with a number type wider than 64 bits, any + integer beyond int64), which BON8 cannot represent - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if `j` contains a string that is not valid UTF-8 - Throws [type_error.321](../../home/exceptions.md#jsonexceptiontype_error321) if `j` or a value nested in it is diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index 05f8b8548..315477e19 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -47,6 +47,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`type_error.317`](../../home/exceptions.md#jsonexceptiontype_error317) if the top-level type of the JSON value is not an object; example: `"to serialize to BSON, top-level type must be object, but is string"` +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains a signed integer + outside the range of int64 or an unsigned integer outside the range of uint64, which is only possible with a number + type wider than 64 bits; example: + `"integer number 9223372036854775808 cannot be represented by BSON as it does not fit int64"` - Throws [`out_of_range.409`](../../home/exceptions.md#jsonexceptionout_of_range409) if a key in the JSON object contains a null byte (code point U+0000); example: `"BSON key cannot contain code point U+0000 (at byte 2)"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a document, array, @@ -119,3 +123,5 @@ pass before anything is written. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316` before anything is written. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_cbor.md b/docs/mkdocs/docs/api/basic_json/to_cbor.md index f5b6a07bb..6508864e1 100644 --- a/docs/mkdocs/docs/api/basic_json/to_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/to_cbor.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^64, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]"` - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not valid UTF-8 and `error_handler` is `strict` (the default only if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) @@ -90,3 +93,5 @@ Linear in the size of the JSON value `j`. [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid CBOR. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_msgpack.md b/docs/mkdocs/docs/api/basic_json/to_msgpack.md index 2e32208bf..dca66271d 100644 --- a/docs/mkdocs/docs/api/basic_json/to_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/to_msgpack.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^63, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a string, binary value, array, or object exceeds 4294967295, the maximum MessagePack can store; example: `"MessagePack length 4294967296 exceeds maximum of 4294967295"` @@ -112,3 +115,5 @@ Linear in the size of the JSON value `j`. `number_unsigned_t`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid MessagePack. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index a7e935538..f4245b660 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -916,9 +916,19 @@ double-precision number when `number_float_t` is `#!cpp float`. ### json.exception.out_of_range.407 -An unsigned integer number greater than 9223372036854775807 cannot be represented in the binary format. This exception -is only thrown by [`to_bon8`](../api/basic_json/to_bon8.md), as [BON8](../features/binary_formats/bon8.md) has no -unsigned 64-bit integer type. +An integer number cannot be represented by the binary format it is serialized to: + +- [BON8](../features/binary_formats/bon8.md) only stores integers that fit into int64, so + [`to_bon8`](../api/basic_json/to_bon8.md) throws for unsigned integers greater than 9223372036854775807. +- [CBOR](../features/binary_formats/cbor.md), [MessagePack](../features/binary_formats/messagepack.md), and + [BSON](../features/binary_formats/bson.md) store integers in at most 64 bits. With the default number types, every + integer fits, but a [`number_integer_t`](../api/basic_json/number_integer_t.md) or + [`number_unsigned_t`](../api/basic_json/number_unsigned_t.md) wider than 64 bits (e.g., `__int128`) can hold values + outside the range of the format: [-2^64, 2^64-1] for CBOR, [-2^63, 2^64-1] for MessagePack, and the range of int64 + (signed integers) or uint64 (unsigned integers) for BSON. + +[UBJSON](../features/binary_formats/ubjson.md) and [BJData](../features/binary_formats/bjdata.md) never throw this +exception, because they serialize integers beyond 64 bits as high-precision numbers. !!! note "Former occurrences" @@ -927,11 +937,17 @@ unsigned 64-bit integer type. - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers. - Since version 3.12.0, integer numbers beyond int64 are serialized as uint64 BSON numbers. -!!! failure "Example message" + Before version 3.13.0, CBOR, MessagePack, and BSON silently truncated integers wider than 64 bits, and BJData + truncated unsigned integers wider than 64 bits. + +!!! failure "Example messages" ``` integer number 18446744073709551615 cannot be represented by BON8 as it does not fit int64 ``` + ``` + integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1] + ``` ### json.exception.out_of_range.408 diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 3a9abbc08..401a6ac50 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -207,6 +207,10 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } // CBOR does not differentiate between positive signed // integers and unsigned integers write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); @@ -214,6 +218,10 @@ class binary_writer else { // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); } break; @@ -221,6 +229,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -423,12 +435,20 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -467,6 +487,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -824,6 +848,133 @@ class binary_writer JSON_THROW(type_error::create(exception_id::discarded_value_used, concat("cannot serialize discarded value to ", format_name), &j)); } + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant < bool, + (std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant < bool, std::numeric_limits::is_signed && + (!std::numeric_limits::is_signed || std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief the decimal digits of the non-negative integer @a n + */ + template + static std::string integer_to_string(NumberType n, std::false_type /*is_signed*/) + { + std::string result; + do + { + result.push_back(static_cast('0' + (n % 10))); + n = static_cast(n / 10); + } + while (n != 0); + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief the decimal digits of the integer @a n, with a leading '-' if it + is negative + */ + template + static std::string integer_to_string(NumberType n, std::true_type /*is_signed*/) + { + // the remainder of a negative n is not positive; each digit is + // negated on its own, because negating the minimum of NumberType + // as a whole would overflow + const bool negative = n < 0; + std::string result; + do + { + const auto remainder = n % 10; + result.push_back(static_cast('0' + (remainder < 0 ? -remainder : remainder))); + n = static_cast(n / 10); + } + while (n != 0); + if (negative) + { + result.push_back('-'); + } + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // neither std::to_string(), which has no overload for integer types + // wider than 64 bits, nor dump(), which would also instantiate the + // serializer for objects and fail for key types it cannot write + using integer_limits = std::numeric_limits; + using unsigned_limits = std::numeric_limits; + const std::string number = j.is_number_unsigned() + ? integer_to_string(j.m_data.m_value.number_unsigned, std::integral_constant()) + : integer_to_string(j.m_data.m_value.number_integer, std::integral_constant()); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -1580,6 +1731,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -1598,9 +1751,17 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: @@ -1639,8 +1800,9 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); @@ -2147,7 +2309,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -2272,14 +2434,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -2511,10 +2675,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -2646,9 +2811,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -2657,6 +2822,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 3621a78d9..1dd5e41e3 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -1525,7 +1525,7 @@ class serializer // without leading zeros if (abs_value < 10000000000000000u) { - const std::uint64_t value = abs_value; + const auto value = static_cast(abs_value); // fits: abs_value < 10^16 const std::uint64_t upper = value / 100000000u; const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) diff --git a/include/nlohmann/detail/value_t.hpp b/include/nlohmann/detail/value_t.hpp index cda0facd2..0c0c8172e 100644 --- a/include/nlohmann/detail/value_t.hpp +++ b/include/nlohmann/detail/value_t.hpp @@ -14,7 +14,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed #include #if JSON_HAS_THREE_WAY_COMPARISON @@ -175,13 +174,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4debe551f..2b7eedc9b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -290,7 +290,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed // #include // __ _____ _____ _____ @@ -3397,13 +3396,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } @@ -23153,6 +23155,10 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } // CBOR does not differentiate between positive signed // integers and unsigned integers write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); @@ -23160,6 +23166,10 @@ class binary_writer else { // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); } break; @@ -23167,6 +23177,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } write_cbor_head(0x00, conditional_static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -23369,12 +23383,20 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -23413,6 +23435,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -23770,6 +23796,133 @@ class binary_writer JSON_THROW(type_error::create(exception_id::discarded_value_used, concat("cannot serialize discarded value to ", format_name), &j)); } + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant < bool, + (std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant < bool, std::numeric_limits::is_signed && + (!std::numeric_limits::is_signed || std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief the decimal digits of the non-negative integer @a n + */ + template + static std::string integer_to_string(NumberType n, std::false_type /*is_signed*/) + { + std::string result; + do + { + result.push_back(static_cast('0' + (n % 10))); + n = static_cast(n / 10); + } + while (n != 0); + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief the decimal digits of the integer @a n, with a leading '-' if it + is negative + */ + template + static std::string integer_to_string(NumberType n, std::true_type /*is_signed*/) + { + // the remainder of a negative n is not positive; each digit is + // negated on its own, because negating the minimum of NumberType + // as a whole would overflow + const bool negative = n < 0; + std::string result; + do + { + const auto remainder = n % 10; + result.push_back(static_cast('0' + (remainder < 0 ? -remainder : remainder))); + n = static_cast(n / 10); + } + while (n != 0); + if (negative) + { + result.push_back('-'); + } + std::reverse(result.begin(), result.end()); + return result; + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // neither std::to_string(), which has no overload for integer types + // wider than 64 bits, nor dump(), which would also instantiate the + // serializer for objects and fail for key types it cannot write + using integer_limits = std::numeric_limits; + using unsigned_limits = std::numeric_limits; + const std::string number = j.is_number_unsigned() + ? integer_to_string(j.m_data.m_value.number_unsigned, std::integral_constant()) + : integer_to_string(j.m_data.m_value.number_integer, std::integral_constant()); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", number, " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -24526,6 +24679,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -24544,9 +24699,17 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } return calc_bson_unsigned_size(conditional_static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: @@ -24585,8 +24748,9 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: return write_bson_unsigned(name, conditional_static_cast(j.m_data.m_value.number_unsigned)); @@ -25093,7 +25257,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -25218,14 +25382,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -25457,10 +25623,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -25592,9 +25759,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(exception_id::integer_too_large, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -25603,6 +25770,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; @@ -29487,7 +29658,7 @@ class serializer // without leading zeros if (abs_value < 10000000000000000u) { - const std::uint64_t value = abs_value; + const auto value = static_cast(abs_value); // fits: abs_value < 10^16 const std::uint64_t upper = value / 100000000u; const std::uint64_t first = dtoa_impl::eight_digit_bytes(upper != 0 ? upper : value); const auto leading = static_cast(count_leading_zeros(first) / 8); // (first is not 0) diff --git a/tests/src/unit-custom-array-type.cpp b/tests/src/unit-custom-array-type.cpp index 6f015d234..e602b4716 100644 --- a/tests/src/unit-custom-array-type.cpp +++ b/tests/src/unit-custom-array-type.cpp @@ -9,7 +9,9 @@ #include "doctest_compatibility.h" #include +using nlohmann::json; +#include #include #include #include @@ -164,3 +166,47 @@ TEST_CASE("array type without at()") CHECK(j.at(no_at_json::json_pointer("/1")) == 2); CHECK_THROWS_AS(j.at(no_at_json::json_pointer("/3")), no_at_json::out_of_range); } + +TEST_CASE("regression test #5476 - array type without reserve()") +{ + // the capacity reserved for definite-length arrays must not require the + // array type to have a reserve() member function + SECTION("std::deque") + { + const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); + CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); + + // the binary formats pass a definite length to start_array() + CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); + CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); + CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j); + + // parse() instantiates the callback parser as well, which reserves too + const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept + { + return true; + }); + CHECK(with_callback == deque_json({1, 2, 3})); + } + + SECTION("std::vector still reserves") + { + json array = json::array(); + for (int i = 0; i < 100; ++i) + { + array.push_back(i); + } + + const auto j = json::from_cbor(json::to_cbor(array)); + CHECK(j == array); + CHECK(j.get_ref().capacity() >= 100); + } + + SECTION("the reservation stays capped") + { + // CBOR array announcing 2^32-1 elements, but truncated right after the + // header: the input must be rejected without reserving that capacity + const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_cbor(truncated, true, false).is_discarded()); + } +} diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp new file mode 100644 index 000000000..f9c197574 --- /dev/null +++ b/tests/src/unit-custom-number-types.cpp @@ -0,0 +1,367 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if JSON_HAS_THREE_WAY_COMPARISON + #include +#endif + +namespace custom_number_types +{ + +// a signed integer of class type: std::is_signed is only true for arithmetic +// types, so the library has to take the signedness from std::numeric_limits, +// as it does for 128-bit and multiprecision integer classes such as +// absl::int128 or boost::multiprecision::cpp_int +class class_int +{ + public: + // trivial, like the 128-bit integer classes: the value is stored in a union + class_int() = default; + + // implicit from built-in integers and explicit from floats, like absl::int128; + // non-template overloads (as absl::int128 has) because clang 3.4 and 3.5 + // reject basic_json's defaulted data() for a number type with template + // constructors + class_int(int v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned long v) : value(static_cast(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(long long v) : value(v) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + class_int(unsigned long long v) : value(static_cast(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + explicit class_int(double v) : value(static_cast(v)) {} + + template::value, int>::type = 0> + explicit operator T() const + { + return static_cast(value); + } + + friend bool operator==(class_int lhs, class_int rhs) + { + return lhs.value == rhs.value; + } + friend bool operator!=(class_int lhs, class_int rhs) + { + return lhs.value != rhs.value; + } + friend bool operator<(class_int lhs, class_int rhs) + { + return lhs.value < rhs.value; + } +#if JSON_HAS_THREE_WAY_COMPARISON + friend std::strong_ordering operator<=>(class_int lhs, class_int rhs) // *NOPAD* + { + return lhs.value <=> rhs.value; // *NOPAD* + } +#endif + + private: + std::int64_t value; +}; + +} // namespace custom_number_types + +using custom_number_types::class_int; + +namespace std +{ +// only the members the library uses +template<> +class numeric_limits +{ + public: + static constexpr bool is_signed = true; + static constexpr int digits = std::numeric_limits::digits; +}; +} // namespace std + +namespace +{ + +using class_int_json = nlohmann::basic_json; + +class_int_json make_class_int(std::int64_t v) +{ + class_int_json j(class_int_json::value_t::number_integer); + j.get_ref() = class_int(v); + return j; +} + +} // namespace + +// the serializer cannot print an integer of class type, so doctest must not +// try when an assertion fails +namespace doctest +{ +template<> +struct StringMaker +{ + static String convert(const class_int_json& j) + { + return j.type_name(); + } +}; +} // namespace doctest + +// __int128 as number type needs the std::numeric_limits (for the range checks) +// and std::is_integral (for the serializer) specializations, which libc++ +// always provides and libstdc++ only outside strict ISO modes; the MSVC +// standard library has none +#if defined(__SIZEOF_INT128__) && (defined(_LIBCPP_VERSION) || defined(__GLIBCXX_TYPE_INT_N_0)) + #define JSON_TEST_INT128_NUMBER_TYPES 1 +#else + #define JSON_TEST_INT128_NUMBER_TYPES 0 +#endif + +#if JSON_TEST_INT128_NUMBER_TYPES +namespace +{ + +// __extension__ keeps -Wpedantic from flagging the non-standard type +__extension__ typedef __int128 int128; // NOLINT(modernize-use-using) +__extension__ typedef unsigned __int128 uint128; // NOLINT(modernize-use-using) + +static_assert(std::numeric_limits::digits == 127 && std::numeric_limits::digits == 128 && + std::is_integral::value && std::is_integral::value, // NOLINT(misc-redundant-expression) + "__int128 is not fully supported by the standard library"); + +using wide_json = nlohmann::basic_json; + +wide_json make_int(int128 v) +{ + wide_json j(wide_json::value_t::number_integer); + j.get_ref() = v; + return j; +} + +wide_json make_uint(uint128 v) +{ + wide_json j(wide_json::value_t::number_unsigned); + j.get_ref() = v; + return j; +} + +wide_json wrap(const wide_json& value) +{ + wide_json o(wide_json::value_t::object); + o["v"] = value; + return o; +} + +} // namespace + +TEST_CASE("custom number types: integers beyond 64 bits") +{ + using out_of_range = wide_json::out_of_range; + + const int128 two_64 = static_cast(1) << 64; + const int128 two_100 = static_cast(1) << 100; + const int128 max_int64 = (std::numeric_limits::max)(); + const int128 min_int64 = (std::numeric_limits::min)(); + const uint128 max_uint64 = (std::numeric_limits::max)(); + + // before the range checks, these values were silently truncated, e.g., + // 2^100 was written as 0 + const wide_json int_big = make_int(two_100); + const wide_json int_big_negative = make_int(-two_100); + const wide_json uint_big = make_uint(static_cast(two_100)); + std::vector _; + + SECTION("CBOR") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit uint64", out_of_range&); + // the extremes of the 128-bit types, whose digits are produced without + // negating the minimum as a whole + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits::min)())), "[json.exception.out_of_range.407] integer number -170141183460469231731687303715884105728 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_int((std::numeric_limits::max)())), "[json.exception.out_of_range.407] integer number 170141183460469231731687303715884105727 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(make_uint((std::numeric_limits::max)())), "[json.exception.out_of_range.407] integer number 340282366920938463463374607431768211455 cannot be represented by CBOR as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(-two_64 - 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_uint(static_cast(max_uint64) + 1)), out_of_range&); + + // CBOR's integers cover [-2^64, 2^64-1] + CHECK(wide_json::to_cbor(make_int(two_64 - 1)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_int(-two_64)) == std::vector({0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_uint(max_uint64)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("MessagePack") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(min_int64 - 1)), out_of_range&); + + // MessagePack's integers cover [-2^63, 2^64-1] + CHECK(wide_json::to_msgpack(make_int(two_64 - 1)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_msgpack(make_int(min_int64)) == std::vector({0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); + CHECK(wide_json::to_msgpack(make_uint(max_uint64)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("BSON") + { + // wrap() stores the value at /v, which JSON_DIAGNOSTICS adds to the message +#if JSON_DIAGNOSTICS + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] (/v) integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] (/v) integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&); +#else + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&); +#endif + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(max_int64 + 1))), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(min_int64 - 1))), out_of_range&); + + // nested values are checked as well, before anything is written + wide_json nested(wide_json::value_t::object); + nested["a"] = wide_json::array({wrap(int_big)}); + std::vector out; + CHECK_THROWS_AS(wide_json::to_bson(nested, out), out_of_range&); + CHECK(out.empty()); + + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(max_int64)))) == wrap(make_int(max_int64))); + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(min_int64)))) == wrap(make_int(min_int64))); + CHECK_NOTHROW(wide_json::to_bson(wrap(make_uint(max_uint64)))); + } + + SECTION("BON8") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(max_int64 + 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(min_int64 - 1)), out_of_range&); + + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(max_int64))) == make_int(max_int64)); + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(min_int64))) == make_int(min_int64)); + } + + SECTION("UBJSON and BJData") + { + // integers beyond 64 bits are written exactly as high-precision numbers + const std::string digits = "1267650600228229401496703205376"; + std::vector expected = {'H', 'i', static_cast(digits.size())}; + expected.insert(expected.end(), digits.begin(), digits.end()); + CHECK(wide_json::to_ubjson(int_big) == expected); + CHECK(wide_json::to_ubjson(uint_big) == expected); + CHECK(wide_json::to_bjdata(int_big) == expected); + // BJData's uint64 marker 'M' was used for any unsigned value beyond + // int64, which truncated the ones beyond 64 bits + CHECK(wide_json::to_bjdata(uint_big) == expected); + + // the uint64 marker is still used where the value fits + CHECK(wide_json::to_bjdata(make_uint(max_uint64)) == std::vector({'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + + // the elements of an optimized container are written the same way; + // BJData does not allow 'H' as the type of an optimized container + std::vector expected_ubjson_array = {'[', '$', 'H', '#', 'i', 2}; + std::vector expected_bjdata_array = {'[', '#', 'i', 2}; + for (int i = 0; i < 2; ++i) + { + expected_ubjson_array.insert(expected_ubjson_array.end(), expected.begin() + 1, expected.end()); + expected_bjdata_array.insert(expected_bjdata_array.end(), expected.begin(), expected.end()); + } + CHECK(wide_json::to_ubjson(wide_json::array({uint_big, uint_big}), true, true) == expected_ubjson_array); + CHECK(wide_json::to_bjdata(wide_json::array({uint_big, uint_big}), true, true) == expected_bjdata_array); + } + + SECTION("BJData ND-array") + { + // an ND-array element beyond 64 bits does not fit any dtype, so the + // annotated object is written as a plain object instead of truncating + // the element into range + wide_json element(wide_json::value_t::object); + element["_ArrayType_"] = "uint8"; + element["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(1)}); + element["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(static_cast(two_64) + 1)}); + const auto element_bytes = wide_json::to_bjdata(element, true, true); + REQUIRE(!element_bytes.empty()); + CHECK(element_bytes[0] == '{'); + + wide_json signed_element = element; + signed_element["_ArrayType_"] = "int8"; + signed_element["_ArrayData_"] = wide_json::array({make_int(1), make_int(-two_64 + 1)}); + const auto signed_element_bytes = wide_json::to_bjdata(signed_element, true, true); + REQUIRE(!signed_element_bytes.empty()); + CHECK(signed_element_bytes[0] == '{'); + + // the same for a dimension beyond 64 bits, which wrapped into a + // dimension matching the size of _ArrayData_ (here: 1) + wide_json dimension = element; + dimension["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(static_cast(two_64) + 1)}); + dimension["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto dimension_bytes = wide_json::to_bjdata(dimension, true, true); + REQUIRE(!dimension_bytes.empty()); + CHECK(dimension_bytes[0] == '{'); + + // in range, the ND-array is still written + wide_json fits = element; + fits["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto fits_bytes = wide_json::to_bjdata(fits, true, true); + REQUIRE(!fits_bytes.empty()); + CHECK(fits_bytes[0] == '['); + } +} + +#endif + +TEST_CASE("custom number types") +{ + SECTION("signed integer of class type compared with a float") + { + // std::is_signed is false for a class type, which made the comparison + // treat any float below zero as less than every integer + const class_int_json minus_five = make_class_int(-5); + const class_int_json minus_two = make_class_int(-2); + const class_int_json five = make_class_int(5); + const class_int_json minus_two_and_a_half = -2.5; + const class_int_json two_and_a_half = 2.5; + const class_int_json huge_negative = -1e30; + const class_int_json huge_positive = 1e30; + + CHECK(minus_five < minus_two_and_a_half); + CHECK_FALSE(minus_two_and_a_half < minus_five); + CHECK(minus_two_and_a_half > minus_five); + CHECK(minus_five <= minus_two_and_a_half); + CHECK(minus_two_and_a_half >= minus_five); + CHECK(minus_five != minus_two_and_a_half); + + CHECK(minus_two_and_a_half < minus_two); + CHECK_FALSE(minus_two < minus_two_and_a_half); + + CHECK(two_and_a_half < five); + CHECK(minus_five < two_and_a_half); + + // floats beyond the integer's range + CHECK(huge_negative < minus_five); + CHECK_FALSE(minus_five < huge_negative); + CHECK(five < huge_positive); + CHECK_FALSE(huge_positive < five); + + // equality + const class_int_json minus_two_float = -2.0; + CHECK(minus_two == minus_two_float); + CHECK(minus_two_float == minus_two); + } + +} diff --git a/tests/src/unit-msgpack-custom-types.cpp b/tests/src/unit-msgpack-custom-types.cpp new file mode 100644 index 000000000..5c4ab6173 --- /dev/null +++ b/tests/src/unit-msgpack-custom-types.cpp @@ -0,0 +1,296 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +// MessagePack tests with basic_json specializations other than nlohmann::json; +// they are kept apart from unit-msgpack.cpp because every specialization adds +// many sections to the object file, and MinGW cannot link objects with more +// than 65535 sections (see the clang job in .github/workflows/windows.yml) + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include // SIZE_MAX, UINT32_MAX +#include +#include +#include +#include +#include +#include +#include + +// the fake sizes below do not fit into a 32-bit std::size_t +// with clang and libstdc++ 10, the std::filesystem::path conversion that +// C++17 builds consider for every string type is ambiguous for a class +// derived from std::string, so the string case is not tested there +#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11) + #define JSON_TEST_BEYOND_UINT32_STRING 1 +#endif + +#if SIZE_MAX > UINT32_MAX +template> +struct huge_array : std::vector +{ + using base = std::vector; + using base::base; + + bool fake_size = false; + + std::size_t size() const noexcept + { + if (fake_size) + { + return (std::numeric_limits::max)() + 1ULL; + } + + return base::size(); + } +}; + +using huge_array_json = nlohmann::json::with_array_t; + +TEST_CASE("MessagePack Size above uint32 for array") +{ + huge_array_json j = huge_array_json::array(); + + j.push_back(1); + j.push_back(2); + j.push_back(3); + + auto& array = j.get_ref(); + array.fake_size = true; + + // write into a caller-owned vector: to_msgpack(j) reserves space based on + // the (faked) element count, which fails with bad_alloc on Windows + std::vector result; + CHECK_THROWS_WITH_AS( + huge_array_json::to_msgpack(j, result), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); + + array.fake_size = false; +} + +template, + typename A = std::allocator>> + struct huge_map : std::map +{ + using base = std::map; + using base::base; + + bool fake_size = false; + + std::size_t size() const noexcept + { + if (fake_size) + { + return static_cast(UINT32_MAX) + 1ULL; + } + + return base::size(); + } +}; + +using huge_object_json = nlohmann::json::with_object_t; + +TEST_CASE("MessagePack Size above uint32 for object") +{ + + huge_object_json j = huge_object_json::object(); + + j["one"] = 1; + j["two"] = 2; + + auto& object = j.get_ref(); + object.fake_size = true; + + // write into a caller-owned vector: to_msgpack(j) reserves space based on + // the (faked) element count, which fails with bad_alloc on Windows + std::vector result; + CHECK_THROWS_WITH_AS( + huge_object_json::to_msgpack(j, result), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); + + object.fake_size = false; +} + +#ifdef JSON_TEST_BEYOND_UINT32_STRING +struct huge_string : std::string +{ + using std::string::string; + + std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return static_cast(UINT32_MAX) + 1ULL; + } +}; + +using huge_string_json = nlohmann::json::with_string_t; + +TEST_CASE("MessagePack Size above uint32 for string") +{ + const huge_string_json j = "hello"; + + CHECK_THROWS_WITH_AS( + huge_string_json::to_msgpack(j), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); +} +#endif + +struct huge_binary : std::vector +{ + using std::vector::vector; + + std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return static_cast(UINT32_MAX) + 1ULL; + } +}; + +using huge_binary_json = nlohmann::json::with_binary_t; + +TEST_CASE("MessagePack Size above uint32 for binary") +{ + + huge_binary_json j = huge_binary_json::binary(huge_binary{}); + + j.get_binary().push_back(0x01); + j.get_binary().push_back(0x02); + + CHECK_THROWS_WITH_AS( + huge_binary_json::to_msgpack(j), + "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", + json::out_of_range&); +} +#endif + +namespace +{ +// types that report a size beyond UINT32_MAX without allocating that much +// memory, so the MessagePack length limit can be tested cheaply; see the +// similar types in unit-bson.cpp +std::size_t beyond_uint32_size() +{ + return static_cast((std::numeric_limits::max)()) + 1; +} + +class beyond_uint32_binary_t : public std::vector +{ + public: + using std::vector::vector; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +#ifdef JSON_TEST_BEYOND_UINT32_STRING +class beyond_uint32_string_t : public std::string +{ + public: + using std::string::string; + + size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) + { + return beyond_uint32_size(); + } +}; + +using beyond_uint32_string_json = nlohmann::json::with_string_t; +#endif + +using beyond_uint32_binary_json = nlohmann::json::with_binary_t; +} // namespace + +TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized") +{ + // MessagePack stores the length of a string, binary value, array, or + // object in at most 32 bits; a larger one used to be written without any + // length at all +#if SIZE_MAX > UINT32_MAX + { + const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295"; + + const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&); + + const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42); + CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&); + +#ifdef JSON_TEST_BEYOND_UINT32_STRING + // created from its type rather than from a beyond_uint32_string_t: + // that would consider the std::filesystem::path conversion, which + // libstdc++ 10 cannot decide for a class derived from std::string + const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string); + CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&); +#endif + } +#endif +} + +TEST_CASE("MessagePack numbers use the active union member (see #5644)") +{ + // when number_integer_t is narrower than number_unsigned_t, to_msgpack() + // used to read the union member that was not the active one, writing + // wrong bytes for some values; std::int64_t/std::uint64_t (the default + // types, where both members have the same width) were not affected + using int32_json = nlohmann::json::with_integers_t; + using int16_json = nlohmann::json::with_integers_t; + + SECTION("number_integer_t = std::int32_t") + { + SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)") + { + const int32_json j = 6442450944ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00}; + const auto result = int32_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int32_json::from_msgpack(result) == j); + } + + SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)") + { + const int32_json j = 4294967496ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; + const auto result = int32_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int32_json::from_msgpack(result) == j); + } + } + + SECTION("number_integer_t = std::int16_t, 98304 (uint 32)") + { + const int16_json j = 98304ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xce, 0x00, 0x01, 0x80, 0x00}; + const auto result = int16_json::to_msgpack(j); + CHECK(result == expected); + CHECK(int16_json::from_msgpack(result) == j); + } + + SECTION("default types (std::int64_t/std::uint64_t) are unaffected") + { + const json j = 4294967496ULL; + CHECK(j.is_number_unsigned()); + + std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + CHECK(json::from_msgpack(result) == j); + } +} diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index aeb0116d3..2890a240c 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -2173,276 +2173,6 @@ TEST_CASE("MessagePack roundtrips" * doctest::skip()) } } -// the fake sizes below do not fit into a 32-bit std::size_t -// with clang and libstdc++ 10, the std::filesystem::path conversion that -// C++17 builds consider for every string type is ambiguous for a class -// derived from std::string, so the string case is not tested there -#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11) - #define JSON_TEST_BEYOND_UINT32_STRING 1 -#endif - -#if SIZE_MAX > UINT32_MAX -template> -struct huge_array : std::vector -{ - using base = std::vector; - using base::base; - - bool fake_size = false; - - std::size_t size() const noexcept - { - if (fake_size) - { - return (std::numeric_limits::max)() + 1ULL; - } - - return base::size(); - } -}; - -using huge_array_json = nlohmann::json::with_array_t; - -TEST_CASE("MessagePack Size above uint32 for array") -{ - huge_array_json j = huge_array_json::array(); - - j.push_back(1); - j.push_back(2); - j.push_back(3); - - auto& array = j.get_ref(); - array.fake_size = true; - - // write into a caller-owned vector: to_msgpack(j) reserves space based on - // the (faked) element count, which fails with bad_alloc on Windows - std::vector result; - CHECK_THROWS_WITH_AS( - huge_array_json::to_msgpack(j, result), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); - - array.fake_size = false; -} - -template, - typename A = std::allocator>> - struct huge_map : std::map -{ - using base = std::map; - using base::base; - - bool fake_size = false; - - std::size_t size() const noexcept - { - if (fake_size) - { - return static_cast(UINT32_MAX) + 1ULL; - } - - return base::size(); - } -}; - -using huge_object_json = nlohmann::json::with_object_t; - -TEST_CASE("MessagePack Size above uint32 for object") -{ - - huge_object_json j = huge_object_json::object(); - - j["one"] = 1; - j["two"] = 2; - - auto& object = j.get_ref(); - object.fake_size = true; - - // write into a caller-owned vector: to_msgpack(j) reserves space based on - // the (faked) element count, which fails with bad_alloc on Windows - std::vector result; - CHECK_THROWS_WITH_AS( - huge_object_json::to_msgpack(j, result), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); - - object.fake_size = false; -} - -#ifdef JSON_TEST_BEYOND_UINT32_STRING -struct huge_string : std::string -{ - using std::string::string; - - std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return static_cast(UINT32_MAX) + 1ULL; - } -}; - -using huge_string_json = nlohmann::json::with_string_t; - -TEST_CASE("MessagePack Size above uint32 for string") -{ - const huge_string_json j = "hello"; - - CHECK_THROWS_WITH_AS( - huge_string_json::to_msgpack(j), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); -} -#endif - -struct huge_binary : std::vector -{ - using std::vector::vector; - - std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return static_cast(UINT32_MAX) + 1ULL; - } -}; - -using huge_binary_json = nlohmann::json::with_binary_t; - -TEST_CASE("MessagePack Size above uint32 for binary") -{ - - huge_binary_json j = huge_binary_json::binary(huge_binary{}); - - j.get_binary().push_back(0x01); - j.get_binary().push_back(0x02); - - CHECK_THROWS_WITH_AS( - huge_binary_json::to_msgpack(j), - "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295", - json::out_of_range&); -} -#endif - -namespace -{ -// types that report a size beyond UINT32_MAX without allocating that much -// memory, so the MessagePack length limit can be tested cheaply; see the -// similar types in unit-bson.cpp -std::size_t beyond_uint32_size() -{ - return static_cast((std::numeric_limits::max)()) + 1; -} - -class beyond_uint32_binary_t : public std::vector -{ - public: - using std::vector::vector; - - size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return beyond_uint32_size(); - } -}; - -#ifdef JSON_TEST_BEYOND_UINT32_STRING -class beyond_uint32_string_t : public std::string -{ - public: - using std::string::string; - - size_type size() const noexcept // NOLINT(readability-convert-member-functions-to-static) - { - return beyond_uint32_size(); - } -}; - -using beyond_uint32_string_json = nlohmann::json::with_string_t; -#endif - -using beyond_uint32_binary_json = nlohmann::json::with_binary_t; -} // namespace - -TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized") -{ - // MessagePack stores the length of a string, binary value, array, or - // object in at most 32 bits; a larger one used to be written without any - // length at all -#if SIZE_MAX > UINT32_MAX - { - const char* const expected = "[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295"; - - const beyond_uint32_binary_json binary = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}); - CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(binary), expected, beyond_uint32_binary_json::out_of_range&); - - const beyond_uint32_binary_json ext = beyond_uint32_binary_json::binary(beyond_uint32_binary_t{}, 42); - CHECK_THROWS_WITH_AS(beyond_uint32_binary_json::to_msgpack(ext), expected, beyond_uint32_binary_json::out_of_range&); - -#ifdef JSON_TEST_BEYOND_UINT32_STRING - // created from its type rather than from a beyond_uint32_string_t: - // that would consider the std::filesystem::path conversion, which - // libstdc++ 10 cannot decide for a class derived from std::string - const beyond_uint32_string_json string(beyond_uint32_string_json::value_t::string); - CHECK_THROWS_WITH_AS(beyond_uint32_string_json::to_msgpack(string), expected, beyond_uint32_string_json::out_of_range&); -#endif - } -#endif -} - -TEST_CASE("MessagePack numbers use the active union member (see #5644)") -{ - // when number_integer_t is narrower than number_unsigned_t, to_msgpack() - // used to read the union member that was not the active one, writing - // wrong bytes for some values; std::int64_t/std::uint64_t (the default - // types, where both members have the same width) were not affected - using int32_json = nlohmann::json::with_integers_t; - using int16_json = nlohmann::json::with_integers_t; - - SECTION("number_integer_t = std::int32_t") - { - SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)") - { - const int32_json j = 6442450944ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00}; - const auto result = int32_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int32_json::from_msgpack(result) == j); - } - - SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)") - { - const int32_json j = 4294967496ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; - const auto result = int32_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int32_json::from_msgpack(result) == j); - } - } - - SECTION("number_integer_t = std::int16_t, 98304 (uint 32)") - { - const int16_json j = 98304ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xce, 0x00, 0x01, 0x80, 0x00}; - const auto result = int16_json::to_msgpack(j); - CHECK(result == expected); - CHECK(int16_json::from_msgpack(result) == j); - } - - SECTION("default types (std::int64_t/std::uint64_t) are unaffected") - { - const json j = 4294967496ULL; - CHECK(j.is_number_unsigned()); - - std::vector const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8}; - const auto result = json::to_msgpack(j); - CHECK(result == expected); - CHECK(json::from_msgpack(result) == j); - } -} - TEST_CASE("MessagePack large strings and binaries (chunked reader)") { // get_msgpack_string()/get_msgpack_binary() both read through get_binary(), diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index c44b1bc92..b31cfd054 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -43,7 +43,6 @@ using ordered_json = nlohmann::ordered_json; #endif #include -#include #include #include #include @@ -867,52 +866,6 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with } -TEST_CASE("regression test #5476 - array type without reserve()") -{ - // the capacity reserved for definite-length arrays must not require the - // array type to have a reserve() member function - using deque_json = nlohmann::json::with_array_t; - - SECTION("std::deque") - { - const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); - CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); - - // the binary formats pass a definite length to start_array() - CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); - CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); - CHECK(deque_json::from_bon8(deque_json::to_bon8(j)) == j); - - // parse() instantiates the callback parser as well, which reserves too - const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept - { - return true; - }); - CHECK(with_callback == deque_json({1, 2, 3})); - } - - SECTION("std::vector still reserves") - { - json array = json::array(); - for (int i = 0; i < 100; ++i) - { - array.push_back(i); - } - - const auto j = json::from_cbor(json::to_cbor(array)); - CHECK(j == array); - CHECK(j.get_ref().capacity() >= 100); - } - - SECTION("the reservation stays capped") - { - // CBOR array announcing 2^32-1 elements, but truncated right after the - // header: the input must be rejected without reserving that capacity - const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; - CHECK(json::from_cbor(truncated, true, false).is_discarded()); - } -} - TEST_CASE("issue #5317 - nested indefinite-length CBOR string chunks are rejected") { json _;