diff --git a/docs/mkdocs/docs/api/basic_json/to_bjdata.md b/docs/mkdocs/docs/api/basic_json/to_bjdata.md index 583139068..aa0240a73 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bjdata.md +++ b/docs/mkdocs/docs/api/basic_json/to_bjdata.md @@ -123,4 +123,6 @@ Linear in the size of the JSON value `j`. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an - array or object was silently skipped, producing invalid BJData. \ No newline at end of file + array or object was silently skipped, producing invalid BJData. +- Writes unsigned integers wider than 64 bits as high-precision numbers since version 3.13.0; previously, they were + silently truncated to 64 bits. \ No newline at end of file diff --git a/docs/mkdocs/docs/api/basic_json/to_bon8.md b/docs/mkdocs/docs/api/basic_json/to_bon8.md index 2d58b2660..508ab1d6b 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bon8.md +++ b/docs/mkdocs/docs/api/basic_json/to_bon8.md @@ -37,8 +37,9 @@ With (2), the bytes written before the exception remain in the output adapter. ## Exceptions -- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an unsigned integer - above 9223372036854775807, which BON8 cannot represent +- Throws [out_of_range.407](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer outside + the range of int64 (an unsigned integer above 9223372036854775807, or, with a number type wider than 64 bits, any + integer beyond int64), which BON8 cannot represent - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if `j` contains a string that is not valid UTF-8 diff --git a/docs/mkdocs/docs/api/basic_json/to_bson.md b/docs/mkdocs/docs/api/basic_json/to_bson.md index 05f8b8548..315477e19 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bson.md +++ b/docs/mkdocs/docs/api/basic_json/to_bson.md @@ -47,6 +47,10 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va - Throws [`type_error.317`](../../home/exceptions.md#jsonexceptiontype_error317) if the top-level type of the JSON value is not an object; example: `"to serialize to BSON, top-level type must be object, but is string"` +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains a signed integer + outside the range of int64 or an unsigned integer outside the range of uint64, which is only possible with a number + type wider than 64 bits; example: + `"integer number 9223372036854775808 cannot be represented by BSON as it does not fit int64"` - Throws [`out_of_range.409`](../../home/exceptions.md#jsonexceptionout_of_range409) if a key in the JSON object contains a null byte (code point U+0000); example: `"BSON key cannot contain code point U+0000 (at byte 2)"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a document, array, @@ -119,3 +123,5 @@ pass before anything is written. that is not valid UTF-8 unchanged, as before; `strict` (the default if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316` before anything is written. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_cbor.md b/docs/mkdocs/docs/api/basic_json/to_cbor.md index f5b6a07bb..6508864e1 100644 --- a/docs/mkdocs/docs/api/basic_json/to_cbor.md +++ b/docs/mkdocs/docs/api/basic_json/to_cbor.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^64, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]"` - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is not valid UTF-8 and `error_handler` is `strict` (the default only if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) @@ -90,3 +93,5 @@ Linear in the size of the JSON value `j`. [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid CBOR. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/api/basic_json/to_msgpack.md b/docs/mkdocs/docs/api/basic_json/to_msgpack.md index 2e32208bf..dca66271d 100644 --- a/docs/mkdocs/docs/api/basic_json/to_msgpack.md +++ b/docs/mkdocs/docs/api/basic_json/to_msgpack.md @@ -46,6 +46,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va ## Exceptions +- Throws [`out_of_range.407`](../../home/exceptions.md#jsonexceptionout_of_range407) if `j` contains an integer + outside [-2^63, 2^64-1], which is only possible with a number type wider than 64 bits; example: + `"integer number 18446744073709551616 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]"` - Throws [`out_of_range.412`](../../home/exceptions.md#jsonexceptionout_of_range412) if the length of a string, binary value, array, or object exceeds 4294967295, the maximum MessagePack can store; example: `"MessagePack length 4294967296 exceeds maximum of 4294967295"` @@ -112,3 +115,5 @@ Linear in the size of the JSON value `j`. `number_unsigned_t`. - Throws `type_error.321` for a discarded value since version 3.13.0; previously, a discarded value nested in an array or object was silently skipped, producing invalid MessagePack. +- Throws `out_of_range.407` for integers that do not fit 64 bits since version 3.13.0; previously, integers of a + number type wider than 64 bits were silently truncated. diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index d6fdce37a..94b554cbf 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -906,14 +906,34 @@ double-precision number when `number_float_t` is `#!cpp float`. ### json.exception.out_of_range.407 -This exception previously indicated that the UBJSON and BSON binary formats did not support integer numbers greater than -9223372036854775807 due to limitations in the implemented mapping. However, these limitations have since been resolved, -and this exception no longer occurs. +An integer number cannot be represented by the binary format it is serialized to: -!!! success "Exception cannot occur any more" +- [BON8](../features/binary_formats/bon8.md) only stores integers that fit into int64. +- [CBOR](../features/binary_formats/cbor.md), [MessagePack](../features/binary_formats/msgpack.md), and + [BSON](../features/binary_formats/bson.md) store integers in at most 64 bits. With the default number types, every + integer fits, but a [`number_integer_t`](../api/basic_json/number_integer_t.md) or + [`number_unsigned_t`](../api/basic_json/number_unsigned_t.md) wider than 64 bits (e.g., `__int128`) can hold values + outside the range of the format: [-2^64, 2^64-1] for CBOR, [-2^63, 2^64-1] for MessagePack, and the range of int64 + (signed integers) or uint64 (unsigned integers) for BSON. - - Since version 3.9.0, integer numbers beyond int64 are serialized as high-precision UBJSON numbers. - - Since version 3.12.0, integer numbers beyond int64 are serialized as uint64 BSON numbers. +[UBJSON](../features/binary_formats/ubjson.md) and [BJData](../features/binary_formats/bjdata.md) never throw this +exception, because they serialize integers beyond 64 bits as high-precision numbers. + +!!! failure "Example messages" + + ``` + integer number 9223372036854775808 cannot be represented by BON8 as it does not fit int64 + ``` + ``` + integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1] + ``` + +!!! note + + Before version 3.13.0, CBOR, MessagePack, and BSON silently truncated integers wider than 64 bits, and BJData + truncated unsigned integers wider than 64 bits. This exception was previously thrown by UBJSON and BSON for + integers greater than 9223372036854775807; since version 3.9.0, such integers are serialized as high-precision + UBJSON numbers, and since version 3.12.0 as uint64 BSON numbers. ### json.exception.out_of_range.408 diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 212d2658f..1bc4e1339 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -207,6 +207,10 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } // CBOR does not differentiate between positive signed // integers and unsigned integers write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); @@ -214,6 +218,10 @@ class binary_writer else { // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); } break; @@ -221,7 +229,11 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } + write_cbor_head(0x00, static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -429,12 +441,20 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -473,6 +493,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -834,6 +858,84 @@ class binary_writer JSON_THROW(type_error::create(321, concat("cannot serialize discarded value to ", format_name), &j)); } + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant < bool, + (std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant < bool, std::numeric_limits::is_signed && + (!std::numeric_limits::is_signed || std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // dump() rather than std::to_string(), which has no overload for + // integer types wider than 64 bits + const auto number = j.dump(); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(407, concat("integer number ", std::string(number.begin(), number.end()), " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -1590,6 +1692,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -1608,10 +1712,18 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } + return calc_bson_unsigned_size(static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -1649,11 +1761,12 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); @@ -2108,7 +2221,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -2233,14 +2346,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -2472,10 +2587,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -2589,9 +2705,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -2600,6 +2716,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; diff --git a/include/nlohmann/detail/value_t.hpp b/include/nlohmann/detail/value_t.hpp index 9fc217bcc..e1139da04 100644 --- a/include/nlohmann/detail/value_t.hpp +++ b/include/nlohmann/detail/value_t.hpp @@ -14,7 +14,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed #include #if JSON_HAS_THREE_WAY_COMPARISON @@ -147,13 +146,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 09f67bdc3..f8c1a5763 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -284,7 +284,6 @@ #include // uint8_t #include // numeric_limits #include // string -#include // is_signed // #include // __ _____ _____ _____ @@ -3363,13 +3362,16 @@ FloatType compare_integer_with_float(const IntegerType i, const FloatType f) noe // values of IntegerType lie in [-bound, bound) when signed and in // [0, bound) when unsigned; digits excludes the sign bit, so bound is a - // power of two that the float represents exactly - const FloatType bound = std::ldexp(static_cast(1), std::numeric_limits::digits); + // power of two that the float represents exactly; the signedness comes + // from numeric_limits as well, because std::is_signed is false for class + // types such as 128-bit or multiprecision integers + using limits = std::numeric_limits; + const FloatType bound = std::ldexp(static_cast(1), limits::digits); if (f >= bound) { return ordered(-1); } - if (std::is_signed::value ? (f < -bound) : (f < static_cast(0))) + if (limits::is_signed ? (f < -bound) : (f < static_cast(0))) { return ordered(1); } @@ -21824,6 +21826,10 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } // CBOR does not differentiate between positive signed // integers and unsigned integers write_cbor_head(0x00, static_cast(j.m_data.m_value.number_integer)); @@ -21831,6 +21837,10 @@ class binary_writer else { // a negative integer n is encoded as -1 - n + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(-1 - j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "CBOR", "[-2^64, 2^64-1]"); + } write_cbor_head(0x20, static_cast(-1 - j.m_data.m_value.number_integer)); } break; @@ -21838,7 +21848,11 @@ class binary_writer case value_t::number_unsigned: { - write_cbor_head(0x00, j.m_data.m_value.number_unsigned); + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "CBOR", "uint64"); + } + write_cbor_head(0x00, static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -22046,12 +22060,20 @@ class binary_writer { if (j.m_data.m_value.number_integer >= 0) { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } // MessagePack does not differentiate between positive // signed integers and unsigned integers. write_msgpack_unsigned(static_cast(j.m_data.m_value.number_integer)); } else { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_min(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "MessagePack", "[-2^63, 2^64-1]"); + } if (j.m_data.m_value.number_integer >= -32) { // negative fixnum @@ -22090,6 +22112,10 @@ class binary_writer case value_t::number_unsigned: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "MessagePack", "uint64"); + } write_msgpack_unsigned(static_cast(j.m_data.m_value.number_unsigned)); break; } @@ -22451,6 +22477,84 @@ class binary_writer JSON_THROW(type_error::create(321, concat("cannot serialize discarded value to ", format_name), &j)); } + /*! + @brief whether integer @a n does not exceed the maximum of @a TargetType + + The binary formats store integers in at most 64 bits, but number_integer_t + and number_unsigned_t may be wider (e.g., __int128). The range of the + integer is taken from std::numeric_limits, so a number type whose range + fits into @a TargetType is never checked: the function is then constant + true, and the default int64_t/uint64_t types pay nothing. + */ + template + static constexpr bool integer_fits_max(const NumberType n) noexcept + { + return integer_fits_max(n, std::integral_constant < bool, + (std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_max(const NumberType /*unused*/, std::false_type /*may_exceed*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_max(const NumberType n, std::true_type /*may_exceed*/) noexcept + { + // NumberType has more digits than TargetType, so it can hold its maximum + return n <= static_cast((std::numeric_limits::max)()); + } + + /*! + @brief whether integer @a n is not below the minimum of @a TargetType; + constant true if NumberType cannot hold such a value + */ + template + static constexpr bool integer_fits_min(const NumberType n) noexcept + { + return integer_fits_min(n, std::integral_constant < bool, std::numeric_limits::is_signed && + (!std::numeric_limits::is_signed || std::numeric_limits::digits > std::numeric_limits::digits) > ()); + } + + template + static constexpr bool integer_fits_min(const NumberType /*unused*/, std::false_type /*may_fall_below*/) noexcept + { + return true; + } + + template + static constexpr bool integer_fits_min(const NumberType n, std::true_type /*may_fall_below*/) noexcept + { + // NumberType is signed and either TargetType is unsigned (minimum 0) + // or NumberType has more digits, so it can hold the minimum + return n >= static_cast((std::numeric_limits::min)()); + } + + /*! + @brief whether integer @a n lies in the range of @a TargetType + */ + template + static constexpr bool integer_fits(const NumberType n) noexcept + { + return integer_fits_min(n) && integer_fits_max(n); + } + + /*! + @brief throws because the integer @a j is too large for @a format_name + @throw out_of_range.407 always + */ + JSON_HEDLEY_NO_RETURN static void throw_integer_out_of_range(const BasicJsonType& j, const char* format_name, const char* range) + { + // dump() rather than std::to_string(), which has no overload for + // integer types wider than 64 bits + const auto number = j.dump(); + static_cast(number); // unused when JSON_NOEXCEPTION is defined + static_cast(format_name); + static_cast(range); + JSON_THROW(out_of_range::create(407, concat("integer number ", std::string(number.begin(), number.end()), " cannot be represented by ", format_name, " as it does not fit ", range), &j)); + } + void write_msgpack_array_prefix(const std::size_t N, const BasicJsonType& j) { const auto n = to_msgpack_length(N, j); @@ -23207,6 +23311,8 @@ class binary_writer is neither an object nor an array @throw out_of_range.415 if @a j is binary with a subtype that does not fit into a byte, before anything is written + @throw out_of_range.407 if @a j is an integer that does not fit the 64 bits + of a BSON integer, before anything is written @throw type_error.316 if @a j is a string that is not valid UTF-8, before anything is written @throw type_error.321 if @a j is discarded @@ -23225,10 +23331,18 @@ class binary_writer return 8ul; case value_t::number_integer: - return calc_bson_integer_size(j.m_data.m_value.number_integer); + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BSON", "int64"); + } + return calc_bson_integer_size(static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: - return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned); + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) + { + throw_integer_out_of_range(j, "BSON", "uint64"); + } + return calc_bson_unsigned_size(static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return calc_bson_string_size(*j.m_data.m_value.string, j); @@ -23266,11 +23380,12 @@ class binary_writer case value_t::number_float: return write_bson_double(name, j.m_data.m_value.number_float); + // calc_bson_value_size() checked that integers fit 64 bits case value_t::number_integer: - return write_bson_integer(name, j.m_data.m_value.number_integer); + return write_bson_integer(name, static_cast(j.m_data.m_value.number_integer)); case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_data.m_value.number_unsigned); + return write_bson_unsigned(name, static_cast(j.m_data.m_value.number_unsigned)); case value_t::string: return write_bson_string(name, *j.m_data.m_value.string); @@ -23725,7 +23840,7 @@ class binary_writer { return 'L'; } - if (use_bjdata && std::is_unsigned::value) + if (use_bjdata && std::is_unsigned::value && integer_fits_max(n)) { return 'M'; } @@ -23850,14 +23965,16 @@ class binary_writer @brief checks whether a JSON number fits into @a TargetType @param[in] el a JSON number of either the signed or unsigned integer kind @return whether @a el's value can be represented by @a TargetType without - wrapping, regardless of which of the two kinds it is stored as + wrapping, regardless of which of the two kinds it is stored as; + false for a value that does not even fit the 64-bit type it is + read as (possible for number types wider than 64 bits) */ template static bool bjdata_ndarray_value_in_range(const BasicJsonType& el) { return el.is_number_unsigned() - ? value_in_range_of(el.template get()) - : value_in_range_of(el.template get()); + ? integer_fits_max(el.m_data.m_value.number_unsigned) && value_in_range_of(el.template get()) + : integer_fits(el.m_data.m_value.number_integer) && value_in_range_of(el.template get()); } /*! @@ -24089,10 +24206,11 @@ class binary_writer for (const auto& el : dims) { // a dimension is read as an unsigned value below, so anything that - // is not a non-negative integer is rejected: a non-integer entry - // would pun unrelated bytes as the dimension, and a negative one - // would wrap into a nonsensical length - if (!el.is_number_integer() || (!el.is_number_unsigned() && el.template get() < 0)) + // is not a non-negative integer in the range of std::uint64_t is + // rejected: a non-integer entry would pun unrelated bytes as the + // dimension, and a negative or wider one would wrap into a + // nonsensical length + if (!el.is_number_integer() || !bjdata_ndarray_value_in_range(el)) { return true; } @@ -24206,9 +24324,9 @@ class binary_writer case value_t::number_unsigned: { - if (j.m_data.m_value.number_unsigned > static_cast((std::numeric_limits::max)())) + if (JSON_HEDLEY_UNLIKELY(!integer_fits_max(j.m_data.m_value.number_unsigned))) { - JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j)); + throw_integer_out_of_range(j, "BON8", "int64"); } write_bon8_integer(static_cast(j.m_data.m_value.number_unsigned)); string_open = false; @@ -24217,6 +24335,10 @@ class binary_writer case value_t::number_integer: { + if (JSON_HEDLEY_UNLIKELY(!integer_fits(j.m_data.m_value.number_integer))) + { + throw_integer_out_of_range(j, "BON8", "int64"); + } write_bon8_integer(static_cast(j.m_data.m_value.number_integer)); string_open = false; break; diff --git a/tests/src/unit-custom-number-types.cpp b/tests/src/unit-custom-number-types.cpp new file mode 100644 index 000000000..5555af34e --- /dev/null +++ b/tests/src/unit-custom-number-types.cpp @@ -0,0 +1,350 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include + +#include +#include +#include +#include +#include +#include + +#if JSON_HAS_THREE_WAY_COMPARISON + #include +#endif + +namespace custom_number_types +{ + +// a signed integer of class type: std::is_signed is only true for arithmetic +// types, so the library has to take the signedness from std::numeric_limits, +// as it does for 128-bit and multiprecision integer classes such as +// absl::int128 or boost::multiprecision::cpp_int +class class_int +{ + public: + // trivial, like the 128-bit integer classes: the value is stored in a union + class_int() = default; + + // implicit from built-in integers and explicit from floats, like absl::int128 + template::value, int>::type = 0> + class_int(T v) : value(static_cast(v)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + template::value, int>::type = 0> + explicit class_int(T v) : value(static_cast(v)) {} + + template::value, int>::type = 0> + explicit operator T() const + { + return static_cast(value); + } + + friend bool operator==(class_int lhs, class_int rhs) + { + return lhs.value == rhs.value; + } + friend bool operator!=(class_int lhs, class_int rhs) + { + return lhs.value != rhs.value; + } + friend bool operator<(class_int lhs, class_int rhs) + { + return lhs.value < rhs.value; + } +#if JSON_HAS_THREE_WAY_COMPARISON + friend std::strong_ordering operator<=>(class_int lhs, class_int rhs) // *NOPAD* + { + return lhs.value <=> rhs.value; // *NOPAD* + } +#endif + + private: + std::int64_t value; +}; + +} // namespace custom_number_types + +using custom_number_types::class_int; + +namespace std +{ +// only the members the library uses +template<> +class numeric_limits +{ + public: + static constexpr bool is_signed = true; + static constexpr int digits = std::numeric_limits::digits; +}; +} // namespace std + +namespace +{ + +using class_int_json = nlohmann::basic_json; + +class_int_json make_class_int(std::int64_t v) +{ + class_int_json j(class_int_json::value_t::number_integer); + j.get_ref() = class_int(v); + return j; +} + +} // namespace + +// the serializer cannot print an integer of class type, so doctest must not +// try when an assertion fails +namespace doctest +{ +template<> +struct StringMaker +{ + static String convert(const class_int_json& j) + { + return j.type_name(); + } +}; +} // namespace doctest + +// __int128 as number type needs the std::numeric_limits (for the range checks) +// and std::is_integral (for the serializer) specializations, which libc++ +// always provides and libstdc++ only outside strict ISO modes; the MSVC +// standard library has none +#if defined(__SIZEOF_INT128__) && (defined(_LIBCPP_VERSION) || defined(__GLIBCXX_TYPE_INT_N_0)) + #define JSON_TEST_INT128_NUMBER_TYPES 1 +#else + #define JSON_TEST_INT128_NUMBER_TYPES 0 +#endif + +#if JSON_TEST_INT128_NUMBER_TYPES +namespace +{ + +// __extension__ keeps -Wpedantic from flagging the non-standard type +__extension__ typedef __int128 int128; // NOLINT(modernize-use-using) +__extension__ typedef unsigned __int128 uint128; // NOLINT(modernize-use-using) + +static_assert(std::numeric_limits::digits == 127 && std::numeric_limits::digits == 128 && + std::is_integral::value && std::is_integral::value, + "__int128 is not fully supported by the standard library"); + +using wide_json = nlohmann::basic_json; + +wide_json make_int(int128 v) +{ + wide_json j(wide_json::value_t::number_integer); + j.get_ref() = v; + return j; +} + +wide_json make_uint(uint128 v) +{ + wide_json j(wide_json::value_t::number_unsigned); + j.get_ref() = v; + return j; +} + +wide_json wrap(const wide_json& value) +{ + wide_json o(wide_json::value_t::object); + o["v"] = value; + return o; +} + +} // namespace + +TEST_CASE("custom number types: integers beyond 64 bits") +{ + using out_of_range = wide_json::out_of_range; + + const int128 two_64 = static_cast(1) << 64; + const int128 two_100 = static_cast(1) << 100; + const int128 max_int64 = (std::numeric_limits::max)(); + const int128 min_int64 = (std::numeric_limits::min)(); + const uint128 max_uint64 = (std::numeric_limits::max)(); + + // before the range checks, these values were silently truncated, e.g., + // 2^100 was written as 0 + const wide_json int_big = make_int(two_100); + const wide_json int_big_negative = make_int(-two_100); + const wide_json uint_big = make_uint(static_cast(two_100)); + std::vector _; + + SECTION("CBOR") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by CBOR as it does not fit [-2^64, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_cbor(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by CBOR as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_int(-two_64 - 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_cbor(make_uint(static_cast(max_uint64) + 1)), out_of_range&); + + // CBOR's integers cover [-2^64, 2^64-1] + CHECK(wide_json::to_cbor(make_int(two_64 - 1)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_int(-two_64)) == std::vector({0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_cbor(make_uint(max_uint64)) == std::vector({0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("MessagePack") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit [-2^63, 2^64-1]", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_msgpack(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by MessagePack as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(two_64)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_msgpack(make_int(min_int64 - 1)), out_of_range&); + + // MessagePack's integers cover [-2^63, 2^64-1] + CHECK(wide_json::to_msgpack(make_int(two_64 - 1)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + CHECK(wide_json::to_msgpack(make_int(min_int64)) == std::vector({0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); + CHECK(wide_json::to_msgpack(make_uint(max_uint64)) == std::vector({0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + } + + SECTION("BSON") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(int_big_negative)), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BSON as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bson(wrap(uint_big)), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BSON as it does not fit uint64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(max_int64 + 1))), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bson(wrap(make_int(min_int64 - 1))), out_of_range&); + + // nested values are checked as well, before anything is written + wide_json nested(wide_json::value_t::object); + nested["a"] = wide_json::array({wrap(int_big)}); + std::vector out; + CHECK_THROWS_AS(wide_json::to_bson(nested, out), out_of_range&); + CHECK(out.empty()); + + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(max_int64)))) == wrap(make_int(max_int64))); + CHECK(wide_json::from_bson(wide_json::to_bson(wrap(make_int(min_int64)))) == wrap(make_int(min_int64))); + CHECK_NOTHROW(wide_json::to_bson(wrap(make_uint(max_uint64)))); + } + + SECTION("BON8") + { + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(int_big_negative), "[json.exception.out_of_range.407] integer number -1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_WITH_AS(_ = wide_json::to_bon8(uint_big), "[json.exception.out_of_range.407] integer number 1267650600228229401496703205376 cannot be represented by BON8 as it does not fit int64", out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(max_int64 + 1)), out_of_range&); + CHECK_THROWS_AS(_ = wide_json::to_bon8(make_int(min_int64 - 1)), out_of_range&); + + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(max_int64))) == make_int(max_int64)); + CHECK(wide_json::from_bon8(wide_json::to_bon8(make_int(min_int64))) == make_int(min_int64)); + } + + SECTION("UBJSON and BJData") + { + // integers beyond 64 bits are written exactly as high-precision numbers + const std::string digits = "1267650600228229401496703205376"; + std::vector expected = {'H', 'i', static_cast(digits.size())}; + expected.insert(expected.end(), digits.begin(), digits.end()); + CHECK(wide_json::to_ubjson(int_big) == expected); + CHECK(wide_json::to_ubjson(uint_big) == expected); + CHECK(wide_json::to_bjdata(int_big) == expected); + // BJData's uint64 marker 'M' was used for any unsigned value beyond + // int64, which truncated the ones beyond 64 bits + CHECK(wide_json::to_bjdata(uint_big) == expected); + + // the uint64 marker is still used where the value fits + CHECK(wide_json::to_bjdata(make_uint(max_uint64)) == std::vector({'M', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + + // the elements of an optimized container are written the same way; + // BJData does not allow 'H' as the type of an optimized container + std::vector expected_ubjson_array = {'[', '$', 'H', '#', 'i', 2}; + std::vector expected_bjdata_array = {'[', '#', 'i', 2}; + for (int i = 0; i < 2; ++i) + { + expected_ubjson_array.insert(expected_ubjson_array.end(), expected.begin() + 1, expected.end()); + expected_bjdata_array.insert(expected_bjdata_array.end(), expected.begin(), expected.end()); + } + CHECK(wide_json::to_ubjson(wide_json::array({uint_big, uint_big}), true, true) == expected_ubjson_array); + CHECK(wide_json::to_bjdata(wide_json::array({uint_big, uint_big}), true, true) == expected_bjdata_array); + } + + SECTION("BJData ND-array") + { + // an ND-array element beyond 64 bits does not fit any dtype, so the + // annotated object is written as a plain object instead of truncating + // the element into range + wide_json element(wide_json::value_t::object); + element["_ArrayType_"] = "uint8"; + element["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(1)}); + element["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(static_cast(two_64) + 1)}); + const auto element_bytes = wide_json::to_bjdata(element, true, true); + REQUIRE(!element_bytes.empty()); + CHECK(element_bytes[0] == '{'); + + wide_json signed_element = element; + signed_element["_ArrayType_"] = "int8"; + signed_element["_ArrayData_"] = wide_json::array({make_int(1), make_int(-two_64 + 1)}); + const auto signed_element_bytes = wide_json::to_bjdata(signed_element, true, true); + REQUIRE(!signed_element_bytes.empty()); + CHECK(signed_element_bytes[0] == '{'); + + // the same for a dimension beyond 64 bits, which wrapped into a + // dimension matching the size of _ArrayData_ (here: 1) + wide_json dimension = element; + dimension["_ArraySize_"] = wide_json::array({make_uint(2), make_uint(static_cast(two_64) + 1)}); + dimension["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto dimension_bytes = wide_json::to_bjdata(dimension, true, true); + REQUIRE(!dimension_bytes.empty()); + CHECK(dimension_bytes[0] == '{'); + + // in range, the ND-array is still written + wide_json fits = element; + fits["_ArrayData_"] = wide_json::array({make_uint(1), make_uint(2)}); + const auto fits_bytes = wide_json::to_bjdata(fits, true, true); + REQUIRE(!fits_bytes.empty()); + CHECK(fits_bytes[0] == '['); + } +} + +#endif + +TEST_CASE("custom number types") +{ + SECTION("signed integer of class type compared with a float") + { + // std::is_signed is false for a class type, which made the comparison + // treat any float below zero as less than every integer + const class_int_json minus_five = make_class_int(-5); + const class_int_json minus_two = make_class_int(-2); + const class_int_json five = make_class_int(5); + const class_int_json minus_two_and_a_half = -2.5; + const class_int_json two_and_a_half = 2.5; + const class_int_json huge_negative = -1e30; + const class_int_json huge_positive = 1e30; + + CHECK(minus_five < minus_two_and_a_half); + CHECK_FALSE(minus_two_and_a_half < minus_five); + CHECK(minus_two_and_a_half > minus_five); + CHECK(minus_five <= minus_two_and_a_half); + CHECK(minus_two_and_a_half >= minus_five); + CHECK(minus_five != minus_two_and_a_half); + + CHECK(minus_two_and_a_half < minus_two); + CHECK_FALSE(minus_two < minus_two_and_a_half); + + CHECK(two_and_a_half < five); + CHECK(minus_five < two_and_a_half); + + // floats beyond the integer's range + CHECK(huge_negative < minus_five); + CHECK_FALSE(minus_five < huge_negative); + CHECK(five < huge_positive); + CHECK_FALSE(huge_positive < five); + + // equality + const class_int_json minus_two_float = -2.0; + CHECK(minus_two == minus_two_float); + CHECK(minus_two_float == minus_two); + } + +}