diff --git a/docs/mkdocs/docs/features/binary_formats/ubjson.md b/docs/mkdocs/docs/features/binary_formats/ubjson.md index 76956d60a..72529a925 100644 --- a/docs/mkdocs/docs/features/binary_formats/ubjson.md +++ b/docs/mkdocs/docs/features/binary_formats/ubjson.md @@ -69,6 +69,12 @@ The library uses the following mapping from JSON values types to UBJSON types ac Note that `use_size = true` alone may result in larger representations - the benefit of this parameter is that the receiving side is immediately informed on the number of elements of the container. + An array whose type marker is `Z` (null), `T` (true) or `F` (false) stores no payload at all, because the marker + already is the value. Its declared count is therefore the only thing that decides how much memory the receiving side + allocates, and a handful of bytes can describe billions of elements. `from_ubjson` rejects such an array with + [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408) when the count exceeds 1,048,576, and + `to_ubjson` writes longer arrays of these types without the annotation, so any value it produces can be read back. + !!! info "Binary values" If the JSON data contains the binary type, the value stored is a list of integers, as suggested by the UBJSON diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 7c7d22e23..44a01df40 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -868,6 +868,12 @@ The size of an array or object in a [binary format](../features/binary_formats/i the size following `#` for [UBJSON](../features/binary_formats/ubjson.md)/[BJData](../features/binary_formats/bjdata.md), or the encoded length for [CBOR](../features/binary_formats/cbor.md). +The exception is also thrown for a [UBJSON](../features/binary_formats/ubjson.md) array of a type that is encoded by its +marker alone (`Z`, `T` or `F`) whose declared count exceeds 1,048,576. Such an array has no payload, so its count alone +decides how much memory is allocated, and a handful of bytes would otherwise describe billions of values. +[`to_ubjson`](../api/basic_json/to_ubjson.md) writes longer arrays of these types without the size and type annotation, +so any value it produces can still be read back. + !!! failure "Example messages" ``` @@ -879,6 +885,9 @@ or the encoded length for [CBOR](../features/binary_formats/cbor.md). ``` [json.exception.out_of_range.408] syntax error while parsing CBOR size: excessive map size ``` + ``` + [json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size + ``` ### json.exception.out_of_range.409 diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index f61aaf32f..f699de1b9 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -58,6 +58,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -2799,6 +2819,17 @@ class binary_reader if (size_and_type.first != npos) { + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) { return false; diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index b8e9efa45..9c59cc840 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -826,7 +826,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 64b8c0ef8..4d70f3b32 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10745,6 +10745,26 @@ inline bool little_endianness(int num = 1) noexcept return *reinterpret_cast(&num) == 1; } +/*! +@brief largest element count accepted for a UBJSON container of a valueless type + +An element of type 'Z' (null), 'T' (true) or 'F' (false) is encoded by its +type marker alone, so an optimized container of one of those types has no +payload at all and its declared count is the only thing that decides how much +is allocated: `[$Z#L` followed by a large count turns some ten bytes of input +into that many values (see #2793, which reports 35 GB and 150 seconds). Every +other type costs at least one byte per element and is bounded by the end of +the input. + +This is a sanity bound rather than a security boundary, and it is far above +any container met in practice. @ref binary_writer falls back to the +unoptimized encoding for longer containers, so that a value serialized by +this library can always be read back. + +@sa https://github.com/nlohmann/json/issues/2793 +*/ +JSON_INLINE_VARIABLE constexpr std::size_t max_valueless_container_size = 1 << 20; + /////////////////// // binary reader // /////////////////// @@ -13486,6 +13506,17 @@ class binary_reader if (size_and_type.first != npos) { + // reading an element of a valueless type consumes no input, so the + // declared count alone decides how much is allocated; the check is + // made before the start event so that no container is opened that + // is then abandoned. See @ref max_valueless_container_size. + if (JSON_HEDLEY_UNLIKELY((size_and_type.second == 'Z' || size_and_type.second == 'T' || size_and_type.second == 'F') + && size_and_type.first > max_valueless_container_size)) + { + return sax->parse_error(chars_read, get_token_string(), out_of_range::create(408, + exception_message(input_format, "excessive array size", "size"), nullptr)); + } + if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) { return false; @@ -17923,7 +17954,17 @@ class binary_writer std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type - if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) + // an optimized array of a valueless type carries no payload, so a + // reader has nothing but the declared count to bound the allocation + // by and refuses an excessive one. Write the unoptimized form for + // those, at one byte per element, so the result can be read back. + // Objects are not affected: every element is preceded by its key. + const bool valueless_type = (first_prefix == 'Z' || first_prefix == 'T' || first_prefix == 'F'); + const bool excessive_valueless = valueless_type + && j.m_data.m_value.array->size() > detail::max_valueless_container_size; + + if (same_prefix && !excessive_valueless + && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index b59ac6b38..03446878f 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2149,6 +2149,67 @@ TEST_CASE("UBJSON") } } +TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") +{ + // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an + // optimized array of one of those has no payload and the declared count is + // the only thing deciding how much is allocated. Ten bytes used to produce + // billions of values (#2793); every other type costs at least one byte per + // element and is bounded by the end of the input. + json _; + + SECTION("an excessive count is rejected") + { + // 'l' is a big-endian int32: 0x7FFFFFFF elements, about 34 GB of value + for (const auto marker : + {'Z', 'T', 'F' + }) + { + const std::vector input = {'[', '$', static_cast(marker), '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.out_of_range.408] syntax error while parsing UBJSON size: excessive array size", json::out_of_range&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + } + + SECTION("ordinary counts are unaffected") + { + CHECK(json::from_ubjson(std::vector({'[', '$', 'Z', '#', 'i', 3})) == json({nullptr, nullptr, nullptr})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'T', '#', 'i', 2})) == json({true, true})); + CHECK(json::from_ubjson(std::vector({'[', '$', 'F', '#', 'i', 2})) == json({false, false})); + // 'N' is a no-op rather than a value, and still yields an empty array + CHECK(json::from_ubjson(std::vector({'[', '$', 'N', '#', 'i', 2})) == json::array()); + } + + SECTION("a type with a payload is unaffected") + { + // A count past the limit is not rejected for 'U', which costs a byte + // per element and is bounded by the end of the input instead. The + // count is kept just past the limit rather than made huge, because a + // count that also exceeds the array's max_size() is reported as + // out_of_range before the input runs out, and max_size() depends on + // the width of std::size_t. + const std::vector input = {'[', '$', 'U', '#', 'l', 0x00, 0x10, 0x00, 0x01}; + CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input", json::parse_error&); + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + + SECTION("the writer stays within what the reader accepts") + { + // below the limit the optimized form is used and is tiny; above it the + // writer falls back so that the result can still be read back + json const at_limit(1048576, nullptr); + const auto v_at_limit = json::to_ubjson(at_limit, true, true); + CHECK(v_at_limit.size() == 9); + CHECK(v_at_limit.at(1) == '$'); + CHECK(json::from_ubjson(v_at_limit) == at_limit); + + json const above_limit(1048577, nullptr); + const auto v_above_limit = json::to_ubjson(above_limit, true, true); + CHECK(v_above_limit.at(1) != '$'); + CHECK(json::from_ubjson(v_above_limit) == above_limit); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value")