diff --git a/docs/mkdocs/docs/community/assurance_case.md b/docs/mkdocs/docs/community/assurance_case.md index 02dc6d7d7..89d7289b3 100644 --- a/docs/mkdocs/docs/community/assurance_case.md +++ b/docs/mkdocs/docs/community/assurance_case.md @@ -50,8 +50,8 @@ flowchart LR enabled explicitly. Invalid UTF-8 is rejected. - **Errors are reported, not ignored.** Malformed input results in a [`parse_error`](../home/exceptions.md#parse-errors) with the byte position of the error. Binary readers do not trust announced sizes: strings and binary values grow - only as bytes are actually read, arrays reserve at most a fixed number of elements up front, and sizes that no - container can hold are rejected. + only as bytes are actually read, arrays reserve at most a fixed number of elements up front (and all of them + together no more than the input read so far can back), and sizes that no container can hold are rejected. - **Memory is owned by values.** Each `basic_json` value owns its content, and there is no manual memory management in user code. The destructor does not recurse, so destroying a deeply nested value does not exhaust the stack. - **Bounded recursion.** The JSON parser and the binary readers keep their state in explicit stacks instead of diff --git a/docs/mkdocs/docs/features/parsing/untrusted_input.md b/docs/mkdocs/docs/features/parsing/untrusted_input.md index 894bbc0a4..c6115aa34 100644 --- a/docs/mkdocs/docs/features/parsing/untrusted_input.md +++ b/docs/mkdocs/docs/features/parsing/untrusted_input.md @@ -71,11 +71,20 @@ For the binary formats, an announced size is never trusted outright: actually consumed, rather than allocating the announced length up front — a truncated input runs out of bytes (reported as a parse error) instead of triggering an oversized allocation. - When an array announces its number of elements and the array container supports `reserve()` (as `#!cpp std::vector`, - the default, does), the library reserves storage for at most 16384 of them upfront, regardless of how large the - announced count is; further elements still grow the container normally as they are read. A count that no - input backs, as in a truncated input, reserves at most that much until the parse error is reported. A parse that - completes keeps storage for such a count only where the elements take no input at all, which the two - [UBJSON](../binary_formats/ubjson.md) cases below cover. + the default, does), the DOM parser reserves storage for them before the elements are read, but for at most 16384 + elements per array, regardless of how large the announced count is; further elements still grow the container + normally as they are read. The arrays that are open at the same time hold their storage at the same time, and an + announcement is not backed by input when it is read, so the elements reserved for all arrays of one input together + may not exceed 65,536 plus 2 for every byte of input read so far; an array that announces more gets what is left + of that and grows as its elements arrive. An element takes at least one byte of input, so the elements of the + arrays that were read use up what was reserved for them, and only arrays that are nested deeply, with an array as + their first element, can announce more than is left. For a truncated or hostile input, the storage that + announcements can hold is therefore at most (65,536 + 2 per byte of input) values, that is about 1 MiB plus 32 bytes + per byte of input with 16-byte values (a value is larger with `JSON_DIAGNOSTICS` or `JSON_DIAGNOSTIC_POSITIONS`, 24 to + 40 bytes, and so is the bound). For example, 100,000 nested CBOR array headers `99 40 00` (300,000 bytes) that + announce 16384 elements each peaked at 23 MiB, against 1.65 GiB if every array reserved what it announced and 12 MiB + with version 3.12.0, which reserved nothing. + A [SAX](../../api/json_sax/index.md) handler is not affected: it is told the announced number of elements. - An announced array or object size that exceeds what the target container could ever hold (its `max_size()`) is rejected immediately as [`out_of_range.408`](../../home/exceptions.md#jsonexceptionout_of_range408), without attempting to allocate anything. diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 170f62409..d0591132d 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -16,6 +16,7 @@ #include // back_inserter #include // numeric_limits #include // char_traits, string +#include // is_convertible, true_type, false_type #include // make_pair, move #include // vector #ifdef __cpp_lib_byteswap @@ -180,6 +181,8 @@ class binary_reader sax = sax_; container_stack.clear(); bon8_pushback_size = 0; + // the position is only known to the DOM parser while this reader lives + const position_binding binding(bound_parser(sax, std::is_convertible {}), &chars_read); bool result = false; switch (input_format) @@ -237,6 +240,61 @@ class binary_reader } private: + /// the DOM parser of @a s whose reserved storage is bounded by the input + /// read so far; nullptr for any other SAX parser, which does not reserve, + /// and for one that derives from a DOM parser privately or protectedly + /// (or ambiguously), which the reader cannot reach + static json_sax_dom_error_state* bound_parser(SAX* s, std::true_type /*is a DOM parser*/) noexcept + { + return s; + } + + static json_sax_dom_error_state* bound_parser(SAX* /*s*/, std::false_type /*is a DOM parser*/) noexcept + { + return nullptr; + } + + /*! + @brief tells a DOM parser where to find the number of characters read + while sax_parse() runs, and puts back what it knew before when + sax_parse() returns or throws + + The reader is a local of the caller and is gone then, so the parser must + not keep its position. A parser that was bound by a reader that is still + running (a parse inside a parse) gets that binding back; that reader is + still alive, as it encloses this one. + */ + class position_binding + { + public: + position_binding(json_sax_dom_error_state* parser_, const std::size_t* position) noexcept + : parser(parser_), saved() + { + if (parser != nullptr) + { + saved = parser->get_reserve_state(); + parser->set_input_position(position); + } + } + + position_binding(const position_binding&) = delete; + position_binding(position_binding&&) = delete; + position_binding& operator=(const position_binding&) = delete; + position_binding& operator=(position_binding&&) = delete; + + ~position_binding() + { + if (parser != nullptr) + { + parser->set_reserve_state(saved); + } + } + + private: + json_sax_dom_error_state* parser; + json_sax_dom_error_state::reserve_state saved; + }; + //////////////////////// // nested containers // //////////////////////// diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index fc9e9b774..d961f1856 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -159,6 +159,40 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/// the largest number of elements the DOM parsers reserve for one array upfront +constexpr std::size_t reserve_cap() +{ + return 16384; +} + +/*! +@brief number of array elements a DOM parser may reserve upfront on top of + @ref reserved_elements_per_byte + +An array that announces its size is given storage for that many elements when +it opens (at most @ref reserve_cap of them), before any input backs the +announcement, and the arrays that are open at the same time keep their storage +until the input is complete: 100000 nested headers of three bytes each that +announce 16384 elements would otherwise need 1.7 GB. The elements reserved for +all arrays of an input are therefore added up, and the sum must stay below +this base plus @ref reserved_elements_per_byte for every byte read so far; an +array that would go beyond reserves what is left, and grows as its elements +arrive. The base is four arrays of the maximum size, so the first nested +arrays of an input are not affected. +*/ +JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_base = 4 * reserve_cap(); + +/*! +@brief number of array elements a DOM parser may reserve upfront per byte of + input, in addition to @ref reserved_elements_base + +An element of an array takes at least one byte of input, so an array that +completes has used what was reserved for it. At a rate of 2 elements per +byte, an array that announces more than the input provides keeps at most +32 bytes of memory per byte of input, plus the base, until the parse fails. +*/ +JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_per_byte = 2; + /*! @brief reserve capacity for @a len elements in array @a arr @@ -174,8 +208,7 @@ template auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) -> decltype(arr.reserve(len), void()) { - constexpr std::size_t reserve_cap = 16384; - arr.reserve((std::min)(len, reserve_cap)); + arr.reserve((std::min)(len, reserve_cap())); } template @@ -342,6 +375,13 @@ struct diagnostic_positions Holds whether a parse error occurred and whether errors are thrown, and implements the parse_error() and is_errored() members of the SAX interface for json_sax_dom_parser and json_sax_dom_callback_parser. + +It also bounds what the parsers reserve for arrays that announce their size +(reserve_count()): the announced size is not backed by input yet when the +array opens, and the arrays that are open at the same time each keep what was +reserved for them, so the binary_reader tells the parser where to find the +number of characters it has read (set_input_position()), which the bound +is derived from. */ class json_sax_dom_error_state { @@ -375,7 +415,103 @@ class json_sax_dom_error_state return errored; } + /// bind the number of characters the input has provided so far; see + /// @ref reserve_count + /// (nullptr: not known, which bounds only the size of each array) + void set_input_position(const std::size_t* position) noexcept + { + input_position = position; + credit_position = 0; + reserve_credit = position != nullptr ? reserved_elements_base : (std::numeric_limits::max)(); + } + + /// what set_input_position() changes, so that a binding can be undone + struct reserve_state + { + const std::size_t* position; + std::size_t position_at_refill; + std::size_t credit; + }; + + reserve_state get_reserve_state() const noexcept + { + return {input_position, credit_position, reserve_credit}; + } + + void set_reserve_state(const reserve_state& state) noexcept + { + input_position = state.position; + credit_position = state.position_at_refill; + reserve_credit = state.credit; + } + + protected: + /*! + @brief the number of elements to reserve for an array that announces @a len + + The announced number is not backed by input when the array opens, and the + arrays that are open at the same time each keep what was reserved for + them, so what is reserved for all arrays together is bounded by the + input read so far: @ref reserved_elements_base plus + @ref reserved_elements_per_byte for every character. An array that + announces more than what is left of that is given what is left, and + grows as its elements arrive. Without the position of the input (the + parser is not driven by a binary_reader that is running) nothing is + bounded but the cap of reserve_array(). + + The common case, an array that announces no more than the credit left, + costs a comparison and a subtraction. It is charged what it announces, which + for an array of more than reserve_cap() elements is more than reserve_array() + reserves for it, which its own elements refill as they are read. + */ + std::size_t reserve_count(const std::size_t len) noexcept + { + if (JSON_HEDLEY_LIKELY(len <= reserve_credit)) + { + reserve_credit -= len; + return len; + } + return reserve_count_slowly(len); + } + private: + JSON_HEDLEY_NEVER_INLINE + std::size_t reserve_count_slowly(const std::size_t len) noexcept + { + constexpr std::size_t max_size = (std::numeric_limits::max)(); + if (input_position == nullptr) + { + reserve_credit = max_size; + return len; + } + + // refill the credit with 'reserved_elements_per_byte' for every + // character read since the last refill; it saturates instead of + // overflowing when size_t is 32 bits wide and a very large input has + // been read (the position goes back by up to two in BON8, which has + // no sizes) + const std::size_t position = *input_position; + if (position > credit_position) + { + const std::size_t bytes = position - credit_position; + reserve_credit = bytes <= (max_size - reserve_credit) / reserved_elements_per_byte + ? reserve_credit + (reserved_elements_per_byte * bytes) + : max_size; + credit_position = position; + } + + std::size_t count = (std::min)(len, reserve_cap()); + count = (std::min)(count, reserve_credit); + reserve_credit -= count; + return count; + } + + /// the number of characters the input has provided so far, if known + const std::size_t* input_position = nullptr; + /// the number of characters the input had provided when the credit was refilled + std::size_t credit_position = 0; + /// the number of elements that may still be reserved + std::size_t reserve_credit = reserved_elements_base; /// whether a syntax error occurred bool errored = false; /// whether to throw exceptions in case of errors @@ -1210,7 +1346,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } } @@ -1558,7 +1694,7 @@ class json_sax_dom_binary_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } return true; @@ -1850,7 +1986,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 8e9d828b2..03b2ee5db 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8866,6 +8866,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // back_inserter #include // numeric_limits #include // char_traits, string +#include // is_convertible, true_type, false_type #include // make_pair, move #include // vector #ifdef __cpp_lib_byteswap @@ -16056,6 +16057,40 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/// the largest number of elements the DOM parsers reserve for one array upfront +constexpr std::size_t reserve_cap() +{ + return 16384; +} + +/*! +@brief number of array elements a DOM parser may reserve upfront on top of + @ref reserved_elements_per_byte + +An array that announces its size is given storage for that many elements when +it opens (at most @ref reserve_cap of them), before any input backs the +announcement, and the arrays that are open at the same time keep their storage +until the input is complete: 100000 nested headers of three bytes each that +announce 16384 elements would otherwise need 1.7 GB. The elements reserved for +all arrays of an input are therefore added up, and the sum must stay below +this base plus @ref reserved_elements_per_byte for every byte read so far; an +array that would go beyond reserves what is left, and grows as its elements +arrive. The base is four arrays of the maximum size, so the first nested +arrays of an input are not affected. +*/ +JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_base = 4 * reserve_cap(); + +/*! +@brief number of array elements a DOM parser may reserve upfront per byte of + input, in addition to @ref reserved_elements_base + +An element of an array takes at least one byte of input, so an array that +completes has used what was reserved for it. At a rate of 2 elements per +byte, an array that announces more than the input provides keeps at most +32 bytes of memory per byte of input, plus the base, until the parse fails. +*/ +JSON_INLINE_VARIABLE constexpr std::size_t reserved_elements_per_byte = 2; + /*! @brief reserve capacity for @a len elements in array @a arr @@ -16071,8 +16106,7 @@ template auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) -> decltype(arr.reserve(len), void()) { - constexpr std::size_t reserve_cap = 16384; - arr.reserve((std::min)(len, reserve_cap)); + arr.reserve((std::min)(len, reserve_cap())); } template @@ -16239,6 +16273,13 @@ struct diagnostic_positions Holds whether a parse error occurred and whether errors are thrown, and implements the parse_error() and is_errored() members of the SAX interface for json_sax_dom_parser and json_sax_dom_callback_parser. + +It also bounds what the parsers reserve for arrays that announce their size +(reserve_count()): the announced size is not backed by input yet when the +array opens, and the arrays that are open at the same time each keep what was +reserved for them, so the binary_reader tells the parser where to find the +number of characters it has read (set_input_position()), which the bound +is derived from. */ class json_sax_dom_error_state { @@ -16272,7 +16313,103 @@ class json_sax_dom_error_state return errored; } + /// bind the number of characters the input has provided so far; see + /// @ref reserve_count + /// (nullptr: not known, which bounds only the size of each array) + void set_input_position(const std::size_t* position) noexcept + { + input_position = position; + credit_position = 0; + reserve_credit = position != nullptr ? reserved_elements_base : (std::numeric_limits::max)(); + } + + /// what set_input_position() changes, so that a binding can be undone + struct reserve_state + { + const std::size_t* position; + std::size_t position_at_refill; + std::size_t credit; + }; + + reserve_state get_reserve_state() const noexcept + { + return {input_position, credit_position, reserve_credit}; + } + + void set_reserve_state(const reserve_state& state) noexcept + { + input_position = state.position; + credit_position = state.position_at_refill; + reserve_credit = state.credit; + } + + protected: + /*! + @brief the number of elements to reserve for an array that announces @a len + + The announced number is not backed by input when the array opens, and the + arrays that are open at the same time each keep what was reserved for + them, so what is reserved for all arrays together is bounded by the + input read so far: @ref reserved_elements_base plus + @ref reserved_elements_per_byte for every character. An array that + announces more than what is left of that is given what is left, and + grows as its elements arrive. Without the position of the input (the + parser is not driven by a binary_reader that is running) nothing is + bounded but the cap of reserve_array(). + + The common case, an array that announces no more than the credit left, + costs a comparison and a subtraction. It is charged what it announces, which + for an array of more than reserve_cap() elements is more than reserve_array() + reserves for it, which its own elements refill as they are read. + */ + std::size_t reserve_count(const std::size_t len) noexcept + { + if (JSON_HEDLEY_LIKELY(len <= reserve_credit)) + { + reserve_credit -= len; + return len; + } + return reserve_count_slowly(len); + } + private: + JSON_HEDLEY_NEVER_INLINE + std::size_t reserve_count_slowly(const std::size_t len) noexcept + { + constexpr std::size_t max_size = (std::numeric_limits::max)(); + if (input_position == nullptr) + { + reserve_credit = max_size; + return len; + } + + // refill the credit with 'reserved_elements_per_byte' for every + // character read since the last refill; it saturates instead of + // overflowing when size_t is 32 bits wide and a very large input has + // been read (the position goes back by up to two in BON8, which has + // no sizes) + const std::size_t position = *input_position; + if (position > credit_position) + { + const std::size_t bytes = position - credit_position; + reserve_credit = bytes <= (max_size - reserve_credit) / reserved_elements_per_byte + ? reserve_credit + (reserved_elements_per_byte * bytes) + : max_size; + credit_position = position; + } + + std::size_t count = (std::min)(len, reserve_cap()); + count = (std::min)(count, reserve_credit); + reserve_credit -= count; + return count; + } + + /// the number of characters the input has provided so far, if known + const std::size_t* input_position = nullptr; + /// the number of characters the input had provided when the credit was refilled + std::size_t credit_position = 0; + /// the number of elements that may still be reserved + std::size_t reserve_credit = reserved_elements_base; /// whether a syntax error occurred bool errored = false; /// whether to throw exceptions in case of errors @@ -17107,7 +17244,7 @@ class json_sax_dom_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } } @@ -17455,7 +17592,7 @@ class json_sax_dom_binary_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } return true; @@ -17747,7 +17884,7 @@ class json_sax_dom_callback_parser : public json_sax_dom_error_state if (len != detail::unknown_size()) { - reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + reserve_array(*ref_stack.back()->m_data.m_value.array, reserve_count(len), priority_tag<1> {}); } } @@ -18459,6 +18596,8 @@ class binary_reader sax = sax_; container_stack.clear(); bon8_pushback_size = 0; + // the position is only known to the DOM parser while this reader lives + const position_binding binding(bound_parser(sax, std::is_convertible {}), &chars_read); bool result = false; switch (input_format) @@ -18516,6 +18655,61 @@ class binary_reader } private: + /// the DOM parser of @a s whose reserved storage is bounded by the input + /// read so far; nullptr for any other SAX parser, which does not reserve, + /// and for one that derives from a DOM parser privately or protectedly + /// (or ambiguously), which the reader cannot reach + static json_sax_dom_error_state* bound_parser(SAX* s, std::true_type /*is a DOM parser*/) noexcept + { + return s; + } + + static json_sax_dom_error_state* bound_parser(SAX* /*s*/, std::false_type /*is a DOM parser*/) noexcept + { + return nullptr; + } + + /*! + @brief tells a DOM parser where to find the number of characters read + while sax_parse() runs, and puts back what it knew before when + sax_parse() returns or throws + + The reader is a local of the caller and is gone then, so the parser must + not keep its position. A parser that was bound by a reader that is still + running (a parse inside a parse) gets that binding back; that reader is + still alive, as it encloses this one. + */ + class position_binding + { + public: + position_binding(json_sax_dom_error_state* parser_, const std::size_t* position) noexcept + : parser(parser_), saved() + { + if (parser != nullptr) + { + saved = parser->get_reserve_state(); + parser->set_input_position(position); + } + } + + position_binding(const position_binding&) = delete; + position_binding(position_binding&&) = delete; + position_binding& operator=(const position_binding&) = delete; + position_binding& operator=(position_binding&&) = delete; + + ~position_binding() + { + if (parser != nullptr) + { + parser->set_reserve_state(saved); + } + } + + private: + json_sax_dom_error_state* parser; + json_sax_dom_error_state::reserve_state saved; + }; + //////////////////////// // nested containers // //////////////////////// diff --git a/tests/src/allocation_counter.hpp b/tests/src/allocation_counter.hpp new file mode 100644 index 000000000..3ee8ba3d8 --- /dev/null +++ b/tests/src/allocation_counter.hpp @@ -0,0 +1,86 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#pragma once + +#include // max +#include // size_t +#include // uint8_t +#include // allocator +#include // vector + +#include + +namespace utils +{ +/// the allocations of the values of a JSON type that uses counting_allocator +struct allocation_counts +{ + /// bytes allocated and not yet deallocated + std::size_t current = 0; + /// the largest value current had + std::size_t peak = 0; + /// number of allocations + std::size_t allocations = 0; +}; + +/// the counts of all counting_allocators; the tests are single-threaded +inline allocation_counts& counts() +{ + static allocation_counts c; // NOLINT(misc-const-correctness) + return c; +} + +/// an allocator that counts the bytes and the number of allocations of a +/// JSON type, so that a test can check how much memory a parse holds at any +/// time without measuring the memory of the process +template +struct counting_allocator : std::allocator +{ + using std::allocator::allocator; + + counting_allocator() = default; + template counting_allocator(const counting_allocator& /*unused*/) noexcept {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + template + struct rebind + { + using other = counting_allocator; + }; + + T* allocate(const std::size_t n) + { + T* const result = std::allocator::allocate(n); + allocation_counts& c = counts(); + c.current += n * sizeof(T); + c.peak = (std::max)(c.peak, c.current); + ++c.allocations; + return result; + } + + void deallocate(T* p, const std::size_t n) + { + counts().current -= n * sizeof(T); + std::allocator::deallocate(p, n); + } +}; + +using counted_json = nlohmann::json::with_allocator_t; + +/// @a count times the bytes of @a unit +inline std::vector repeated(const std::vector& unit, const std::size_t count) +{ + std::vector result; + result.reserve(unit.size() * count); + for (std::size_t i = 0; i < count; ++i) + { + result.insert(result.end(), unit.begin(), unit.end()); + } + return result; +} +} // namespace utils diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index dd5831dbd..304db5676 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -31,6 +31,7 @@ using nlohmann::ordered_json; #include "make_test_data_available.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "allocation_counter.hpp" #include "sax_event_loggers.hpp" using utils::SaxCountdown; using utils::SaxEventLogger; @@ -4030,6 +4031,60 @@ TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") } } +TEST_CASE("BJData - open arrays reserve only what the input has backed") +{ + // an array that announces its size gets storage for (up to 16384 of) its + // elements when it opens; the arrays that are open at the same time must + // not hold more than the input read so far can back + SECTION("nested headers that announce 16384 elements and are never completed") + { + // 200 levels would hold 200 * 16384 * sizeof(json), + // if each of them reserved what it announces + const auto input = utils::repeated({'[', '#', 'I', 0x00, 0x40}, 200); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_bjdata(input, true, false); + CHECK(j.is_discarded()); + CAPTURE(utils::counts().peak) + // the credit is 65,536 values; the base peaks at more than 3 million + CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024); + } + + SECTION("a completed array is still reserved in one allocation") + { + json array = json::array(); + for (int i = 0; i < 10000; ++i) + { + array.push_back(i % 100); + } + + // an array without a size is not reserved, so only these two + { + const auto packed = json::to_bjdata(array, true); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_bjdata(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + + { + const auto packed = json::to_bjdata(array, true, true); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_bjdata(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index c4d29543d..05252b205 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -29,6 +29,7 @@ using nlohmann::json; #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "allocation_counter.hpp" #include "sax_event_loggers.hpp" using utils::SaxCountdown; using utils::SaxEventLogger; @@ -2709,6 +2710,289 @@ TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") } } +namespace +{ +// NOLINTBEGIN(bugprone-derived-method-shadowing-base-method) +using dom_parser = nlohmann::detail::json_sax_dom_parser; + +// handlers that wrap a DOM parser, as users of the SAX interface do +struct public_wrapper : dom_parser +{ + explicit public_wrapper(json& result) : dom_parser(result, false) + {} +}; + +/// parses another input with itself while the first parse is still running +struct reentrant_wrapper : dom_parser +{ + explicit reentrant_wrapper(json& result) : dom_parser(result, false) + {} + + bool start_array(std::size_t len) + { + if (!inner_done) + { + inner_done = true; + const std::vector input = {0x80}; + json::sax_parse(input, this, json::input_format_t::cbor); + } + return dom_parser::start_array(len); + } + + bool inner_done = false; +}; + +class private_wrapper : private dom_parser +{ + public: + explicit private_wrapper(json& result) : dom_parser(result, false) + {} + + using dom_parser::null; + using dom_parser::boolean; + using dom_parser::number_integer; + using dom_parser::number_unsigned; + using dom_parser::number_float; + using dom_parser::string; + using dom_parser::binary; + using dom_parser::start_array; + using dom_parser::end_array; + using dom_parser::parse_error; + + // the DOM parser has private overloads of these, which a using + // declaration cannot expose; the input of the test has no objects + static bool start_object(std::size_t /*len*/) + { + return false; + } + + static bool key(std::string& /*val*/) + { + return false; + } + + static bool end_object() + { + return false; + } +}; + +class protected_wrapper : protected dom_parser +{ + public: + explicit protected_wrapper(json& result) : dom_parser(result, false) + {} + + using dom_parser::null; + using dom_parser::boolean; + using dom_parser::number_integer; + using dom_parser::number_unsigned; + using dom_parser::number_float; + using dom_parser::string; + using dom_parser::binary; + using dom_parser::start_array; + using dom_parser::end_array; + using dom_parser::parse_error; + + // the DOM parser has private overloads of these, which a using + // declaration cannot expose; the input of the test has no objects + static bool start_object(std::size_t /*len*/) + { + return false; + } + + static bool key(std::string& /*val*/) + { + return false; + } + + static bool end_object() + { + return false; + } +}; + +// NOLINTEND(bugprone-derived-method-shadowing-base-method) + +/// the capacity of the fifth array of five that are open +std::size_t innermost_capacity(const json& result) +{ + const json* node = &result; + for (int level = 0; level < 4; ++level) + { + node = &node->front(); + } + return node->get_ref().capacity(); +} +} // namespace + +TEST_CASE("CBOR - open arrays reserve only what the input has backed") +{ + // an array that announces its size gets storage for (up to 16384 of) its + // elements when it opens; the arrays that are open at the same time must + // not hold more than the input read so far can back + SECTION("nested headers that announce 16384 elements and are never completed") + { + // 200 levels would hold 200 * 16384 * sizeof(json), + // if each of them reserved what it announces + const auto input = utils::repeated({0x99, 0x40, 0x00}, 200); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_cbor(input, true, false); + CHECK(j.is_discarded()); + CAPTURE(utils::counts().peak) + // the credit is 65,536 values; the base peaks at more than 3 million + CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024); + } + + SECTION("a completed array is still reserved in one allocation") + { + json array = json::array(); + for (int i = 0; i < 10000; ++i) + { + array.push_back(i % 100); + } + + { + const auto packed = json::to_cbor(array); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_cbor(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + } + + SECTION("an array that is given less than it announces is still decoded") + { + // five levels that announce 16384 elements each, the first of which + // is the next level: they are more than the input could back when + // they open + constexpr std::size_t levels = 5; + std::vector input; + for (std::size_t level = 0; level < levels; ++level) + { + input.insert(input.end(), {0x99, 0x40, 0x00}); + } + input.insert(input.end(), 16384, 0x01); + for (std::size_t level = 1; level < levels; ++level) + { + input.insert(input.end(), 16383, 0x02); + } + + const auto j = json::from_cbor(input); + const json* node = &j; + for (std::size_t level = 0; level < levels; ++level) + { + REQUIRE(node->is_array()); + CHECK(node->size() == 16384); + node = &node->front(); + } + CHECK(*node == 1); + CHECK(j.back() == 2); + } + + SECTION("a DOM parser that a parse has finished with is not bounded by the input of that parse") + { + // the DOM parser learns where the reader counts its characters, and + // must forget it when the parse is over: a handler may be driven by + // other code afterwards, while the reader no longer exists + // five levels announce more than the credit of one input, which a + // parse of one byte has hardly refilled + const auto check_unbounded = [](dom_parser & parser, json & result) + { + for (int level = 0; level < 5; ++level) + { + CHECK(parser.start_array(16384)); + } + + const json* node = &result; + for (int level = 0; level < 4; ++level) + { + REQUIRE(node->is_array()); + REQUIRE(node->size() == 1); + node = &node->front(); + } + REQUIRE(node->is_array()); + CHECK(node->get_ref().capacity() == 16384); + }; + + SECTION("after a parse that succeeded") + { + json result; + dom_parser parser(result, false); + const std::vector input = {0x80}; + CHECK(json::sax_parse(input, &parser, json::input_format_t::cbor)); + check_unbounded(parser, result); + } + +#if !defined(JSON_NOEXCEPTION) + SECTION("after a parse that threw") + { + json result; + dom_parser parser(result, true); + const std::vector input = {0x1C}; // not a CBOR value + CHECK_THROWS_AS(json::sax_parse(input, &parser, json::input_format_t::cbor), json::parse_error&); + check_unbounded(parser, result); + } +#endif + } + + SECTION("a handler that wraps a DOM parser is bound if the reader can reach the parser") + { + // five arrays announce 16384 elements: the credit of one input covers + // four of them, and a parse of 15 bytes has hardly refilled it + const auto input = utils::repeated({0x99, 0x40, 0x00}, 5); + + SECTION("public base") + { + json result; + public_wrapper handler(result); + CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor)); + CHECK(innermost_capacity(result) < 16384); + } + + SECTION("public base, a parse inside the parse") + { + json result; + reentrant_wrapper handler(result); + CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor)); + CHECK(handler.inner_done); + CHECK(innermost_capacity(result) < 16384); + } + + // the reader cannot convert these to the DOM parser: they are not bound + SECTION("private base") + { + json result; + private_wrapper handler(result); + CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor)); + CHECK(innermost_capacity(result) == 16384); + } + + SECTION("protected base") + { + json result; + protected_wrapper handler(result); + CHECK_FALSE(json::sax_parse(input, &handler, json::input_format_t::cbor)); + CHECK(innermost_capacity(result) == 16384); + } + } + + SECTION("SAX events report the announced size") + { + SaxEventLogger logger; + const std::vector input = {0x99, 0x40, 0x00}; + CHECK_FALSE(json::sax_parse(input, &logger, json::input_format_t::cbor)); + REQUIRE(logger.events.size() == 2); + CHECK(logger.events[0] == "start_array(16384)"); + CHECK(logger.events[1] == "parse_error(4)"); + } +} + #if !defined(JSON_NOEXCEPTION) // corpus values that do not survive the round trip are skipped by catching the exception TEST_CASE("CBOR round-trip invariants") { diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 9c50d22fd..9a229f652 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -33,6 +33,7 @@ using nlohmann::json; #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "allocation_counter.hpp" #include "sax_event_loggers.hpp" using utils::SaxCountdown; using utils::SaxEventLogger; @@ -1759,6 +1760,47 @@ TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") } } +TEST_CASE("MessagePack - open arrays reserve only what the input has backed") +{ + // an array that announces its size gets storage for (up to 16384 of) its + // elements when it opens; the arrays that are open at the same time must + // not hold more than the input read so far can back + SECTION("nested headers that announce 16384 elements and are never completed") + { + // 200 levels would hold 200 * 16384 * sizeof(json), + // if each of them reserved what it announces + const auto input = utils::repeated({0xDC, 0x40, 0x00}, 200); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_msgpack(input, true, false); + CHECK(j.is_discarded()); + CAPTURE(utils::counts().peak) + // the credit is 65,536 values; the base peaks at more than 3 million + CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024); + } + + SECTION("a completed array is still reserved in one allocation") + { + json array = json::array(); + for (int i = 0; i < 10000; ++i) + { + array.push_back(i % 100); + } + + { + const auto packed = json::to_msgpack(array); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_msgpack(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + } +} + TEST_CASE("regression test - MessagePack ext type rejects a subtype that doesn't fit a single byte") { // subtype 0-255 must still round-trip correctly (regression guard, pre-existing behavior) diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index cabb6be96..ef56fab26 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -31,6 +31,7 @@ using nlohmann::json; #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "allocation_counter.hpp" #include "sax_event_loggers.hpp" using utils::SaxCountdown; using utils::SaxEventLogger; @@ -2880,6 +2881,60 @@ TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") } } +TEST_CASE("UBJSON - open arrays reserve only what the input has backed") +{ + // an array that announces its size gets storage for (up to 16384 of) its + // elements when it opens; the arrays that are open at the same time must + // not hold more than the input read so far can back + SECTION("nested headers that announce 16384 elements and are never completed") + { + // 200 levels would hold 200 * 16384 * sizeof(json), + // if each of them reserved what it announces + const auto input = utils::repeated({'[', '#', 'I', 0x40, 0x00}, 200); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_ubjson(input, true, false); + CHECK(j.is_discarded()); + CAPTURE(utils::counts().peak) + // the credit is 65,536 values; the base peaks at more than 3 million + CHECK(utils::counts().peak < sizeof(utils::counted_json) * 128 * 1024); + } + + SECTION("a completed array is still reserved in one allocation") + { + json array = json::array(); + for (int i = 0; i < 10000; ++i) + { + array.push_back(i % 100); + } + + // an array without a size is not reserved, so only these two + { + const auto packed = json::to_ubjson(array, true); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_ubjson(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + + { + const auto packed = json::to_ubjson(array, true, true); + + utils::counts() = utils::allocation_counts{}; + const auto j = utils::counted_json::from_ubjson(packed); + CHECK(j.size() == 10000); + // the array and its storage; growing the storage as the elements + // arrive would take more than a dozen allocations + CAPTURE(utils::counts().allocations) + CHECK(utils::counts().allocations <= 4); + } + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") {