diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 033cfebd7..c183a94fb 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -12,7 +12,7 @@ #include // array #include // ldexp #include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t #include // snprintf #include // memcpy #include // back_inserter @@ -2908,18 +2908,7 @@ class binary_reader const NumberType len, string_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) - { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) - { - success = false; - break; - } - result.push_back(static_cast(current)); - } - return success; + return get_bytes(format, len, "string", result); } /*! @@ -2941,18 +2930,66 @@ class binary_reader const NumberType len, binary_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) + return get_bytes(format, len, "binary", result); + } + + /*! + @brief read @a len bytes from the input into a string or byte container + + @tparam NumberType the type of the length + @tparam ContainerType the destination container (string_t or binary_t) + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[in] context further context information (for diagnostics) + @param[out] result container the bytes are appended to + + @return whether reading completed + + @note We cannot reserve @a len bytes for the result up front, because + @a len may be far larger than the actual input. Instead we read in + bounded chunks, so the peak allocation is capped regardless of the + claimed length while the per-byte loop is replaced by block copies + (a std::memcpy for contiguous inputs). @ref unexpect_eof() still + detects a premature end of input. + */ + template + bool get_bytes(const input_format_t format, + NumberType len, + const char* context, + ContainerType& result) + { + // upper bound on the number of bytes read (and allocated) per chunk + constexpr std::size_t chunk_size = 4096; + + while (len > 0) { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + // number of bytes to read this iteration: min(chunk_size, len), + // computed without truncating chunk_size to a narrow NumberType + const std::size_t wanted = (static_cast(len) < static_cast(chunk_size)) + ? static_cast(len) + : chunk_size; + const std::size_t old_size = result.size(); + result.resize(old_size + wanted); + // resize() is required to make size() exactly old_size + wanted; + // that is the room get_elements() is allowed to write into + JSON_ASSERT(result.size() == old_size + wanted); + const std::size_t bytes_read = ia.get_elements(&result[old_size], wanted); + chars_read += bytes_read; + if (JSON_HEDLEY_UNLIKELY(bytes_read < wanted)) { - success = false; - break; + // premature end of input: shrink to what was actually read and + // report the failure at the first missing byte (same position + // accounting as get_to() for partial number reads) + result.resize(old_size + bytes_read); + ++chars_read; + current = char_traits::eof(); + return unexpect_eof(format, context); } - result.push_back(static_cast(current)); + // a full chunk was read; get_elements() never returns more than requested + JSON_ASSERT(bytes_read == wanted); + len = static_cast(len - static_cast(wanted)); } - return success; + return true; } /*! diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index 5a41b0c40..94af90202 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -203,9 +203,51 @@ class iterator_input_adapter out.insert(out.end(), from, to); } - // for general iterators, we cannot really do something better than falling back to processing the range one-by-one + // Copy up to count * sizeof(T) bytes into dest, returning the number of + // bytes actually read. For contiguous iterators (e.g. pointers) this is a + // single std::memcpy; for general iterators we fall back to processing the + // range one-by-one. template std::size_t get_elements(T* dest, std::size_t count = 1) + { + return get_elements_impl(dest, count, std::integral_constant {}); + } + + private: + // whether IteratorType refers to a contiguous range and therefore supports + // a std::memcpy fast path (pointers always do; in C++20 we can also detect + // library iterators such as those of std::vector and std::string) + static constexpr bool iterator_is_contiguous = +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::contiguous_iterator || +#endif + std::is_pointer::value; + + // contiguous fast path: bulk copy the remaining range with std::memcpy + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) + { + const std::size_t wanted = count * sizeof(T); + const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); + const std::size_t copied = (std::min)(wanted, available); + if (JSON_HEDLEY_LIKELY(copied != 0)) + { + // the copy must stay within both buffers: the caller-provided + // destination holds `wanted` bytes and the remaining input range + // holds `available` bytes, and `copied` is the minimum of the two + JSON_ASSERT(copied <= wanted); // does not overrun the destination + JSON_ASSERT(copied <= available); // does not read past the input end + // &*current yields the raw address for both raw pointers and + // non-pointer contiguous iterators (e.g. std::vector's iterator) + std::memcpy(dest, &*current, copied); + std::advance(current, static_cast::difference_type>(copied / sizeof(char_type))); + } + return copied; + } + + // general fallback: copy the range one element at a time + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/) { auto* ptr = reinterpret_cast(dest); for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index) @@ -223,7 +265,6 @@ class iterator_input_adapter return count * sizeof(T); } - private: IteratorType begin; IteratorType current; IteratorType end; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index badcb4f47..d93bcc8bd 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6811,7 +6811,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // array #include // ldexp #include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t #include // snprintf #include // memcpy #include // back_inserter @@ -7035,9 +7035,51 @@ class iterator_input_adapter out.insert(out.end(), from, to); } - // for general iterators, we cannot really do something better than falling back to processing the range one-by-one + // Copy up to count * sizeof(T) bytes into dest, returning the number of + // bytes actually read. For contiguous iterators (e.g. pointers) this is a + // single std::memcpy; for general iterators we fall back to processing the + // range one-by-one. template std::size_t get_elements(T* dest, std::size_t count = 1) + { + return get_elements_impl(dest, count, std::integral_constant {}); + } + + private: + // whether IteratorType refers to a contiguous range and therefore supports + // a std::memcpy fast path (pointers always do; in C++20 we can also detect + // library iterators such as those of std::vector and std::string) + static constexpr bool iterator_is_contiguous = +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + std::contiguous_iterator || +#endif + std::is_pointer::value; + + // contiguous fast path: bulk copy the remaining range with std::memcpy + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) + { + const std::size_t wanted = count * sizeof(T); + const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); + const std::size_t copied = (std::min)(wanted, available); + if (JSON_HEDLEY_LIKELY(copied != 0)) + { + // the copy must stay within both buffers: the caller-provided + // destination holds `wanted` bytes and the remaining input range + // holds `available` bytes, and `copied` is the minimum of the two + JSON_ASSERT(copied <= wanted); // does not overrun the destination + JSON_ASSERT(copied <= available); // does not read past the input end + // &*current yields the raw address for both raw pointers and + // non-pointer contiguous iterators (e.g. std::vector's iterator) + std::memcpy(dest, &*current, copied); + std::advance(current, static_cast::difference_type>(copied / sizeof(char_type))); + } + return copied; + } + + // general fallback: copy the range one element at a time + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/) { auto* ptr = reinterpret_cast(dest); for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index) @@ -7055,7 +7097,6 @@ class iterator_input_adapter return count * sizeof(T); } - private: IteratorType begin; IteratorType current; IteratorType end; @@ -13193,18 +13234,7 @@ class binary_reader const NumberType len, string_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) - { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) - { - success = false; - break; - } - result.push_back(static_cast(current)); - } - return success; + return get_bytes(format, len, "string", result); } /*! @@ -13226,18 +13256,66 @@ class binary_reader const NumberType len, binary_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) + return get_bytes(format, len, "binary", result); + } + + /*! + @brief read @a len bytes from the input into a string or byte container + + @tparam NumberType the type of the length + @tparam ContainerType the destination container (string_t or binary_t) + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[in] context further context information (for diagnostics) + @param[out] result container the bytes are appended to + + @return whether reading completed + + @note We cannot reserve @a len bytes for the result up front, because + @a len may be far larger than the actual input. Instead we read in + bounded chunks, so the peak allocation is capped regardless of the + claimed length while the per-byte loop is replaced by block copies + (a std::memcpy for contiguous inputs). @ref unexpect_eof() still + detects a premature end of input. + */ + template + bool get_bytes(const input_format_t format, + NumberType len, + const char* context, + ContainerType& result) + { + // upper bound on the number of bytes read (and allocated) per chunk + constexpr std::size_t chunk_size = 4096; + + while (len > 0) { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + // number of bytes to read this iteration: min(chunk_size, len), + // computed without truncating chunk_size to a narrow NumberType + const std::size_t wanted = (static_cast(len) < static_cast(chunk_size)) + ? static_cast(len) + : chunk_size; + const std::size_t old_size = result.size(); + result.resize(old_size + wanted); + // resize() is required to make size() exactly old_size + wanted; + // that is the room get_elements() is allowed to write into + JSON_ASSERT(result.size() == old_size + wanted); + const std::size_t bytes_read = ia.get_elements(&result[old_size], wanted); + chars_read += bytes_read; + if (JSON_HEDLEY_UNLIKELY(bytes_read < wanted)) { - success = false; - break; + // premature end of input: shrink to what was actually read and + // report the failure at the first missing byte (same position + // accounting as get_to() for partial number reads) + result.resize(old_size + bytes_read); + ++chars_read; + current = char_traits::eof(); + return unexpect_eof(format, context); } - result.push_back(static_cast(current)); + // a full chunk was read; get_elements() never returns more than requested + JSON_ASSERT(bytes_read == wanted); + len = static_cast(len - static_cast(wanted)); } - return success; + return true; } /*! diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 5da5b0c7a..52c808027 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2778,3 +2778,43 @@ TEST_CASE("Tagged values") CHECK(!jb["binary"].get_binary().has_subtype()); } } + +TEST_CASE("CBOR large strings and binaries (chunked reader)") +{ + // The binary reader reads strings and byte arrays in bounded chunks; make + // sure roundtripping is correct for lengths around and beyond the internal + // chunk size (4096 bytes), for both vector (iterator) and pointer inputs. + for (const std::size_t len : + { + std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096}, + std::size_t{4097}, std::size_t{8192}, std::size_t{100000} + }) + { + CAPTURE(len); + + // text string + const json j_string = std::string(len, 'x'); + const std::vector v_string = json::to_cbor(j_string); + CHECK(json::from_cbor(v_string) == j_string); + // pointer input exercises the std::memcpy fast path + CHECK(json::from_cbor(reinterpret_cast(v_string.data()), + reinterpret_cast(v_string.data()) + v_string.size()) == j_string); + + // byte string + const json j_binary = json::binary(std::vector(len, 0xCD)); + const std::vector v_binary = json::to_cbor(j_binary); + CHECK(json::from_cbor(v_binary) == j_binary); + CHECK(json::from_cbor(reinterpret_cast(v_binary.data()), + reinterpret_cast(v_binary.data()) + v_binary.size()) == j_binary); + + // a truncated payload must still be reported as an error, never crash + // or loop, regardless of the (large) announced length + if (len > 16) + { + std::vector truncated = v_string; + truncated.resize(truncated.size() - 8); + json _; + CHECK_THROWS_AS(_ = json::from_cbor(truncated), json::parse_error); + } + } +}