diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 033cfebd7..f57868b9f 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -12,7 +12,7 @@ #include // array #include // ldexp #include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t #include // snprintf #include // memcpy #include // back_inserter @@ -2908,18 +2908,7 @@ class binary_reader const NumberType len, string_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) - { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) - { - success = false; - break; - } - result.push_back(static_cast(current)); - } - return success; + return get_bytes(format, len, "string", result); } /*! @@ -2941,18 +2930,60 @@ class binary_reader const NumberType len, binary_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) + return get_bytes(format, len, "binary", result); + } + + /*! + @brief read @a len bytes from the input into a string or byte container + + @tparam NumberType the type of the length + @tparam ContainerType the destination container (string_t or binary_t) + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[in] context further context information (for diagnostics) + @param[out] result container the bytes are appended to + + @return whether reading completed + + @note We cannot reserve @a len bytes for the result up front, because + @a len may be far larger than the actual input. Instead we read in + bounded chunks, so the peak allocation is capped regardless of the + claimed length while the per-byte loop is replaced by block copies + (a std::memcpy for contiguous inputs). @ref unexpect_eof() still + detects a premature end of input. + */ + template + bool get_bytes(const input_format_t format, + NumberType len, + const char* context, + ContainerType& result) + { + // upper bound on the number of bytes read (and allocated) per chunk + constexpr std::size_t chunk_size = 4096; + + while (len > 0) { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + // number of bytes to read this iteration: min(chunk_size, len), + // computed without truncating chunk_size to a narrow NumberType + const std::size_t wanted = (static_cast(len) < static_cast(chunk_size)) + ? static_cast(len) + : chunk_size; + const std::size_t old_size = result.size(); + result.resize(old_size + wanted); + const std::size_t read = ia.get_elements(&result[old_size], wanted); + chars_read += read; + if (JSON_HEDLEY_UNLIKELY(read < wanted)) { - success = false; - break; + // premature end of input: shrink to what was actually read and + // report the failure at the first missing byte + result.resize(old_size + read); + ++chars_read; + current = char_traits::eof(); + return unexpect_eof(format, context); } - result.push_back(static_cast(current)); + len -= static_cast(wanted); } - return success; + return true; } /*! diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index be58b1110..161adf4da 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -177,9 +177,44 @@ class iterator_input_adapter return char_traits::eof(); } - // for general iterators, we cannot really do something better than falling back to processing the range one-by-one + // Copy up to count * sizeof(T) bytes into dest, returning the number of + // bytes actually read. For contiguous iterators (e.g. pointers) this is a + // single std::memcpy; for general iterators we fall back to processing the + // range one-by-one. template std::size_t get_elements(T* dest, std::size_t count = 1) + { + return get_elements_impl(dest, count, std::integral_constant {}); + } + + private: + // whether IteratorType refers to a contiguous range and therefore supports + // a std::memcpy fast path (pointers always do; in C++20 we can also detect + // library iterators such as those of std::vector and std::string) +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + static constexpr bool iterator_is_contiguous = std::is_pointer::value || std::contiguous_iterator; +#else + static constexpr bool iterator_is_contiguous = std::is_pointer::value; +#endif + + // contiguous fast path: bulk copy the remaining range with std::memcpy + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) + { + const std::size_t wanted = count * sizeof(T); + const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); + const std::size_t copied = (std::min)(wanted, available); + if (JSON_HEDLEY_LIKELY(copied != 0)) + { + std::memcpy(dest, &(*current), copied); + std::advance(current, static_cast::difference_type>(copied / sizeof(char_type))); + } + return copied; + } + + // general fallback: copy the range one element at a time + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/) { auto* ptr = reinterpret_cast(dest); for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index) @@ -197,7 +232,6 @@ class iterator_input_adapter return count * sizeof(T); } - private: IteratorType current; IteratorType end; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1a2b3d357..75d9a1aec 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6811,7 +6811,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // array #include // ldexp #include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t #include // snprintf #include // memcpy #include // back_inserter @@ -7009,9 +7009,44 @@ class iterator_input_adapter return char_traits::eof(); } - // for general iterators, we cannot really do something better than falling back to processing the range one-by-one + // Copy up to count * sizeof(T) bytes into dest, returning the number of + // bytes actually read. For contiguous iterators (e.g. pointers) this is a + // single std::memcpy; for general iterators we fall back to processing the + // range one-by-one. template std::size_t get_elements(T* dest, std::size_t count = 1) + { + return get_elements_impl(dest, count, std::integral_constant {}); + } + + private: + // whether IteratorType refers to a contiguous range and therefore supports + // a std::memcpy fast path (pointers always do; in C++20 we can also detect + // library iterators such as those of std::vector and std::string) +#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20) + static constexpr bool iterator_is_contiguous = std::is_pointer::value || std::contiguous_iterator; +#else + static constexpr bool iterator_is_contiguous = std::is_pointer::value; +#endif + + // contiguous fast path: bulk copy the remaining range with std::memcpy + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/) + { + const std::size_t wanted = count * sizeof(T); + const std::size_t available = static_cast(std::distance(current, end)) * sizeof(char_type); + const std::size_t copied = (std::min)(wanted, available); + if (JSON_HEDLEY_LIKELY(copied != 0)) + { + std::memcpy(dest, &(*current), copied); + std::advance(current, static_cast::difference_type>(copied / sizeof(char_type))); + } + return copied; + } + + // general fallback: copy the range one element at a time + template + std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/) { auto* ptr = reinterpret_cast(dest); for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index) @@ -7029,7 +7064,6 @@ class iterator_input_adapter return count * sizeof(T); } - private: IteratorType current; IteratorType end; @@ -13073,18 +13107,7 @@ class binary_reader const NumberType len, string_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) - { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string"))) - { - success = false; - break; - } - result.push_back(static_cast(current)); - } - return success; + return get_bytes(format, len, "string", result); } /*! @@ -13106,18 +13129,60 @@ class binary_reader const NumberType len, binary_t& result) { - bool success = true; - for (NumberType i = 0; i < len; i++) + return get_bytes(format, len, "binary", result); + } + + /*! + @brief read @a len bytes from the input into a string or byte container + + @tparam NumberType the type of the length + @tparam ContainerType the destination container (string_t or binary_t) + @param[in] format the current format (for diagnostics) + @param[in] len number of bytes to read + @param[in] context further context information (for diagnostics) + @param[out] result container the bytes are appended to + + @return whether reading completed + + @note We cannot reserve @a len bytes for the result up front, because + @a len may be far larger than the actual input. Instead we read in + bounded chunks, so the peak allocation is capped regardless of the + claimed length while the per-byte loop is replaced by block copies + (a std::memcpy for contiguous inputs). @ref unexpect_eof() still + detects a premature end of input. + */ + template + bool get_bytes(const input_format_t format, + NumberType len, + const char* context, + ContainerType& result) + { + // upper bound on the number of bytes read (and allocated) per chunk + constexpr std::size_t chunk_size = 4096; + + while (len > 0) { - get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary"))) + // number of bytes to read this iteration: min(chunk_size, len), + // computed without truncating chunk_size to a narrow NumberType + const std::size_t wanted = (static_cast(len) < static_cast(chunk_size)) + ? static_cast(len) + : chunk_size; + const std::size_t old_size = result.size(); + result.resize(old_size + wanted); + const std::size_t read = ia.get_elements(&result[old_size], wanted); + chars_read += read; + if (JSON_HEDLEY_UNLIKELY(read < wanted)) { - success = false; - break; + // premature end of input: shrink to what was actually read and + // report the failure at the first missing byte + result.resize(old_size + read); + ++chars_read; + current = char_traits::eof(); + return unexpect_eof(format, context); } - result.push_back(static_cast(current)); + len -= static_cast(wanted); } - return success; + return true; } /*! diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 5da5b0c7a..8b1ae46ab 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2778,3 +2778,43 @@ TEST_CASE("Tagged values") CHECK(!jb["binary"].get_binary().has_subtype()); } } + +TEST_CASE("CBOR large strings and binaries (chunked reader)") +{ + // The binary reader reads strings and byte arrays in bounded chunks; make + // sure roundtripping is correct for lengths around and beyond the internal + // chunk size (4096 bytes), for both vector (iterator) and pointer inputs. + for (const std::size_t len : + { + std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096}, + std::size_t{4097}, std::size_t{8192}, std::size_t{100000} + }) + { + CAPTURE(len); + + // text string + const json j_string = std::string(len, 'x'); + const std::vector v_string = json::to_cbor(j_string); + CHECK(json::from_cbor(v_string) == j_string); + // pointer input exercises the std::memcpy fast path + CHECK(json::from_cbor(reinterpret_cast(v_string.data()), + reinterpret_cast(v_string.data()) + v_string.size()) == j_string); + + // byte string + const json j_binary = json::binary(std::vector(len, 0xCD)); + const std::vector v_binary = json::to_cbor(j_binary); + CHECK(json::from_cbor(v_binary) == j_binary); + CHECK(json::from_cbor(reinterpret_cast(v_binary.data()), + reinterpret_cast(v_binary.data()) + v_binary.size()) == j_binary); + + // a truncated payload must still be reported as an error, never crash + // or loop, regardless of the (large) announced length + if (len > 16) + { + std::vector truncated = v_string; + truncated.resize(truncated.size() - 8); + json _; + CHECK_THROWS_AS(_ = json::from_cbor(truncated), json::parse_error); + } + } +}