amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-07-07 11:05:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Read binary strings/blobs in bulk chunks with a memcpy fast path (#5233)

Browse Source
This commit is contained in:
Niels Lohmann
2026-07-05 19:26:02 +02:00
committed by GitHub
parent eed1587000
commit 83c87cb9e0
4 changed files with 242 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/input/binary_reader.hpp
+58 -21
View File
@@ -12,7 +12,7 @@
#include <array> // array
#include <cmath> // ldexp
#include <cstddef> // size_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t
#include <cstdio> // snprintf
#include <cstring> // memcpy
#include <iterator> // back_inserter
@@ -2908,18 +2908,7 @@ class binary_reader
const NumberType len,
string_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string")))
{
success = false;
break;
}
result.push_back(static_cast<typename string_t::value_type>(current));
}
return success;
return get_bytes(format, len, "string", result);
}
/*!
@@ -2941,18 +2930,66 @@ class binary_reader
const NumberType len,
binary_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
return get_bytes(format, len, "binary", result);
}
/*!
@brief read @a len bytes from the input into a string or byte container
@tparam NumberType the type of the length
@tparam ContainerType the destination container (string_t or binary_t)
@param[in] format the current format (for diagnostics)
@param[in] len number of bytes to read
@param[in] context further context information (for diagnostics)
@param[out] result container the bytes are appended to
@return whether reading completed
@note We cannot reserve @a len bytes for the result up front, because
@a len may be far larger than the actual input. Instead we read in
bounded chunks, so the peak allocation is capped regardless of the
claimed length while the per-byte loop is replaced by block copies
(a std::memcpy for contiguous inputs). @ref unexpect_eof() still
detects a premature end of input.
*/
template<typename NumberType, typename ContainerType>
bool get_bytes(const input_format_t format,
NumberType len,
const char* context,
ContainerType& result)
{
// upper bound on the number of bytes read (and allocated) per chunk
constexpr std::size_t chunk_size = 4096;
while (len > 0)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary")))
// number of bytes to read this iteration: min(chunk_size, len),
// computed without truncating chunk_size to a narrow NumberType
const std::size_t wanted = (static_cast<std::uintmax_t>(len) < static_cast<std::uintmax_t>(chunk_size))
? static_cast<std::size_t>(len)
: chunk_size;
const std::size_t old_size = result.size();
result.resize(old_size + wanted);
// resize() is required to make size() exactly old_size + wanted;
// that is the room get_elements() is allowed to write into
JSON_ASSERT(result.size() == old_size + wanted);
const std::size_t bytes_read = ia.get_elements(&result[old_size], wanted);
chars_read += bytes_read;
if (JSON_HEDLEY_UNLIKELY(bytes_read < wanted))
{
success = false;
break;
// premature end of input: shrink to what was actually read and
// report the failure at the first missing byte (same position
// accounting as get_to() for partial number reads)
result.resize(old_size + bytes_read);
++chars_read;
current = char_traits<char_type>::eof();
return unexpect_eof(format, context);
}
result.push_back(static_cast<typename binary_t::value_type>(current));
// a full chunk was read; get_elements() never returns more than requested
JSON_ASSERT(bytes_read == wanted);
len = static_cast<NumberType>(len - static_cast<NumberType>(wanted));
}
return success;
return true;
}
/*!
include/nlohmann/detail/input/input_adapters.hpp
+43 -2
View File
@@ -203,9 +203,51 @@ class iterator_input_adapter
out.insert(out.end(), from, to);
}
// for general iterators, we cannot really do something better than falling back to processing the range one-by-one
// Copy up to count * sizeof(T) bytes into dest, returning the number of
// bytes actually read. For contiguous iterators (e.g. pointers) this is a
// single std::memcpy; for general iterators we fall back to processing the
// range one-by-one.
template<class T>
std::size_t get_elements(T* dest, std::size_t count = 1)
{
return get_elements_impl(dest, count, std::integral_constant<bool, iterator_is_contiguous> {});
}
private:
// whether IteratorType refers to a contiguous range and therefore supports
// a std::memcpy fast path (pointers always do; in C++20 we can also detect
// library iterators such as those of std::vector and std::string)
static constexpr bool iterator_is_contiguous =
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
std::contiguous_iterator<IteratorType> ||
#endif
std::is_pointer<IteratorType>::value;
// contiguous fast path: bulk copy the remaining range with std::memcpy
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
{
const std::size_t wanted = count * sizeof(T);
const std::size_t available = static_cast<std::size_t>(std::distance(current, end)) * sizeof(char_type);
const std::size_t copied = (std::min)(wanted, available);
if (JSON_HEDLEY_LIKELY(copied != 0))
{
// the copy must stay within both buffers: the caller-provided
// destination holds `wanted` bytes and the remaining input range
// holds `available` bytes, and `copied` is the minimum of the two
JSON_ASSERT(copied <= wanted); // does not overrun the destination
JSON_ASSERT(copied <= available); // does not read past the input end
// &*current yields the raw address for both raw pointers and
// non-pointer contiguous iterators (e.g. std::vector's iterator)
std::memcpy(dest, &*current, copied);
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(copied / sizeof(char_type)));
}
return copied;
}
// general fallback: copy the range one element at a time
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/)
{
auto* ptr = reinterpret_cast<char*>(dest);
for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index)
@@ -223,7 +265,6 @@ class iterator_input_adapter
return count * sizeof(T);
}
private:
IteratorType begin;
IteratorType current;
IteratorType end;
single_include/nlohmann/json.hpp
+101 -23
View File
@@ -6811,7 +6811,7 @@ NLOHMANN_JSON_NAMESPACE_END
#include <array> // array
#include <cmath> // ldexp
#include <cstddef> // size_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t
#include <cstdio> // snprintf
#include <cstring> // memcpy
#include <iterator> // back_inserter
@@ -7035,9 +7035,51 @@ class iterator_input_adapter
out.insert(out.end(), from, to);
}
// for general iterators, we cannot really do something better than falling back to processing the range one-by-one
// Copy up to count * sizeof(T) bytes into dest, returning the number of
// bytes actually read. For contiguous iterators (e.g. pointers) this is a
// single std::memcpy; for general iterators we fall back to processing the
// range one-by-one.
template<class T>
std::size_t get_elements(T* dest, std::size_t count = 1)
{
return get_elements_impl(dest, count, std::integral_constant<bool, iterator_is_contiguous> {});
}
private:
// whether IteratorType refers to a contiguous range and therefore supports
// a std::memcpy fast path (pointers always do; in C++20 we can also detect
// library iterators such as those of std::vector and std::string)
static constexpr bool iterator_is_contiguous =
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
std::contiguous_iterator<IteratorType> ||
#endif
std::is_pointer<IteratorType>::value;
// contiguous fast path: bulk copy the remaining range with std::memcpy
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
{
const std::size_t wanted = count * sizeof(T);
const std::size_t available = static_cast<std::size_t>(std::distance(current, end)) * sizeof(char_type);
const std::size_t copied = (std::min)(wanted, available);
if (JSON_HEDLEY_LIKELY(copied != 0))
{
// the copy must stay within both buffers: the caller-provided
// destination holds `wanted` bytes and the remaining input range
// holds `available` bytes, and `copied` is the minimum of the two
JSON_ASSERT(copied <= wanted); // does not overrun the destination
JSON_ASSERT(copied <= available); // does not read past the input end
// &*current yields the raw address for both raw pointers and
// non-pointer contiguous iterators (e.g. std::vector's iterator)
std::memcpy(dest, &*current, copied);
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(copied / sizeof(char_type)));
}
return copied;
}
// general fallback: copy the range one element at a time
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/)
{
auto* ptr = reinterpret_cast<char*>(dest);
for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index)
@@ -7055,7 +7097,6 @@ class iterator_input_adapter
return count * sizeof(T);
}
private:
IteratorType begin;
IteratorType current;
IteratorType end;
@@ -13193,18 +13234,7 @@ class binary_reader
const NumberType len,
string_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string")))
{
success = false;
break;
}
result.push_back(static_cast<typename string_t::value_type>(current));
}
return success;
return get_bytes(format, len, "string", result);
}
/*!
@@ -13226,18 +13256,66 @@ class binary_reader
const NumberType len,
binary_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
return get_bytes(format, len, "binary", result);
}
/*!
@brief read @a len bytes from the input into a string or byte container
@tparam NumberType the type of the length
@tparam ContainerType the destination container (string_t or binary_t)
@param[in] format the current format (for diagnostics)
@param[in] len number of bytes to read
@param[in] context further context information (for diagnostics)
@param[out] result container the bytes are appended to
@return whether reading completed
@note We cannot reserve @a len bytes for the result up front, because
@a len may be far larger than the actual input. Instead we read in
bounded chunks, so the peak allocation is capped regardless of the
claimed length while the per-byte loop is replaced by block copies
(a std::memcpy for contiguous inputs). @ref unexpect_eof() still
detects a premature end of input.
*/
template<typename NumberType, typename ContainerType>
bool get_bytes(const input_format_t format,
NumberType len,
const char* context,
ContainerType& result)
{
// upper bound on the number of bytes read (and allocated) per chunk
constexpr std::size_t chunk_size = 4096;
while (len > 0)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary")))
// number of bytes to read this iteration: min(chunk_size, len),
// computed without truncating chunk_size to a narrow NumberType
const std::size_t wanted = (static_cast<std::uintmax_t>(len) < static_cast<std::uintmax_t>(chunk_size))
? static_cast<std::size_t>(len)
: chunk_size;
const std::size_t old_size = result.size();
result.resize(old_size + wanted);
// resize() is required to make size() exactly old_size + wanted;
// that is the room get_elements() is allowed to write into
JSON_ASSERT(result.size() == old_size + wanted);
const std::size_t bytes_read = ia.get_elements(&result[old_size], wanted);
chars_read += bytes_read;
if (JSON_HEDLEY_UNLIKELY(bytes_read < wanted))
{
success = false;
break;
// premature end of input: shrink to what was actually read and
// report the failure at the first missing byte (same position
// accounting as get_to() for partial number reads)
result.resize(old_size + bytes_read);
++chars_read;
current = char_traits<char_type>::eof();
return unexpect_eof(format, context);
}
result.push_back(static_cast<typename binary_t::value_type>(current));
// a full chunk was read; get_elements() never returns more than requested
JSON_ASSERT(bytes_read == wanted);
len = static_cast<NumberType>(len - static_cast<NumberType>(wanted));
}
return success;
return true;
}
/*!
tests/src/unit-cbor.cpp
+40
View File
@@ -2778,3 +2778,43 @@ TEST_CASE("Tagged values")
CHECK(!jb["binary"].get_binary().has_subtype());
}
}
TEST_CASE("CBOR large strings and binaries (chunked reader)")
{
// The binary reader reads strings and byte arrays in bounded chunks; make
// sure roundtripping is correct for lengths around and beyond the internal
// chunk size (4096 bytes), for both vector (iterator) and pointer inputs.
for (const std::size_t len :
{
std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
})
{
CAPTURE(len);
// text string
const json j_string = std::string(len, 'x');
const std::vector<std::uint8_t> v_string = json::to_cbor(j_string);
CHECK(json::from_cbor(v_string) == j_string);
// pointer input exercises the std::memcpy fast path
CHECK(json::from_cbor(reinterpret_cast<const char*>(v_string.data()),
reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
// byte string
const json j_binary = json::binary(std::vector<std::uint8_t>(len, 0xCD));
const std::vector<std::uint8_t> v_binary = json::to_cbor(j_binary);
CHECK(json::from_cbor(v_binary) == j_binary);
CHECK(json::from_cbor(reinterpret_cast<const char*>(v_binary.data()),
reinterpret_cast<const char*>(v_binary.data()) + v_binary.size()) == j_binary);
// a truncated payload must still be reported as an error, never crash
// or loop, regardless of the (large) announced length
if (len > 16)
{
std::vector<std::uint8_t> truncated = v_string;
truncated.resize(truncated.size() - 8);
json _;
CHECK_THROWS_AS(_ = json::from_cbor(truncated), json::parse_error);
}
}
}