amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-01 05:57:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Read binary strings/blobs in bulk chunks with a memcpy fast path

Browse Source 
The binary reader read CBOR/MessagePack/BSON/UBJSON strings and byte
arrays one byte at a time via get()/push_back(), and the iterator input
adapter's get_elements() fallback was itself a per-byte loop, so even
contiguous inputs never benefited from a block copy.

Two changes:

1. iterator_input_adapter::get_elements() gains a contiguous fast path
   that copies the whole requested range with std::memcpy. Contiguity is
   detected via std::is_pointer (all standards) and, in C++20,
   std::contiguous_iterator (so std::vector/std::string iterators also
   qualify). Non-contiguous iterators keep the element-by-element loop.

2. get_string()/get_binary() now share get_bytes(), which reads into the
   result in bounded chunks through get_elements() instead of byte by
   byte. Capping the chunk size preserves the deliberate "do not
   reserve(len) for an untrusted length" DoS protection while turning the
   inner loop into block copies. The min(chunk_size, len) computation is
   width-safe so narrow length types (e.g. MessagePack ext-8's uint8_t)
   cannot truncate chunk_size to zero.

Microbenchmark (2 MiB string + 2 MiB blob + 2000x1 KiB strings, Apple
clang, -O2):

  C++20  from_cbor(vector)   20.1 ms -> 1.0 ms  (~20x)
         from_cbor(pointer)  18.9 ms -> 1.0 ms  (~19x)
  C++17  from_cbor(pointer)  18.7 ms -> 1.0 ms  (~19x, memcpy)
         from_cbor(vector)   18.7 ms -> 4.0 ms  (~4.6x, tight loop)

Behavior is unchanged: truncated input still throws parse_error.110 at
the same byte offset, and all binary-format unit tests pass.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Niels Lohmann
2026-07-04 00:00:12 +02:00
co-authored by Claude Opus 4.8
parent c363dc3e4d
commit 9eee7a7b9b
4 changed files with 216 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/input/binary_reader.hpp
+52 -21
View File
@@ -12,7 +12,7 @@
#include <array> // array
#include <cmath> // ldexp
#include <cstddef> // size_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t
#include <cstdio> // snprintf
#include <cstring> // memcpy
#include <iterator> // back_inserter
@@ -2908,18 +2908,7 @@ class binary_reader
const NumberType len,
string_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string")))
{
success = false;
break;
}
result.push_back(static_cast<typename string_t::value_type>(current));
}
return success;
return get_bytes(format, len, "string", result);
}
/*!
@@ -2941,18 +2930,60 @@ class binary_reader
const NumberType len,
binary_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
return get_bytes(format, len, "binary", result);
}
/*!
@brief read @a len bytes from the input into a string or byte container
@tparam NumberType the type of the length
@tparam ContainerType the destination container (string_t or binary_t)
@param[in] format the current format (for diagnostics)
@param[in] len number of bytes to read
@param[in] context further context information (for diagnostics)
@param[out] result container the bytes are appended to
@return whether reading completed
@note We cannot reserve @a len bytes for the result up front, because
@a len may be far larger than the actual input. Instead we read in
bounded chunks, so the peak allocation is capped regardless of the
claimed length while the per-byte loop is replaced by block copies
(a std::memcpy for contiguous inputs). @ref unexpect_eof() still
detects a premature end of input.
*/
template<typename NumberType, typename ContainerType>
bool get_bytes(const input_format_t format,
NumberType len,
const char* context,
ContainerType& result)
{
// upper bound on the number of bytes read (and allocated) per chunk
constexpr std::size_t chunk_size = 4096;
while (len > 0)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary")))
// number of bytes to read this iteration: min(chunk_size, len),
// computed without truncating chunk_size to a narrow NumberType
const std::size_t wanted = (static_cast<std::uintmax_t>(len) < static_cast<std::uintmax_t>(chunk_size))
? static_cast<std::size_t>(len)
: chunk_size;
const std::size_t old_size = result.size();
result.resize(old_size + wanted);
const std::size_t read = ia.get_elements(&result[old_size], wanted);
chars_read += read;
if (JSON_HEDLEY_UNLIKELY(read < wanted))
{
success = false;
break;
// premature end of input: shrink to what was actually read and
// report the failure at the first missing byte
result.resize(old_size + read);
++chars_read;
current = char_traits<char_type>::eof();
return unexpect_eof(format, context);
}
result.push_back(static_cast<typename binary_t::value_type>(current));
len -= static_cast<NumberType>(wanted);
}
return success;
return true;
}
/*!
include/nlohmann/detail/input/input_adapters.hpp
+36 -2
View File
@@ -177,9 +177,44 @@ class iterator_input_adapter
return char_traits<char_type>::eof();
}
// for general iterators, we cannot really do something better than falling back to processing the range one-by-one
// Copy up to count * sizeof(T) bytes into dest, returning the number of
// bytes actually read. For contiguous iterators (e.g. pointers) this is a
// single std::memcpy; for general iterators we fall back to processing the
// range one-by-one.
template<class T>
std::size_t get_elements(T* dest, std::size_t count = 1)
{
return get_elements_impl(dest, count, std::integral_constant<bool, iterator_is_contiguous> {});
}
private:
// whether IteratorType refers to a contiguous range and therefore supports
// a std::memcpy fast path (pointers always do; in C++20 we can also detect
// library iterators such as those of std::vector and std::string)
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
static constexpr bool iterator_is_contiguous = std::is_pointer<IteratorType>::value || std::contiguous_iterator<IteratorType>;
#else
static constexpr bool iterator_is_contiguous = std::is_pointer<IteratorType>::value;
#endif
// contiguous fast path: bulk copy the remaining range with std::memcpy
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
{
const std::size_t wanted = count * sizeof(T);
const std::size_t available = static_cast<std::size_t>(std::distance(current, end)) * sizeof(char_type);
const std::size_t copied = (std::min)(wanted, available);
if (JSON_HEDLEY_LIKELY(copied != 0))
{
std::memcpy(dest, &(*current), copied);
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(copied / sizeof(char_type)));
}
return copied;
}
// general fallback: copy the range one element at a time
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/)
{
auto* ptr = reinterpret_cast<char*>(dest);
for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index)
@@ -197,7 +232,6 @@ class iterator_input_adapter
return count * sizeof(T);
}
private:
IteratorType current;
IteratorType end;
single_include/nlohmann/json.hpp
+88 -23
View File
@@ -6811,7 +6811,7 @@ NLOHMANN_JSON_NAMESPACE_END
#include <array> // array
#include <cmath> // ldexp
#include <cstddef> // size_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t
#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t, uintmax_t
#include <cstdio> // snprintf
#include <cstring> // memcpy
#include <iterator> // back_inserter
@@ -7009,9 +7009,44 @@ class iterator_input_adapter
return char_traits<char_type>::eof();
}
// for general iterators, we cannot really do something better than falling back to processing the range one-by-one
// Copy up to count * sizeof(T) bytes into dest, returning the number of
// bytes actually read. For contiguous iterators (e.g. pointers) this is a
// single std::memcpy; for general iterators we fall back to processing the
// range one-by-one.
template<class T>
std::size_t get_elements(T* dest, std::size_t count = 1)
{
return get_elements_impl(dest, count, std::integral_constant<bool, iterator_is_contiguous> {});
}
private:
// whether IteratorType refers to a contiguous range and therefore supports
// a std::memcpy fast path (pointers always do; in C++20 we can also detect
// library iterators such as those of std::vector and std::string)
#if defined(__cpp_lib_concepts) && defined(JSON_HAS_CPP_20)
static constexpr bool iterator_is_contiguous = std::is_pointer<IteratorType>::value || std::contiguous_iterator<IteratorType>;
#else
static constexpr bool iterator_is_contiguous = std::is_pointer<IteratorType>::value;
#endif
// contiguous fast path: bulk copy the remaining range with std::memcpy
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::true_type /*contiguous*/)
{
const std::size_t wanted = count * sizeof(T);
const std::size_t available = static_cast<std::size_t>(std::distance(current, end)) * sizeof(char_type);
const std::size_t copied = (std::min)(wanted, available);
if (JSON_HEDLEY_LIKELY(copied != 0))
{
std::memcpy(dest, &(*current), copied);
std::advance(current, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(copied / sizeof(char_type)));
}
return copied;
}
// general fallback: copy the range one element at a time
template<class T>
std::size_t get_elements_impl(T* dest, std::size_t count, std::false_type /*contiguous*/)
{
auto* ptr = reinterpret_cast<char*>(dest);
for (std::size_t read_index = 0; read_index < count * sizeof(T); ++read_index)
@@ -7029,7 +7064,6 @@ class iterator_input_adapter
return count * sizeof(T);
}
private:
IteratorType current;
IteratorType end;
@@ -13073,18 +13107,7 @@ class binary_reader
const NumberType len,
string_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "string")))
{
success = false;
break;
}
result.push_back(static_cast<typename string_t::value_type>(current));
}
return success;
return get_bytes(format, len, "string", result);
}
/*!
@@ -13106,18 +13129,60 @@ class binary_reader
const NumberType len,
binary_t& result)
{
bool success = true;
for (NumberType i = 0; i < len; i++)
return get_bytes(format, len, "binary", result);
}
/*!
@brief read @a len bytes from the input into a string or byte container
@tparam NumberType the type of the length
@tparam ContainerType the destination container (string_t or binary_t)
@param[in] format the current format (for diagnostics)
@param[in] len number of bytes to read
@param[in] context further context information (for diagnostics)
@param[out] result container the bytes are appended to
@return whether reading completed
@note We cannot reserve @a len bytes for the result up front, because
@a len may be far larger than the actual input. Instead we read in
bounded chunks, so the peak allocation is capped regardless of the
claimed length while the per-byte loop is replaced by block copies
(a std::memcpy for contiguous inputs). @ref unexpect_eof() still
detects a premature end of input.
*/
template<typename NumberType, typename ContainerType>
bool get_bytes(const input_format_t format,
NumberType len,
const char* context,
ContainerType& result)
{
// upper bound on the number of bytes read (and allocated) per chunk
constexpr std::size_t chunk_size = 4096;
while (len > 0)
{
get();
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(format, "binary")))
// number of bytes to read this iteration: min(chunk_size, len),
// computed without truncating chunk_size to a narrow NumberType
const std::size_t wanted = (static_cast<std::uintmax_t>(len) < static_cast<std::uintmax_t>(chunk_size))
? static_cast<std::size_t>(len)
: chunk_size;
const std::size_t old_size = result.size();
result.resize(old_size + wanted);
const std::size_t read = ia.get_elements(&result[old_size], wanted);
chars_read += read;
if (JSON_HEDLEY_UNLIKELY(read < wanted))
{
success = false;
break;
// premature end of input: shrink to what was actually read and
// report the failure at the first missing byte
result.resize(old_size + read);
++chars_read;
current = char_traits<char_type>::eof();
return unexpect_eof(format, context);
}
result.push_back(static_cast<typename binary_t::value_type>(current));
len -= static_cast<NumberType>(wanted);
}
return success;
return true;
}
/*!
tests/src/unit-cbor.cpp
+40
View File
@@ -2778,3 +2778,43 @@ TEST_CASE("Tagged values")
CHECK(!jb["binary"].get_binary().has_subtype());
}
}
TEST_CASE("CBOR large strings and binaries (chunked reader)")
{
// The binary reader reads strings and byte arrays in bounded chunks; make
// sure roundtripping is correct for lengths around and beyond the internal
// chunk size (4096 bytes), for both vector (iterator) and pointer inputs.
for (const std::size_t len :
{
std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
})
{
CAPTURE(len);
// text string
const json j_string = std::string(len, 'x');
const std::vector<std::uint8_t> v_string = json::to_cbor(j_string);
CHECK(json::from_cbor(v_string) == j_string);
// pointer input exercises the std::memcpy fast path
CHECK(json::from_cbor(reinterpret_cast<const char*>(v_string.data()),
reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
// byte string
const json j_binary = json::binary(std::vector<std::uint8_t>(len, 0xCD));
const std::vector<std::uint8_t> v_binary = json::to_cbor(j_binary);
CHECK(json::from_cbor(v_binary) == j_binary);
CHECK(json::from_cbor(reinterpret_cast<const char*>(v_binary.data()),
reinterpret_cast<const char*>(v_binary.data()) + v_binary.size()) == j_binary);
// a truncated payload must still be reported as an error, never crash
// or loop, regardless of the (large) announced length
if (len > 16)
{
std::vector<std::uint8_t> truncated = v_string;
truncated.resize(truncated.size() - 8);
json _;
CHECK_THROWS_AS(_ = json::from_cbor(truncated), json::parse_error);
}
}
}