Compare commits

..
Author SHA1 Message Date
Niels Lohmann 59ccf04a39 Fix the build of develop: tests from #5997 and #5991
- unit-msgpack-cpp17: assign the result of json::parse in
  CHECK_THROWS_WITH_AS (GCC -Werror=unused-result)
- unit-json_view: failing_allocator::allocate is [[noreturn]]
  (clang -Wmissing-noreturn)
- unit-class_lexer: build the quoted string once
  (clang-tidy performance-inefficient-string-concatenation)
- unit-allocator: sweep_allocator provides pointer, reference and size
  typedefs, which GCC 4.8's std::map takes from the allocator
- unit-allocator: skip the "sax_parse into an existing value" sweep with
  iterator debugging, like the parser callback sweep: MSVC's std::vector
  default constructor is noexcept but allocates its debug proxy through
  the allocator, so a failing allocation in create<array_t>() terminates

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-11-02 22:31:50 +01:00
8 changed files with 122 additions and 57 deletions

No files matched your search

+41 -16
View File
@@ -344,8 +344,8 @@ class binary_reader
nested document could declare a length that disagrees with where its
terminator actually falls and quietly hand the bytes in between to the
enclosing document. A well-formed document is at least 5 bytes (the prefix
plus the terminator); open_bson_document() rejects a smaller size before
it is stored, so @a document_size is never negative here.
plus the terminator); the equality also rejects those impossible sizes,
since at least 5 bytes are always consumed.
@param[in] document_start value of chars_read before the size prefix
@param[in] document_size the declared document size
@@ -353,7 +353,7 @@ class binary_reader
*/
bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
{
if (JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(document_size) != chars_read - document_start))
if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
{
return bson_document_size_error(document_size, chars_read - document_start);
}
@@ -3091,6 +3091,8 @@ class binary_reader
if (size_and_type.first != npos)
{
if (size_and_type.second != 0)
{
if (size_and_type.second != 'N')
{
for (std::size_t i = 0; i < size_and_type.first; ++i)
{
@@ -3101,6 +3103,7 @@ class binary_reader
dim.push_back(dimlen);
}
}
}
else
{
for (std::size_t i = 0; i < size_and_type.first; ++i)
@@ -3300,9 +3303,11 @@ class binary_reader
// follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
if (ndarray_dtype != 0)
{
// every optimized type marker is_valid_optimized_type() accepts for BJData has a name
const char* type_name = bjd_type_name(ndarray_dtype);
JSON_ASSERT(type_name != nullptr);
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
return unexpected_byte("invalid byte", "type");
}
string_t type_key = "_ArrayType_";
string_t type = type_name; // sax->string() takes a reference
@@ -3385,6 +3390,12 @@ class binary_reader
if (current == '$')
{
result.second = get(); // must not ignore 'N', because 'N' maybe the type
if (input_format == input_format_t::bjdata
&& JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second)))
{
return optimized_type_error();
}
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type")))
{
return false;
@@ -3412,7 +3423,7 @@ class binary_reader
// an ndarray was read here only if the flag flipped; when it was
// seeded true, get_ubjson_size_value() already rejected the nested
// dimension vector
if (is_ndarray && !inside_ndarray)
if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
{
result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters
}
@@ -3422,7 +3433,7 @@ class binary_reader
if (current == '#')
{
const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
if (is_ndarray && !inside_ndarray)
if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
{
return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size");
}
@@ -3596,6 +3607,7 @@ class binary_reader
JSON_HEDLEY_NEVER_INLINE
bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/)
{
const bool bjdata = input_format == input_format_t::bjdata;
switch (type)
{
case 'U':
@@ -3609,11 +3621,11 @@ class binary_reader
case 'L':
return parse_ubjson_typed_run<std::int64_t>(remaining);
case 'u':
return parse_ubjson_typed_run<std::uint16_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint16_t>(remaining);
case 'm':
return parse_ubjson_typed_run<std::uint32_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint32_t>(remaining);
case 'M':
return parse_ubjson_typed_run<std::uint64_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint64_t>(remaining);
case 'd':
return parse_ubjson_typed_run<float>(remaining);
case 'D':
@@ -3694,7 +3706,7 @@ class binary_reader
// if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):
// {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]}
if ((size_and_type.second & (1 << 8)) != 0)
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
@@ -3753,7 +3765,7 @@ class binary_reader
}
// If BJData type marker is 'B' decode as binary
if (size_and_type.first != npos && size_and_type.second == 'B')
if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B')
{
binary_t result;
return get_binary(size_and_type.first, result) && sax->binary(result);
@@ -3802,7 +3814,7 @@ class binary_reader
}
// do not accept ND-array size in objects in BJData
if ((size_and_type.second & (1 << 8)) != 0)
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object");
}
@@ -5083,9 +5095,8 @@ class binary_reader
JSON_PRIVATE_UNLESS_TESTED:
/*!
@brief check whether @a marker is a type marker that may follow '$'
UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C.
The markers BJData excludes (F H N S T Z [ {) are the ones
binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync.
UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C
(BJData's excluded markers are rejected separately).
*/
bool is_valid_optimized_type(const char_int_type marker) const noexcept
{
@@ -5120,6 +5131,20 @@ class binary_reader
}
}
/*!
@brief whether @a marker is excluded from BJData's optimized ND-array types
@return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{'
Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker
"is_bjdata_excluded_type_marker()`, which encodes the same list the other
way; keep the two in sync.
*/
static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept
{
return marker == '[' || marker == '{' || marker == 'S' || marker == 'H'
|| marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z';
}
/*!
@brief look up the ND-array element type name for a BJData dtype marker
@return the type name ("uint8", "int8", ...), or nullptr if @a marker does
@@ -3126,19 +3126,22 @@ class binary_writer
// negative fixnum
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
{
// int 8
oa.write_character(to_char_type(0xD0));
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
{
// int 16
oa.write_character(to_char_type(0xD1));
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
{
// int 32
oa.write_character(to_char_type(0xD2));
@@ -4054,7 +4057,7 @@ class binary_writer
Containers, strings, high-precision numbers, booleans and null cannot be
declared as the single type of an optimized container in BJData; such a
container is written unoptimized. The reader rejects them with the same
list (binary_reader::is_valid_optimized_type()).
list (binary_reader::is_bjd_excluded_optimized_type()).
*/
static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept
{
+48 -20
View File
@@ -18359,8 +18359,8 @@ class binary_reader
nested document could declare a length that disagrees with where its
terminator actually falls and quietly hand the bytes in between to the
enclosing document. A well-formed document is at least 5 bytes (the prefix
plus the terminator); open_bson_document() rejects a smaller size before
it is stored, so @a document_size is never negative here.
plus the terminator); the equality also rejects those impossible sizes,
since at least 5 bytes are always consumed.
@param[in] document_start value of chars_read before the size prefix
@param[in] document_size the declared document size
@@ -18368,7 +18368,7 @@ class binary_reader
*/
bool check_bson_document_size(const std::size_t document_start, const std::int32_t document_size)
{
if (JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(document_size) != chars_read - document_start))
if (JSON_HEDLEY_UNLIKELY(document_size < 0 || static_cast<std::size_t>(document_size) != chars_read - document_start))
{
return bson_document_size_error(document_size, chars_read - document_start);
}
@@ -21106,6 +21106,8 @@ class binary_reader
if (size_and_type.first != npos)
{
if (size_and_type.second != 0)
{
if (size_and_type.second != 'N')
{
for (std::size_t i = 0; i < size_and_type.first; ++i)
{
@@ -21116,6 +21118,7 @@ class binary_reader
dim.push_back(dimlen);
}
}
}
else
{
for (std::size_t i = 0; i < size_and_type.first; ++i)
@@ -21315,9 +21318,11 @@ class binary_reader
// follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
if (ndarray_dtype != 0)
{
// every optimized type marker is_valid_optimized_type() accepts for BJData has a name
const char* type_name = bjd_type_name(ndarray_dtype);
JSON_ASSERT(type_name != nullptr);
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
return unexpected_byte("invalid byte", "type");
}
string_t type_key = "_ArrayType_";
string_t type = type_name; // sax->string() takes a reference
@@ -21400,6 +21405,12 @@ class binary_reader
if (current == '$')
{
result.second = get(); // must not ignore 'N', because 'N' maybe the type
if (input_format == input_format_t::bjdata
&& JSON_HEDLEY_UNLIKELY(is_bjd_excluded_optimized_type(result.second)))
{
return optimized_type_error();
}
if (JSON_HEDLEY_UNLIKELY(!unexpect_eof("type")))
{
return false;
@@ -21427,7 +21438,7 @@ class binary_reader
// an ndarray was read here only if the flag flipped; when it was
// seeded true, get_ubjson_size_value() already rejected the nested
// dimension vector
if (is_ndarray && !inside_ndarray)
if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
{
result.second |= (1 << 8); // use bit 8 to indicate ndarray, all UBJSON and BJData markers should be ASCII letters
}
@@ -21437,7 +21448,7 @@ class binary_reader
if (current == '#')
{
const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
if (is_ndarray && !inside_ndarray)
if (input_format == input_format_t::bjdata && is_ndarray && !inside_ndarray)
{
return last_byte_error(exception_id::unexpected_byte, "ndarray requires both type and size", "size");
}
@@ -21611,6 +21622,7 @@ class binary_reader
JSON_HEDLEY_NEVER_INLINE
bool parse_ubjson_typed_run(std::size_t& remaining, const char_int_type type, std::true_type /*bulk*/)
{
const bool bjdata = input_format == input_format_t::bjdata;
switch (type)
{
case 'U':
@@ -21624,11 +21636,11 @@ class binary_reader
case 'L':
return parse_ubjson_typed_run<std::int64_t>(remaining);
case 'u':
return parse_ubjson_typed_run<std::uint16_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint16_t>(remaining);
case 'm':
return parse_ubjson_typed_run<std::uint32_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint32_t>(remaining);
case 'M':
return parse_ubjson_typed_run<std::uint64_t>(remaining);
return !bjdata || parse_ubjson_typed_run<std::uint64_t>(remaining);
case 'd':
return parse_ubjson_typed_run<float>(remaining);
case 'D':
@@ -21709,7 +21721,7 @@ class binary_reader
// if bit-8 of size_and_type.second is set to 1, encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata):
// {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]}
if ((size_and_type.second & (1 << 8)) != 0)
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
@@ -21768,7 +21780,7 @@ class binary_reader
}
// If BJData type marker is 'B' decode as binary
if (size_and_type.first != npos && size_and_type.second == 'B')
if (input_format == input_format_t::bjdata && size_and_type.first != npos && size_and_type.second == 'B')
{
binary_t result;
return get_binary(size_and_type.first, result) && sax->binary(result);
@@ -21817,7 +21829,7 @@ class binary_reader
}
// do not accept ND-array size in objects in BJData
if ((size_and_type.second & (1 << 8)) != 0)
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
return last_byte_error(exception_id::unexpected_byte, "BJData object does not support ND-array size in optimized format", "object");
}
@@ -23098,9 +23110,8 @@ class binary_reader
JSON_PRIVATE_UNLESS_TESTED:
/*!
@brief check whether @a marker is a type marker that may follow '$'
UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C.
The markers BJData excludes (F H N S T Z [ {) are the ones
binary_writer::is_bjdata_excluded_type_marker() lists; keep the two in sync.
UBJSON: Z N T F i U I l L d D H C S [ {; BJData: the numeric markers and C
(BJData's excluded markers are rejected separately).
*/
bool is_valid_optimized_type(const char_int_type marker) const noexcept
{
@@ -23135,6 +23146,20 @@ class binary_reader
}
}
/*!
@brief whether @a marker is excluded from BJData's optimized ND-array types
@return whether @a marker is one of 'F', 'H', 'N', 'S', 'T', 'Z', '[', '{'
Mirrors binary_writer's @ref binary_writer::is_bjdata_excluded_type_marker
"is_bjdata_excluded_type_marker()`, which encodes the same list the other
way; keep the two in sync.
*/
static constexpr bool is_bjd_excluded_optimized_type(const char_int_type marker) noexcept
{
return marker == '[' || marker == '{' || marker == 'S' || marker == 'H'
|| marker == 'T' || marker == 'F' || marker == 'N' || marker == 'Z';
}
/*!
@brief look up the ND-array element type name for a BJData dtype marker
@return the type name ("uint8", "int8", ...), or nullptr if @a marker does
@@ -31085,19 +31110,22 @@ class binary_writer
// negative fixnum
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int8_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int8_t>::max)())
{
// int 8
oa.write_character(to_char_type(0xD0));
write_number(static_cast<std::int8_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int16_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int16_t>::max)())
{
// int 16
oa.write_character(to_char_type(0xD1));
write_number(static_cast<std::int16_t>(j.m_data.m_value.number_integer));
}
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)())
else if (j.m_data.m_value.number_integer >= (std::numeric_limits<std::int32_t>::min)() &&
j.m_data.m_value.number_integer <= (std::numeric_limits<std::int32_t>::max)())
{
// int 32
oa.write_character(to_char_type(0xD2));
@@ -32013,7 +32041,7 @@ class binary_writer
Containers, strings, high-precision numbers, booleans and null cannot be
declared as the single type of an optimized container in BJData; such a
container is written unoptimized. The reader rejects them with the same
list (binary_reader::is_valid_optimized_type()).
list (binary_reader::is_bjd_excluded_optimized_type()).
*/
static constexpr bool is_bjdata_excluded_type_marker(const CharType marker) noexcept
{
+12 -3
View File
@@ -1038,6 +1038,14 @@ template<class T>
struct sweep_allocator
{
using value_type = T;
// GCC 4.8's std::map takes these from the allocator instead of from
// std::allocator_traits
using pointer = T*;
using const_pointer = const T*;
using reference = T&;
using const_reference = const T&;
using size_type = std::size_t;
using difference_type = std::ptrdiff_t;
sweep_allocator() = default;
template<class U> sweep_allocator(const sweep_allocator<U>& /*unused*/) noexcept {}
@@ -1155,8 +1163,9 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
// With iterator debugging (MSVC STL debug builds, also used by clang-cl),
// containers allocate a debug proxy through the allocator inside their
// noexcept move constructors, so failing that allocation terminates the
// program instead of throwing std::bad_alloc. Nothing to check there.
// noexcept default and move constructors, so failing that allocation
// terminates the program instead of throwing std::bad_alloc. Nothing to
// check there.
#if !(defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
SECTION("parser callback")
{
@@ -1178,7 +1187,6 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
return sweep_json::parse(text, drop_d);
});
}
#endif
SECTION("sax_parse into an existing value")
{
@@ -1192,5 +1200,6 @@ TEST_CASE("parse() survives a failing allocation at every point (#5985)")
return value;
});
}
#endif
}
#endif
+7 -9
View File
@@ -78,25 +78,23 @@ TEST_CASE("BJData")
{
SECTION("binary_reader BJData lookup tables")
{
// bjd_type_name() is static; is_valid_optimized_type() needs a BJData reader
// both lookups are static member functions
std::vector<std::uint8_t> const data;
using reader_t = nlohmann::detail::binary_reader<json, decltype(nlohmann::detail::input_adapter(data))>;
// the markers BJData excludes as optimized types must match
// binary_writer's is_bjdata_excluded_type_marker(), which encodes the
// same 8 markers
reader_t reader(nlohmann::detail::input_adapter(data), json::input_format_t::bjdata);
// the excluded optimized-type markers must match binary_writer's
// is_bjdata_excluded_type_marker(), which encodes the same 8 markers
for (const char marker :
{'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z', 'x'
{'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'
})
{
CHECK(!reader.is_valid_optimized_type(static_cast<unsigned char>(marker)));
CHECK(reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
}
for (const char marker :
{'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'h'
{'U', 'i', 'u', 'I', 'm', 'l', 'M', 'L', 'd', 'D', 'C', 'B', 'x'
})
{
CHECK(reader.is_valid_optimized_type(static_cast<unsigned char>(marker)));
CHECK(!reader_t::is_bjd_excluded_optimized_type(static_cast<unsigned char>(marker)));
}
// every dtype marker must round-trip to its ND-array type name
+4 -3
View File
@@ -2222,9 +2222,10 @@ TEST_CASE("serialization of short strings")
{
CAPTURE(s)
const json j = s;
CHECK(j.dump() == "\"" + s + "\"");
CHECK(j.dump(-1, ' ', true) == "\"" + s + "\"");
CHECK(json({{s, s}}).dump() == "{\"" + s + "\":\"" + s + "\"}");
const std::string quoted = "\"" + s + "\""; // NOLINT(performance-inefficient-string-concatenation)
CHECK(j.dump() == quoted);
CHECK(j.dump(-1, ' ', true) == quoted);
CHECK(json({{s, s}}).dump() == "{" + quoted + ":" + quoted + "}"); // NOLINT(performance-inefficient-string-concatenation)
}
// a character that needs an escape
+1 -1
View File
@@ -111,7 +111,7 @@ struct failing_allocator
template<typename U>
explicit failing_allocator(const failing_allocator<U>& /*other*/) noexcept {}
T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
[[noreturn]] T* allocate(std::size_t /*n*/) // NOLINT(readability-convert-member-functions-to-static)
{
throw std::bad_alloc();
}
+2 -1
View File
@@ -122,7 +122,8 @@ TEST_CASE("JSON text with std::byte")
SECTION("invalid text")
{
CHECK_FALSE(json::accept(to_bytes("[1,\n2,")));
CHECK_THROWS_WITH_AS(json::parse(to_bytes("[1,\n2,")),
json _;
CHECK_THROWS_WITH_AS(_ = json::parse(to_bytes("[1,\n2,")),
"[json.exception.parse_error.101] parse error at line 2, column 3: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error&);
}
}