amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-09-29 11:10:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Validate UTF-8 in CBOR/MessagePack/BSON/UBJSON/BJData text strings on develop

Browse Source 
PR #5531 fixed the UTF-8-validation gap described in #5529, but it was
merged onto the still-unmerged bson-sizes branch rather than develop, so
develop was left with the original bug for all affected formats. A
follow-up comment on #5529 reproduced this on develop and additionally
found that UBJSON (and, by the same code path, BJData) has the identical
gap, undocumented.

Port the same fix directly onto develop: extract the UTF-8 DFA decoder
out of serializer<>::decode() into a shared detail::decode()/is_valid_utf8()
in string_utils.hpp, and call it from binary_reader::get_string() - the
single choke point shared by all five binary readers - so malformed text
strings are rejected at decode time (parse_error.113) instead of only
failing later on dump() (type_error.316). Byte/binary payloads are
unaffected. Add matching decode-time tests for CBOR, MessagePack, BSON,
UBJSON, and BJData, and document the new behavior on all five binary
format pages (the two UBJSON/BJData pages didn't get this note in #5531).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017sdieJCn6BHxzRMaXP49sP
This commit is contained in:
Claude
2026-09-16 05:20:00 +00:00
parent c72f37a40d
commit 5e5a087cb3
15 changed files with 448 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/input/binary_reader.hpp
+19 -1
View File
@@ -32,6 +32,7 @@
#include <nlohmann/detail/meta/is_sax.hpp>
#include <nlohmann/detail/meta/type_traits.hpp>
#include <nlohmann/detail/string_concat.hpp>
#include <nlohmann/detail/string_utils.hpp>
#include <nlohmann/detail/value_t.hpp>
NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -3304,7 +3305,24 @@ class binary_reader
const NumberType len,
string_t& result)
{
return get_bytes(format, len, "string", result);
if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result)))
{
return false;
}
// RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications
// all require text strings to be valid UTF-8; reject anything else
// right here so malformed input is caught at decode time instead of
// only surfacing later as a type_error.316 when the value is dumped
// (which would defeat allow_exceptions=false / strict discarding).
if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result)))
{
return sax->parse_error(chars_read, get_token_string(),
parse_error::create(113, chars_read,
exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr));
}
return true;
}
/*!
include/nlohmann/detail/output/serializer.hpp
+1 -58
View File
@@ -31,6 +31,7 @@
#include <nlohmann/detail/output/binary_writer.hpp>
#include <nlohmann/detail/output/output_adapters.hpp>
#include <nlohmann/detail/string_concat.hpp>
#include <nlohmann/detail/string_utils.hpp>
#include <nlohmann/detail/value_t.hpp>
NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -57,8 +58,6 @@ class serializer
using number_integer_t = typename BasicJsonType::number_integer_t;
using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using binary_char_t = typename BasicJsonType::binary_t::value_type;
static constexpr std::uint8_t UTF8_ACCEPT = 0;
static constexpr std::uint8_t UTF8_REJECT = 1;
public:
/*!
@@ -1598,62 +1597,6 @@ class serializer
}
}
/*!
@brief check whether a string is UTF-8 encoded
The function checks each byte of a string whether it is UTF-8 encoded. The
result of the check is stored in the @a state parameter. The function must
be called initially with state 0 (accept). State 1 means the string must
be rejected, because the current byte is not allowed. If the string is
completely processed, but the state is non-zero, the string ended
prematurely; that is, the last byte indicated more bytes should have
followed.
@param[in,out] state the state of the decoding
@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT)
@param[in] byte next byte to decode
@return new state
@note The function has been edited: a std::array is used.
@copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
*/
static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept
{
static const std::array<std::uint8_t, 400> utf8d =
{
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
}
};
JSON_ASSERT(static_cast<std::size_t>(byte) < utf8d.size());
const std::uint8_t type = utf8d[byte];
codep = (state != UTF8_ACCEPT)
? (byte & 0x3fu) | (codep << 6u)
: (0xFFu >> type) & (byte);
const std::size_t index = 256u + (static_cast<size_t>(state) * 16u) + static_cast<size_t>(type);
JSON_ASSERT(index < utf8d.size());
state = utf8d[index];
return state;
}
/*
* Overload to make the compiler happy while it is instantiating
* dump_integer for number_unsigned_t.
include/nlohmann/detail/string_utils.hpp
+97
View File
@@ -8,10 +8,13 @@
#pragma once
#include <array> // array
#include <cstddef> // size_t
#include <cstdint> // uint8_t, uint32_t
#include <string> // string, to_string
#include <nlohmann/detail/abi_macros.hpp>
#include <nlohmann/detail/macro_scope.hpp>
NLOHMANN_JSON_NAMESPACE_BEGIN
namespace detail
@@ -33,5 +36,99 @@ StringType to_string(std::size_t value)
return result;
}
///////////////////
// UTF-8 decoding //
///////////////////
// UTF-8 decoder states used by decode() below
static constexpr std::uint8_t UTF8_ACCEPT = 0;
static constexpr std::uint8_t UTF8_REJECT = 1;
/*!
@brief process a byte of a UTF-8 sequence
This is a single-byte step of a "shift-based" UTF-8 decoder originally
written by Björn Hoehrmann. See
http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
This decoder is the single source of truth for UTF-8 validation in this
library: it is used both by the serializer (to escape and, in strict mode,
reject ill-formed UTF-8 when dumping a string) and by the binary readers
(to reject ill-formed UTF-8 in CBOR/MessagePack/BSON/UBJSON text strings at
decode time; see @ref is_valid_utf8 below).
@param[in,out] state the current decoder state
@param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT)
@param[in] byte next byte to decode
@return new state
@note Original source: http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
@sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
*/
inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept
{
static const std::array<std::uint8_t, 400> utf8d =
{
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF
8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF
0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF
0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF
0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2
1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4
1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6
1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8
}
};
JSON_ASSERT(static_cast<std::size_t>(byte) < utf8d.size());
const std::uint8_t type = utf8d[byte];
codep = (state != UTF8_ACCEPT)
? (byte & 0x3fu) | (codep << 6u)
: (0xFFu >> type) & (byte);
const std::size_t index = 256u + (static_cast<std::size_t>(state) * 16u) + static_cast<std::size_t>(type);
JSON_ASSERT(index < utf8d.size());
state = utf8d[index];
return state;
}
/*!
@brief check whether a string consists solely of valid UTF-8
Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text
strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the
MessagePack/BSON specifications all require text strings to be UTF-8), so
that malformed input is caught immediately instead of only surfacing later
as a type_error.316 when the resulting value is dumped.
@param[in] s the string to check
@return whether @a s is valid UTF-8
*/
template<typename StringType>
inline bool is_valid_utf8(const StringType& s) noexcept
{
std::uint8_t state = UTF8_ACCEPT;
std::uint32_t codepoint = 0;
for (std::size_t i = 0; i < s.size(); ++i)
{
decode(state, codepoint, static_cast<std::uint8_t>(s[i]));
if (state == UTF8_REJECT)
{
return false;
}
}
return state == UTF8_ACCEPT;
}
} // namespace detail
NLOHMANN_JSON_NAMESPACE_END