mirror of
https://github.com/nlohmann/json.git
synced 2026-11-16 16:43:59 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04fbced73d |
No files matched your search
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
|
||||
|
||||
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
|
||||
|
||||
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
|
||||
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character.
|
||||
|
||||
!!! failure "Example message"
|
||||
|
||||
|
||||
@@ -2279,9 +2279,14 @@ scan_number_done:
|
||||
std::string result;
|
||||
if (chars.size() > 2 * shown)
|
||||
{
|
||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
||||
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||
// move both cuts to character boundaries to keep the message
|
||||
// valid UTF-8; this only shortens the two parts
|
||||
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
|
||||
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||
append_escaped(result, chars.begin(), head_end);
|
||||
result += "...";
|
||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
||||
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -2291,6 +2296,55 @@ scan_number_done:
|
||||
}
|
||||
|
||||
private:
|
||||
/// whether @a c is a UTF-8 continuation byte
|
||||
static bool is_utf8_continuation(const char_type c) noexcept
|
||||
{
|
||||
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||
character that begins there but does not end there
|
||||
|
||||
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||
bytes of a character can be before the cut.
|
||||
*/
|
||||
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
|
||||
{
|
||||
std::size_t lead = n - 1;
|
||||
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||
{
|
||||
--lead;
|
||||
}
|
||||
const auto c = static_cast<unsigned char>(data[lead]);
|
||||
std::size_t length = 1;
|
||||
if (c >= 0xF0u)
|
||||
{
|
||||
length = 4;
|
||||
}
|
||||
else if (c >= 0xE0u)
|
||||
{
|
||||
length = 3;
|
||||
}
|
||||
else if (c >= 0xC0u)
|
||||
{
|
||||
length = 2;
|
||||
}
|
||||
return lead + length > n ? lead : n;
|
||||
}
|
||||
|
||||
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
|
||||
/// belong to a character that begins before it
|
||||
static std::size_t utf8_tail_skip(const char_type* data) noexcept
|
||||
{
|
||||
std::size_t skip = 0;
|
||||
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||
{
|
||||
++skip;
|
||||
}
|
||||
return skip;
|
||||
}
|
||||
|
||||
/// append the characters [first, last) to @a result, escaping control characters
|
||||
template<typename Iterator>
|
||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||
|
||||
@@ -636,7 +636,7 @@ class serializer
|
||||
put_literal("\"subtype\": ");
|
||||
if (binary.has_subtype())
|
||||
{
|
||||
dump_subtype(binary.subtype());
|
||||
dump_integer(binary.subtype());
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -662,7 +662,7 @@ class serializer
|
||||
put_literal("],\"subtype\":");
|
||||
if (binary.has_subtype())
|
||||
{
|
||||
dump_subtype(binary.subtype());
|
||||
dump_integer(binary.subtype());
|
||||
put_char('}');
|
||||
}
|
||||
else
|
||||
@@ -1883,40 +1883,6 @@ class serializer
|
||||
write_buffer_pos = pos;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief dump the subtype of a binary value
|
||||
|
||||
The subtype is a std::uint64_t, whatever the number types are. @ref
|
||||
dump_integer converts its argument to the magnitude type of
|
||||
number_unsigned_t, so it writes the subtype only if that type has at least
|
||||
64 bits; otherwise (it would truncate the subtype) the digits are written
|
||||
one by one.
|
||||
|
||||
@param[in] subtype the subtype to dump
|
||||
*/
|
||||
void dump_subtype(std::uint64_t subtype)
|
||||
{
|
||||
dump_subtype(subtype, std::integral_constant<bool, magnitude_has_at_least_64_bits> {});
|
||||
}
|
||||
|
||||
void dump_subtype(std::uint64_t subtype, std::true_type /*magnitude has at least 64 bits*/)
|
||||
{
|
||||
dump_integer(subtype);
|
||||
}
|
||||
|
||||
void dump_subtype(std::uint64_t subtype, std::false_type /*magnitude has less than 64 bits*/)
|
||||
{
|
||||
std::array<char, 20> digits{{}}; // 2^64 - 1 has 20 digits
|
||||
std::size_t start = digits.size();
|
||||
do
|
||||
{
|
||||
digits[--start] = static_cast<char>('0' + (subtype % 10));
|
||||
subtype /= 10;
|
||||
}
|
||||
while (subtype != 0);
|
||||
put_string(digits, start, digits.size());
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief dump an integer
|
||||
|
||||
@@ -2170,11 +2136,6 @@ class serializer
|
||||
static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
|
||||
&& std::numeric_limits<number_magnitude_t>::digits <= 64;
|
||||
|
||||
/// whether every std::uint64_t fits number_magnitude_t (then dump_integer
|
||||
/// can write a binary subtype)
|
||||
static constexpr bool magnitude_has_at_least_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
|
||||
&& std::numeric_limits<number_magnitude_t>::digits >= 64;
|
||||
|
||||
/// whether @a x fits 64 bits: always if number_magnitude_t has at most
|
||||
/// 64 bits (selected by type, as the comparison would be reported as
|
||||
/// always true), else compared
|
||||
|
||||
Generated
+58
-43
@@ -15782,9 +15782,14 @@ scan_number_done:
|
||||
std::string result;
|
||||
if (chars.size() > 2 * shown)
|
||||
{
|
||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
||||
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||
// move both cuts to character boundaries to keep the message
|
||||
// valid UTF-8; this only shortens the two parts
|
||||
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(utf8_head_cut(chars.data(), shown));
|
||||
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||
append_escaped(result, chars.begin(), head_end);
|
||||
result += "...";
|
||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
||||
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(utf8_tail_skip(&*tail_begin)), chars.end());
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -15794,6 +15799,55 @@ scan_number_done:
|
||||
}
|
||||
|
||||
private:
|
||||
/// whether @a c is a UTF-8 continuation byte
|
||||
static bool is_utf8_continuation(const char_type c) noexcept
|
||||
{
|
||||
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||
character that begins there but does not end there
|
||||
|
||||
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||
bytes of a character can be before the cut.
|
||||
*/
|
||||
static std::size_t utf8_head_cut(const char_type* data, const std::size_t n) noexcept
|
||||
{
|
||||
std::size_t lead = n - 1;
|
||||
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||
{
|
||||
--lead;
|
||||
}
|
||||
const auto c = static_cast<unsigned char>(data[lead]);
|
||||
std::size_t length = 1;
|
||||
if (c >= 0xF0u)
|
||||
{
|
||||
length = 4;
|
||||
}
|
||||
else if (c >= 0xE0u)
|
||||
{
|
||||
length = 3;
|
||||
}
|
||||
else if (c >= 0xC0u)
|
||||
{
|
||||
length = 2;
|
||||
}
|
||||
return lead + length > n ? lead : n;
|
||||
}
|
||||
|
||||
/// the number of UTF-8 continuation bytes (at most 3) at @a data, which
|
||||
/// belong to a character that begins before it
|
||||
static std::size_t utf8_tail_skip(const char_type* data) noexcept
|
||||
{
|
||||
std::size_t skip = 0;
|
||||
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||
{
|
||||
++skip;
|
||||
}
|
||||
return skip;
|
||||
}
|
||||
|
||||
/// append the characters [first, last) to @a result, escaping control characters
|
||||
template<typename Iterator>
|
||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||
@@ -38122,7 +38176,7 @@ class serializer
|
||||
put_literal("\"subtype\": ");
|
||||
if (binary.has_subtype())
|
||||
{
|
||||
dump_subtype(binary.subtype());
|
||||
dump_integer(binary.subtype());
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -38148,7 +38202,7 @@ class serializer
|
||||
put_literal("],\"subtype\":");
|
||||
if (binary.has_subtype())
|
||||
{
|
||||
dump_subtype(binary.subtype());
|
||||
dump_integer(binary.subtype());
|
||||
put_char('}');
|
||||
}
|
||||
else
|
||||
@@ -39369,40 +39423,6 @@ class serializer
|
||||
write_buffer_pos = pos;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief dump the subtype of a binary value
|
||||
|
||||
The subtype is a std::uint64_t, whatever the number types are. @ref
|
||||
dump_integer converts its argument to the magnitude type of
|
||||
number_unsigned_t, so it writes the subtype only if that type has at least
|
||||
64 bits; otherwise (it would truncate the subtype) the digits are written
|
||||
one by one.
|
||||
|
||||
@param[in] subtype the subtype to dump
|
||||
*/
|
||||
void dump_subtype(std::uint64_t subtype)
|
||||
{
|
||||
dump_subtype(subtype, std::integral_constant<bool, magnitude_has_at_least_64_bits> {});
|
||||
}
|
||||
|
||||
void dump_subtype(std::uint64_t subtype, std::true_type /*magnitude has at least 64 bits*/)
|
||||
{
|
||||
dump_integer(subtype);
|
||||
}
|
||||
|
||||
void dump_subtype(std::uint64_t subtype, std::false_type /*magnitude has less than 64 bits*/)
|
||||
{
|
||||
std::array<char, 20> digits{{}}; // 2^64 - 1 has 20 digits
|
||||
std::size_t start = digits.size();
|
||||
do
|
||||
{
|
||||
digits[--start] = static_cast<char>('0' + (subtype % 10));
|
||||
subtype /= 10;
|
||||
}
|
||||
while (subtype != 0);
|
||||
put_string(digits, start, digits.size());
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief dump an integer
|
||||
|
||||
@@ -39656,11 +39676,6 @@ class serializer
|
||||
static constexpr bool magnitude_has_at_most_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
|
||||
&& std::numeric_limits<number_magnitude_t>::digits <= 64;
|
||||
|
||||
/// whether every std::uint64_t fits number_magnitude_t (then dump_integer
|
||||
/// can write a binary subtype)
|
||||
static constexpr bool magnitude_has_at_least_64_bits = std::numeric_limits<number_magnitude_t>::is_specialized
|
||||
&& std::numeric_limits<number_magnitude_t>::digits >= 64;
|
||||
|
||||
/// whether @a x fits 64 bits: always if number_magnitude_t has at most
|
||||
/// 64 bits (selected by type, as the comparison would be reported as
|
||||
/// always true), else compared
|
||||
|
||||
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
|
||||
CHECK(message(text, kind) == "no error");
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("a long string of multi-byte characters")
|
||||
{
|
||||
// the parts of a long token are cut at character boundaries, so the
|
||||
// message of valid UTF-8 input is valid UTF-8, also for wide strings
|
||||
const auto last_read = [](const std::string & m) -> std::string
|
||||
{
|
||||
const auto pos = m.find("last read: '");
|
||||
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
|
||||
};
|
||||
const auto utf8 = [](const char32_t c) -> std::string
|
||||
{
|
||||
std::string result;
|
||||
if (c < 0x800)
|
||||
{
|
||||
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
|
||||
}
|
||||
else if (c < 0x10000)
|
||||
{
|
||||
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
|
||||
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||
}
|
||||
else
|
||||
{
|
||||
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
|
||||
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
|
||||
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||
}
|
||||
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
||||
return result;
|
||||
};
|
||||
// characters of 2, 3, and 4 bytes
|
||||
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
|
||||
for (const char32_t c : characters)
|
||||
{
|
||||
for (std::size_t offset = 0; offset < 4; ++offset)
|
||||
{
|
||||
CAPTURE(static_cast<std::uint32_t>(c))
|
||||
CAPTURE(offset)
|
||||
// an unterminated string: the token is the whole input
|
||||
std::string text = "\"" + std::string(offset, 'a');
|
||||
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
|
||||
while (text.size() < 9000)
|
||||
{
|
||||
text += utf8(c);
|
||||
text32.push_back(c);
|
||||
}
|
||||
std::u16string text16;
|
||||
for (const char32_t c32 : text32)
|
||||
{
|
||||
if (c32 < 0x10000)
|
||||
{
|
||||
text16.push_back(static_cast<char16_t>(c32));
|
||||
}
|
||||
else
|
||||
{
|
||||
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
|
||||
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
|
||||
}
|
||||
}
|
||||
|
||||
const std::string shown = last_read(message(text, 0));
|
||||
const auto dots = shown.find("...");
|
||||
REQUIRE(dots != std::string::npos);
|
||||
// the first and the last at most 4096 bytes, without parts of characters
|
||||
CHECK(dots <= 4096);
|
||||
CHECK(dots > 4092);
|
||||
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
|
||||
const std::size_t tail = shown.size() - dots - 3;
|
||||
CHECK(tail <= 4096);
|
||||
CHECK(tail > 4092);
|
||||
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
|
||||
CHECK_NOTHROW(json(shown).dump());
|
||||
|
||||
for (int kind = 1; kind < 4; ++kind)
|
||||
{
|
||||
CAPTURE(kind)
|
||||
CHECK(last_read(message(text, kind)) == shown);
|
||||
}
|
||||
json _;
|
||||
std::string m32;
|
||||
std::string m16;
|
||||
try
|
||||
{
|
||||
_ = json::parse(text32);
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
m32 = e.what();
|
||||
}
|
||||
try
|
||||
{
|
||||
_ = json::parse(text16);
|
||||
}
|
||||
catch (const json::parse_error& e)
|
||||
{
|
||||
m16 = e.what();
|
||||
}
|
||||
CHECK(last_read(m32) == shown);
|
||||
CHECK(last_read(m16) == shown);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -483,25 +483,3 @@ TEST_CASE("custom number types: 32-bit integers in the binary formats")
|
||||
CHECK(json32::to_msgpack(values) == nlohmann::json::to_msgpack(values64));
|
||||
CHECK(json32::to_ubjson(values) == nlohmann::json::to_ubjson(values64));
|
||||
}
|
||||
|
||||
TEST_CASE("custom number types: binary subtypes wider than the integer types")
|
||||
{
|
||||
// the subtype of a binary value is a std::uint64_t, whatever the number types are
|
||||
using json8 = nlohmann::json::with_integers_t<std::int8_t, std::uint8_t>;
|
||||
using json16 = nlohmann::json::with_integers_t<std::int16_t, std::uint16_t>;
|
||||
using json32 = nlohmann::json::with_integers_t<std::int32_t, std::uint32_t>;
|
||||
const std::uint64_t max_uint64 = (std::numeric_limits<std::uint64_t>::max)();
|
||||
|
||||
CHECK(json8::binary({1}, 300).dump() == "{\"bytes\":[1],\"subtype\":300}");
|
||||
CHECK(json8::binary({200}, 255).dump() == "{\"bytes\":[200],\"subtype\":255}");
|
||||
CHECK(json16::binary({1}, 65543).dump() == "{\"bytes\":[1],\"subtype\":65543}");
|
||||
CHECK(json32::binary({1}, 4294967301u).dump() == "{\"bytes\":[1],\"subtype\":4294967301}");
|
||||
CHECK(json32::binary({1}, max_uint64).dump() == "{\"bytes\":[1],\"subtype\":18446744073709551615}");
|
||||
CHECK(json32::binary({1}, 0).dump() == "{\"bytes\":[1],\"subtype\":0}");
|
||||
CHECK(json32::binary({1}, 4294967301u).dump(1) == "{\n \"bytes\": [1],\n \"subtype\": 4294967301\n}");
|
||||
CHECK(json8::binary({1}).dump() == "{\"bytes\":[1],\"subtype\":null}");
|
||||
|
||||
// unchanged for the default number types
|
||||
CHECK(nlohmann::json::binary({1}, 4294967301u).dump() == "{\"bytes\":[1],\"subtype\":4294967301}");
|
||||
CHECK(nlohmann::json::binary({1}, max_uint64).dump(1) == "{\n \"bytes\": [1],\n \"subtype\": 18446744073709551615\n}");
|
||||
}
|
||||
Reference in new issue
Block a user