mirror of
https://github.com/nlohmann/json.git
synced 2026-11-17 17:01:30 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31b51c95db | ||
|
|
04fbced73d |
No files matched your search
@@ -140,7 +140,7 @@ Exceptions have ids 1xx.
|
|||||||
|
|
||||||
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
|
This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position.
|
||||||
|
|
||||||
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`.
|
The message ends with the last read token (`last read: '...'`), with control characters written as `<U+XXXX>`. A token of more than 8192 bytes is shown by its first and its last 4096 bytes, separated by `...`; each part is shortened by up to three bytes so that it does not end or begin within a UTF-8 character.
|
||||||
|
|
||||||
!!! failure "Example message"
|
!!! failure "Example message"
|
||||||
|
|
||||||
|
|||||||
@@ -48,11 +48,10 @@ namespace detail
|
|||||||
{
|
{
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief implements binary to decimal floating-point conversion: the shortest
|
@brief implements the Grisu2 algorithm for binary to decimal floating-point
|
||||||
digits of Zmij (see zmij.hpp) for IEEE 754 binary64 numbers, and the Grisu2
|
conversion.
|
||||||
algorithm for all other formats.
|
|
||||||
|
|
||||||
The Grisu2 implementation is a slightly modified version of the reference
|
This implementation is a slightly modified version of the reference
|
||||||
implementation which may be obtained from
|
implementation which may be obtained from
|
||||||
http://florian.loitsch.com/publications (bench.tar.gz).
|
http://florian.loitsch.com/publications (bench.tar.gz).
|
||||||
|
|
||||||
|
|||||||
@@ -2279,9 +2279,14 @@ scan_number_done:
|
|||||||
std::string result;
|
std::string result;
|
||||||
if (chars.size() > 2 * shown)
|
if (chars.size() > 2 * shown)
|
||||||
{
|
{
|
||||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||||
|
// move both cuts to character boundaries to keep the message
|
||||||
|
// valid UTF-8; this only shortens the two parts
|
||||||
|
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
|
||||||
|
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||||
|
append_escaped(result, chars.begin(), head_end);
|
||||||
result += "...";
|
result += "...";
|
||||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -2291,6 +2296,55 @@ scan_number_done:
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
/// whether @a c is a UTF-8 continuation byte
|
||||||
|
static bool is_utf8_continuation(const char_type c) noexcept
|
||||||
|
{
|
||||||
|
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||||
|
character that begins there but does not end there
|
||||||
|
|
||||||
|
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||||
|
bytes of a character can be before the cut.
|
||||||
|
*/
|
||||||
|
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
|
||||||
|
{
|
||||||
|
std::size_t lead = n - 1;
|
||||||
|
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||||
|
{
|
||||||
|
--lead;
|
||||||
|
}
|
||||||
|
const auto c = static_cast<unsigned char>(data[lead]);
|
||||||
|
std::size_t length = 1;
|
||||||
|
if (c >= 0xF0u)
|
||||||
|
{
|
||||||
|
length = 4;
|
||||||
|
}
|
||||||
|
else if (c >= 0xE0u)
|
||||||
|
{
|
||||||
|
length = 3;
|
||||||
|
}
|
||||||
|
else if (c >= 0xC0u)
|
||||||
|
{
|
||||||
|
length = 2;
|
||||||
|
}
|
||||||
|
return lead + length > n ? lead : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
|
||||||
|
/// belong to a character that begins before it
|
||||||
|
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
|
||||||
|
{
|
||||||
|
std::size_t skip = 0;
|
||||||
|
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||||
|
{
|
||||||
|
++skip;
|
||||||
|
}
|
||||||
|
return skip;
|
||||||
|
}
|
||||||
|
|
||||||
/// append the characters [first, last) to @a result, escaping control characters
|
/// append the characters [first, last) to @a result, escaping control characters
|
||||||
template<typename Iterator>
|
template<typename Iterator>
|
||||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||||
|
|||||||
@@ -257,7 +257,7 @@ struct ieee_binary_format<53> // binary64
|
|||||||
|
|
||||||
These formats (float, double, and long double where it is binary64, e.g.
|
These formats (float, double, and long double where it is binary64, e.g.
|
||||||
with MSVC or on Apple arm64) are converted by parse_float_native(). The
|
with MSVC or on Apple arm64) are converted by parse_float_native(). The
|
||||||
predicate is the one the serializer uses to choose to_chars() over snprintf.
|
predicate is the one the serializer uses to choose Grisu2.
|
||||||
*/
|
*/
|
||||||
template<typename FloatType>
|
template<typename FloatType>
|
||||||
struct has_native_float_format
|
struct has_native_float_format
|
||||||
|
|||||||
@@ -915,7 +915,7 @@ class binary_writer
|
|||||||
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
|
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// @throw other_error.502 always
|
/// @throw other_error.407 always
|
||||||
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
|
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
|
||||||
static void throw_size_marker_required(const BasicJsonType& j)
|
static void throw_size_marker_required(const BasicJsonType& j)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2033,9 +2033,8 @@ class serializer
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If number_float_t is an IEEE-754 single or double precision number,
|
// If number_float_t is an IEEE-754 single or double precision number,
|
||||||
// use to_chars() to produce short numbers which are guaranteed to
|
// use the Grisu2 algorithm to produce short numbers which are
|
||||||
// round-trip, using strtof and strtod, resp. (the shortest digits of
|
// guaranteed to round-trip, using strtof and strtod, resp.
|
||||||
// Zmij for binary64, the Grisu2 algorithm for binary32).
|
|
||||||
//
|
//
|
||||||
// NB: The test below works if <long double> == <double>.
|
// NB: The test below works if <long double> == <double>.
|
||||||
static constexpr bool is_ieee_single_or_double
|
static constexpr bool is_ieee_single_or_double
|
||||||
|
|||||||
@@ -849,7 +849,7 @@ class basic_json_view
|
|||||||
return !navigation::editable() || m_doc->edits == nullptr;
|
return !navigation::editable() || m_doc->edits == nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// the member of this object with @a key (the first one if the key occurs more than once), or nullptr
|
/// the member of this object with @a key (the last one if the key occurs more than once), or nullptr
|
||||||
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
|
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
|
||||||
{
|
{
|
||||||
return find_key(key, std::integral_constant<bool, is_editable> {});
|
return find_key(key, std::integral_constant<bool, is_editable> {});
|
||||||
|
|||||||
Generated
+63
-11
@@ -11642,7 +11642,7 @@ struct ieee_binary_format<53> // binary64
|
|||||||
|
|
||||||
These formats (float, double, and long double where it is binary64, e.g.
|
These formats (float, double, and long double where it is binary64, e.g.
|
||||||
with MSVC or on Apple arm64) are converted by parse_float_native(). The
|
with MSVC or on Apple arm64) are converted by parse_float_native(). The
|
||||||
predicate is the one the serializer uses to choose to_chars() over snprintf.
|
predicate is the one the serializer uses to choose Grisu2.
|
||||||
*/
|
*/
|
||||||
template<typename FloatType>
|
template<typename FloatType>
|
||||||
struct has_native_float_format
|
struct has_native_float_format
|
||||||
@@ -15782,9 +15782,14 @@ scan_number_done:
|
|||||||
std::string result;
|
std::string result;
|
||||||
if (chars.size() > 2 * shown)
|
if (chars.size() > 2 * shown)
|
||||||
{
|
{
|
||||||
append_escaped(result, chars.begin(), chars.begin() + static_cast<std::ptrdiff_t>(shown));
|
// the lexer reads bytes (wide characters arrive as UTF-8), so
|
||||||
|
// move both cuts to character boundaries to keep the message
|
||||||
|
// valid UTF-8; this only shortens the two parts
|
||||||
|
const auto head_end = chars.begin() + static_cast<std::ptrdiff_t>(length_without_partial_utf8_at_end(chars.data(), shown));
|
||||||
|
const auto tail_begin = chars.end() - static_cast<std::ptrdiff_t>(shown);
|
||||||
|
append_escaped(result, chars.begin(), head_end);
|
||||||
result += "...";
|
result += "...";
|
||||||
append_escaped(result, chars.end() - static_cast<std::ptrdiff_t>(shown), chars.end());
|
append_escaped(result, tail_begin + static_cast<std::ptrdiff_t>(partial_utf8_length_at_start(&*tail_begin)), chars.end());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -15794,6 +15799,55 @@ scan_number_done:
|
|||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
/// whether @a c is a UTF-8 continuation byte
|
||||||
|
static bool is_utf8_continuation(const char_type c) noexcept
|
||||||
|
{
|
||||||
|
return (static_cast<unsigned char>(c) & 0xC0u) == 0x80u;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*!
|
||||||
|
@brief the length of the first @a n characters at @a data without a UTF-8
|
||||||
|
character that begins there but does not end there
|
||||||
|
|
||||||
|
Looks only at the last 4 of these characters: in valid UTF-8, at most 3
|
||||||
|
bytes of a character can be before the cut.
|
||||||
|
*/
|
||||||
|
static std::size_t length_without_partial_utf8_at_end(const char_type* data, const std::size_t n) noexcept
|
||||||
|
{
|
||||||
|
std::size_t lead = n - 1;
|
||||||
|
while (lead > n - 4 && is_utf8_continuation(data[lead]))
|
||||||
|
{
|
||||||
|
--lead;
|
||||||
|
}
|
||||||
|
const auto c = static_cast<unsigned char>(data[lead]);
|
||||||
|
std::size_t length = 1;
|
||||||
|
if (c >= 0xF0u)
|
||||||
|
{
|
||||||
|
length = 4;
|
||||||
|
}
|
||||||
|
else if (c >= 0xE0u)
|
||||||
|
{
|
||||||
|
length = 3;
|
||||||
|
}
|
||||||
|
else if (c >= 0xC0u)
|
||||||
|
{
|
||||||
|
length = 2;
|
||||||
|
}
|
||||||
|
return lead + length > n ? lead : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// the number of UTF-8 continuation bytes (at most 3) at the start of @a data, which
|
||||||
|
/// belong to a character that begins before it
|
||||||
|
static std::size_t partial_utf8_length_at_start(const char_type* data) noexcept
|
||||||
|
{
|
||||||
|
std::size_t skip = 0;
|
||||||
|
while (skip < 3 && is_utf8_continuation(data[skip]))
|
||||||
|
{
|
||||||
|
++skip;
|
||||||
|
}
|
||||||
|
return skip;
|
||||||
|
}
|
||||||
|
|
||||||
/// append the characters [first, last) to @a result, escaping control characters
|
/// append the characters [first, last) to @a result, escaping control characters
|
||||||
template<typename Iterator>
|
template<typename Iterator>
|
||||||
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
static void append_escaped(std::string& result, Iterator first, Iterator last)
|
||||||
@@ -31147,7 +31201,7 @@ class binary_writer
|
|||||||
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
|
JSON_THROW(out_of_range::create(exception_id::subtype_out_of_range, concat("subtype ", std::to_string(subtype), " is too large for ", what, " (max 255)"), &j));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// @throw other_error.502 always
|
/// @throw other_error.407 always
|
||||||
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
|
JSON_HEDLEY_NO_RETURN JSON_HEDLEY_NEVER_INLINE
|
||||||
static void throw_size_marker_required(const BasicJsonType& j)
|
static void throw_size_marker_required(const BasicJsonType& j)
|
||||||
{
|
{
|
||||||
@@ -36006,11 +36060,10 @@ namespace detail
|
|||||||
{
|
{
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
@brief implements binary to decimal floating-point conversion: the shortest
|
@brief implements the Grisu2 algorithm for binary to decimal floating-point
|
||||||
digits of Zmij (see zmij.hpp) for IEEE 754 binary64 numbers, and the Grisu2
|
conversion.
|
||||||
algorithm for all other formats.
|
|
||||||
|
|
||||||
The Grisu2 implementation is a slightly modified version of the reference
|
This implementation is a slightly modified version of the reference
|
||||||
implementation which may be obtained from
|
implementation which may be obtained from
|
||||||
http://florian.loitsch.com/publications (bench.tar.gz).
|
http://florian.loitsch.com/publications (bench.tar.gz).
|
||||||
|
|
||||||
@@ -39520,9 +39573,8 @@ class serializer
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If number_float_t is an IEEE-754 single or double precision number,
|
// If number_float_t is an IEEE-754 single or double precision number,
|
||||||
// use to_chars() to produce short numbers which are guaranteed to
|
// use the Grisu2 algorithm to produce short numbers which are
|
||||||
// round-trip, using strtof and strtod, resp. (the shortest digits of
|
// guaranteed to round-trip, using strtof and strtod, resp.
|
||||||
// Zmij for binary64, the Grisu2 algorithm for binary32).
|
|
||||||
//
|
//
|
||||||
// NB: The test below works if <long double> == <double>.
|
// NB: The test below works if <long double> == <double>.
|
||||||
static constexpr bool is_ieee_single_or_double
|
static constexpr bool is_ieee_single_or_double
|
||||||
|
|||||||
Generated
+1
-1
@@ -11016,7 +11016,7 @@ class basic_json_view
|
|||||||
return !navigation::editable() || m_doc->edits == nullptr;
|
return !navigation::editable() || m_doc->edits == nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// the member of this object with @a key (the first one if the key occurs more than once), or nullptr
|
/// the member of this object with @a key (the last one if the key occurs more than once), or nullptr
|
||||||
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
|
NLOHMANN_VIEW_ALWAYS_INLINE const node* find_key(string_view_t key) const
|
||||||
{
|
{
|
||||||
return find_key(key, std::integral_constant<bool, is_editable> {});
|
return find_key(key, std::integral_constant<bool, is_editable> {});
|
||||||
|
|||||||
@@ -3380,5 +3380,108 @@ TEST_CASE("parse error message of a long token")
|
|||||||
CHECK(message(text, kind) == "no error");
|
CHECK(message(text, kind) == "no error");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SECTION("a long string of multi-byte characters")
|
||||||
|
{
|
||||||
|
// the parts of a long token are cut at character boundaries, so the
|
||||||
|
// message of valid UTF-8 input is valid UTF-8, also for wide strings
|
||||||
|
const auto last_read = [](const std::string & m) -> std::string
|
||||||
|
{
|
||||||
|
const auto pos = m.find("last read: '");
|
||||||
|
return pos == std::string::npos ? std::string() : m.substr(pos + 12, m.size() - pos - 13);
|
||||||
|
};
|
||||||
|
const auto utf8 = [](const char32_t c) -> std::string
|
||||||
|
{
|
||||||
|
std::string result;
|
||||||
|
if (c < 0x800)
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xC0 | (c >> 6)));
|
||||||
|
}
|
||||||
|
else if (c < 0x10000)
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xE0 | (c >> 12)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
result.push_back(static_cast<char>(0xF0 | (c >> 18)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 12) & 0x3F)));
|
||||||
|
result.push_back(static_cast<char>(0x80 | ((c >> 6) & 0x3F)));
|
||||||
|
}
|
||||||
|
result.push_back(static_cast<char>(0x80 | (c & 0x3F)));
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
// characters of 2, 3, and 4 bytes
|
||||||
|
const std::vector<char32_t> characters = {0xE9, 0x20AC, 0x1F600};
|
||||||
|
for (const char32_t c : characters)
|
||||||
|
{
|
||||||
|
for (std::size_t offset = 0; offset < 4; ++offset)
|
||||||
|
{
|
||||||
|
CAPTURE(static_cast<std::uint32_t>(c))
|
||||||
|
CAPTURE(offset)
|
||||||
|
// an unterminated string: the token is the whole input
|
||||||
|
std::string text = "\"" + std::string(offset, 'a');
|
||||||
|
std::u32string text32 = U"\"" + std::u32string(offset, U'a');
|
||||||
|
while (text.size() < 9000)
|
||||||
|
{
|
||||||
|
text += utf8(c);
|
||||||
|
text32.push_back(c);
|
||||||
|
}
|
||||||
|
std::u16string text16;
|
||||||
|
for (const char32_t c32 : text32)
|
||||||
|
{
|
||||||
|
if (c32 < 0x10000)
|
||||||
|
{
|
||||||
|
text16.push_back(static_cast<char16_t>(c32));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
text16.push_back(static_cast<char16_t>(0xD800 + ((c32 - 0x10000) >> 10)));
|
||||||
|
text16.push_back(static_cast<char16_t>(0xDC00 + ((c32 - 0x10000) & 0x3FF)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const std::string shown = last_read(message(text, 0));
|
||||||
|
const auto dots = shown.find("...");
|
||||||
|
REQUIRE(dots != std::string::npos);
|
||||||
|
// the first and the last at most 4096 bytes, without parts of characters
|
||||||
|
CHECK(dots <= 4096);
|
||||||
|
CHECK(dots > 4092);
|
||||||
|
CHECK(text.compare(0, dots, shown, 0, dots) == 0);
|
||||||
|
const std::size_t tail = shown.size() - dots - 3;
|
||||||
|
CHECK(tail <= 4096);
|
||||||
|
CHECK(tail > 4092);
|
||||||
|
CHECK(text.compare(text.size() - tail, tail, shown, dots + 3, tail) == 0);
|
||||||
|
CHECK_NOTHROW(json(shown).dump());
|
||||||
|
|
||||||
|
for (int kind = 1; kind < 4; ++kind)
|
||||||
|
{
|
||||||
|
CAPTURE(kind)
|
||||||
|
CHECK(last_read(message(text, kind)) == shown);
|
||||||
|
}
|
||||||
|
json _;
|
||||||
|
std::string m32;
|
||||||
|
std::string m16;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_ = json::parse(text32);
|
||||||
|
}
|
||||||
|
catch (const json::parse_error& e)
|
||||||
|
{
|
||||||
|
m32 = e.what();
|
||||||
|
}
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_ = json::parse(text16);
|
||||||
|
}
|
||||||
|
catch (const json::parse_error& e)
|
||||||
|
{
|
||||||
|
m16 = e.what();
|
||||||
|
}
|
||||||
|
CHECK(last_read(m32) == shown);
|
||||||
|
CHECK(last_read(m16) == shown);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
Reference in new issue
Block a user