mirror of
https://github.com/nlohmann/json.git
synced 2026-09-27 18:20:32 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
509c07041f |
@@ -206,6 +206,7 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
|
||||
: ia(std::move(adapter))
|
||||
, ignore_comments(ignore_comments_)
|
||||
, decimal_point_char(static_cast<char_int_type>(get_decimal_point()))
|
||||
, discard_number_values(discard_number_values_)
|
||||
{}
|
||||
|
||||
@@ -221,7 +222,8 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
// locales
|
||||
/////////////////////
|
||||
|
||||
/// return the decimal point of the current locale
|
||||
/// return the locale-dependent decimal point
|
||||
JSON_HEDLEY_PURE
|
||||
static char get_decimal_point() noexcept
|
||||
{
|
||||
const auto* loc = localeconv();
|
||||
@@ -1090,10 +1092,9 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
token_type::value_float if number could be successfully scanned,
|
||||
token_type::parse_error otherwise
|
||||
|
||||
@note The scanner is independent of the current locale: token_buffer
|
||||
always holds `.`. Only the std::strtod fallback of convert_number()
|
||||
depends on the locale, and it looks up the decimal point right
|
||||
before converting (see convert_float_locale_aware()).
|
||||
@note The scanner is independent of the current locale. Internally, the
|
||||
locale's decimal point is used instead of `.` to work with the
|
||||
locale-dependent converters.
|
||||
*/
|
||||
token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated.
|
||||
{
|
||||
@@ -1182,7 +1183,7 @@ scan_number_zero:
|
||||
{
|
||||
case '.':
|
||||
{
|
||||
add(current);
|
||||
add(decimal_point_char);
|
||||
decimal_point_position = token_buffer.size() - 1;
|
||||
goto scan_number_decimal1;
|
||||
}
|
||||
@@ -1219,7 +1220,7 @@ scan_number_any1:
|
||||
|
||||
case '.':
|
||||
{
|
||||
add(current);
|
||||
add(decimal_point_char);
|
||||
decimal_point_position = token_buffer.size() - 1;
|
||||
goto scan_number_decimal1;
|
||||
}
|
||||
@@ -1461,9 +1462,9 @@ scan_number_done:
|
||||
|
||||
// Only a number below 1 can carry further insignificant zeros, and only
|
||||
// while the count stays at the limit does removing them change the
|
||||
// answer - so this loop is skipped for all but a few tokens. The
|
||||
// fraction is located through decimal_point_position rather than by
|
||||
// searching '.'.
|
||||
// answer - so this loop is skipped for all but a few tokens. Note
|
||||
// token_buffer holds the locale's decimal point, so the fraction is
|
||||
// located through decimal_point_position rather than by searching '.'.
|
||||
if (lead_zero != 0)
|
||||
{
|
||||
JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit
|
||||
@@ -1481,8 +1482,8 @@ scan_number_done:
|
||||
@brief convert the number text in token_buffer to its value and token type
|
||||
|
||||
The digit sequence in token_buffer has already been validated (by the
|
||||
scan_number() state machine or by the contiguous fast path) and holds '.'
|
||||
as decimal point, independent of the locale. Integers are parsed first and fall
|
||||
scan_number() state machine or by the contiguous fast path) and holds the
|
||||
locale decimal point in place of '.'. Integers are parsed first and fall
|
||||
back to floating point on overflow. This is shared so both scanners produce
|
||||
identical results.
|
||||
|
||||
@@ -1562,7 +1563,7 @@ scan_number_done:
|
||||
// integer conversion above overflowed. Prefer std::from_chars
|
||||
// (Eisel-Lemire, locale-independent, correctly rounded) when available;
|
||||
// otherwise the exact Clinger fast path (double only); otherwise the
|
||||
// locale-aware strtof/strtod/strtold.
|
||||
// locale-aware strtof/strtod.
|
||||
if (parse_float_from_chars(num_begin, num_end, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
@@ -1571,75 +1572,26 @@ scan_number_done:
|
||||
// extra pass over the token's bytes, which otherwise shows up on
|
||||
// high-precision inputs such as canada.json
|
||||
if (mantissa_fits_clinger(mantissa_end)
|
||||
&& parse_float_fast(num_begin, num_end, value_float))
|
||||
&& parse_float_fast(num_begin, num_end, decimal_point_char, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
convert_float_locale_aware();
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief convert the float in token_buffer with strtof/strtod/strtold
|
||||
|
||||
These functions expect the decimal point of the *current* locale, so it is
|
||||
looked up right before the conversion instead of once when the lexer is
|
||||
constructed: a locale change in between (by a parser callback, a SAX
|
||||
handler, or another thread) must not truncate the value (#5198). The
|
||||
token has been validated before, so if the conversion stops early and the
|
||||
decimal point changed in the meantime, the locale changed between the
|
||||
lookup and the call, and the conversion is repeated with the new decimal
|
||||
point. If the decimal point did not change, a retry cannot succeed: the
|
||||
locale's decimal point is not a single character (e.g., the two-byte
|
||||
U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place.
|
||||
The value strtod parsed up to that point is kept, as before this change.
|
||||
|
||||
Note that changing the locale in another thread *while* strtod runs is
|
||||
undefined behavior of the C library, which this function cannot prevent.
|
||||
*/
|
||||
void convert_float_locale_aware()
|
||||
{
|
||||
const bool has_dot = decimal_point_position != std::string::npos;
|
||||
char decimal_point = get_decimal_point();
|
||||
for (;;)
|
||||
{
|
||||
const bool substitute = has_dot && decimal_point != '.';
|
||||
if (substitute)
|
||||
{
|
||||
token_buffer[decimal_point_position] = static_cast<typename string_t::value_type>(decimal_point);
|
||||
}
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
if (substitute)
|
||||
{
|
||||
// get_string() hands the token to the SAX interface with '.'
|
||||
token_buffer[decimal_point_position] = '.';
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_LIKELY(endptr == token_buffer.data() + token_buffer.size()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// retry only if the locale changed; otherwise, this would loop forever
|
||||
const char current_decimal_point = get_decimal_point();
|
||||
if (current_decimal_point == decimal_point)
|
||||
{
|
||||
return;
|
||||
}
|
||||
decimal_point = current_decimal_point;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief contiguous fast path for scanning a number
|
||||
|
||||
Parses the whole number token straight from the input buffer, avoiding the
|
||||
per-character get()/add() of scan_number(). On success it fills token_buffer
|
||||
(as scan_number() does) and
|
||||
(with the locale decimal point substituted, as scan_number() does) and
|
||||
returns the token type. On anything it does not fully recognize as a
|
||||
well-formed number it makes no state change and returns
|
||||
token_type::uninitialized, so the caller falls back to scan_number(), which
|
||||
@@ -1755,11 +1707,16 @@ scan_number_done:
|
||||
}
|
||||
#endif
|
||||
|
||||
// materialize the token exactly as scan_number() would. reset() already
|
||||
// cleared token_buffer, so append() fills it (assign() is avoided
|
||||
// because custom string_t types need not provide it)
|
||||
// materialize the token exactly as scan_number() would, substituting the
|
||||
// locale decimal point so convert_number()'s strtof fallback stays valid.
|
||||
// reset() already cleared token_buffer, so append() fills it (assign() is
|
||||
// avoided because custom string_t types need not provide it)
|
||||
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
|
||||
decimal_point_position = dot_index;
|
||||
if (dot_index != std::string::npos)
|
||||
{
|
||||
token_buffer[dot_index] = static_cast<typename string_t::value_type>(decimal_point_char);
|
||||
decimal_point_position = dot_index;
|
||||
}
|
||||
|
||||
ia.bulk_skip(len - 1);
|
||||
position.chars_read_total += (len - 1);
|
||||
@@ -2026,7 +1983,11 @@ scan_number_done:
|
||||
/// return current string value (implicitly resets the token; useful only once)
|
||||
string_t& get_string()
|
||||
{
|
||||
// a number token holds '.' regardless of the locale (#4084)
|
||||
// translate decimal points from locale back to '.' (#4084)
|
||||
if (decimal_point_char != '.' && decimal_point_position != std::string::npos)
|
||||
{
|
||||
token_buffer[decimal_point_position] = '.';
|
||||
}
|
||||
return token_buffer;
|
||||
}
|
||||
|
||||
@@ -2322,7 +2283,9 @@ scan_number_done:
|
||||
number_unsigned_t value_unsigned = 0;
|
||||
number_float_t value_float = 0;
|
||||
|
||||
/// the position of the decimal point in token_buffer
|
||||
/// the decimal point
|
||||
const char_int_type decimal_point_char = '.';
|
||||
/// the position of the decimal point in the input
|
||||
std::size_t decimal_point_position = std::string::npos;
|
||||
|
||||
/// whether the caller (e.g. accept()/json_sax_acceptor) only needs the
|
||||
|
||||
@@ -118,12 +118,14 @@ std::strtod. The parser only activates for number_float_t == double; float and
|
||||
long double keep the std::strtof/std::strtold paths (see the templated overload
|
||||
below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[in] decimal_point the (locale-dependent) decimal point character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept
|
||||
template<typename DecimalPointType>
|
||||
bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept
|
||||
{
|
||||
#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0
|
||||
// Clinger's fast path is only exact when double operations are evaluated in
|
||||
@@ -134,6 +136,7 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
// std::from_chars / std::strtod path.
|
||||
static_cast<void>(first);
|
||||
static_cast<void>(last);
|
||||
static_cast<void>(decimal_point);
|
||||
static_cast<void>(out);
|
||||
return false;
|
||||
#else
|
||||
@@ -172,7 +175,7 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (c == '.')
|
||||
else if (static_cast<DecimalPointType>(c) == decimal_point)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
@@ -257,8 +260,8 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept
|
||||
template<typename DecimalPointType, typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -270,7 +273,9 @@ std::from_chars is locale-independent, correctly rounded, and - via the
|
||||
Eisel-Lemire algorithm in modern standard libraries - much faster than strtod
|
||||
over the whole value range (not just the Clinger subset). It is used only when
|
||||
__cpp_lib_to_chars indicates full floating-point support and only when it
|
||||
consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so
|
||||
consumes the entire token ([first, last)); a partial parse means the buffer
|
||||
uses a non-'.' locale decimal point, in which case the caller falls back to the
|
||||
locale-aware path. An under-/overflow (result_out_of_range) also declines, so
|
||||
the caller's strtod fallback supplies the well-defined ±inf/0 result the parser
|
||||
expects (side-stepping the P4168 divergence between implementations).
|
||||
|
||||
|
||||
@@ -8605,12 +8605,14 @@ std::strtod. The parser only activates for number_float_t == double; float and
|
||||
long double keep the std::strtof/std::strtold paths (see the templated overload
|
||||
below).
|
||||
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[out] out the parsed value on success
|
||||
@param[in] first pointer to the first character of the number
|
||||
@param[in] last pointer past the last character
|
||||
@param[in] decimal_point the (locale-dependent) decimal point character
|
||||
@param[out] out the parsed value on success
|
||||
@return true if the value was parsed exactly; false to fall back to strtod
|
||||
*/
|
||||
inline bool parse_float_fast(const char* first, const char* last, double& out) noexcept
|
||||
template<typename DecimalPointType>
|
||||
bool parse_float_fast(const char* first, const char* last, DecimalPointType decimal_point, double& out) noexcept
|
||||
{
|
||||
#if defined(FLT_EVAL_METHOD) && FLT_EVAL_METHOD != 0
|
||||
// Clinger's fast path is only exact when double operations are evaluated in
|
||||
@@ -8621,6 +8623,7 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
// std::from_chars / std::strtod path.
|
||||
static_cast<void>(first);
|
||||
static_cast<void>(last);
|
||||
static_cast<void>(decimal_point);
|
||||
static_cast<void>(out);
|
||||
return false;
|
||||
#else
|
||||
@@ -8659,7 +8662,7 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
++num_digits;
|
||||
fractional_digits += static_cast<int>(seen_dot);
|
||||
}
|
||||
else if (c == '.')
|
||||
else if (static_cast<DecimalPointType>(c) == decimal_point)
|
||||
{
|
||||
if (JSON_HEDLEY_UNLIKELY(seen_dot))
|
||||
{
|
||||
@@ -8744,8 +8747,8 @@ inline bool parse_float_fast(const char* first, const char* last, double& out) n
|
||||
}
|
||||
|
||||
/// fast float path is only exact for `double`; decline for float/long double
|
||||
template<typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, FloatType& /*out*/) noexcept
|
||||
template<typename DecimalPointType, typename FloatType>
|
||||
bool parse_float_fast(const char* /*first*/, const char* /*last*/, DecimalPointType /*decimal_point*/, FloatType& /*out*/) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@@ -8757,7 +8760,9 @@ std::from_chars is locale-independent, correctly rounded, and - via the
|
||||
Eisel-Lemire algorithm in modern standard libraries - much faster than strtod
|
||||
over the whole value range (not just the Clinger subset). It is used only when
|
||||
__cpp_lib_to_chars indicates full floating-point support and only when it
|
||||
consumes the entire token ([first, last)). An under-/overflow (result_out_of_range) also declines, so
|
||||
consumes the entire token ([first, last)); a partial parse means the buffer
|
||||
uses a non-'.' locale decimal point, in which case the caller falls back to the
|
||||
locale-aware path. An under-/overflow (result_out_of_range) also declines, so
|
||||
the caller's strtod fallback supplies the well-defined ±inf/0 result the parser
|
||||
expects (side-stepping the P4168 divergence between implementations).
|
||||
|
||||
@@ -9298,6 +9303,7 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
|
||||
: ia(std::move(adapter))
|
||||
, ignore_comments(ignore_comments_)
|
||||
, decimal_point_char(static_cast<char_int_type>(get_decimal_point()))
|
||||
, discard_number_values(discard_number_values_)
|
||||
{}
|
||||
|
||||
@@ -9313,7 +9319,8 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
// locales
|
||||
/////////////////////
|
||||
|
||||
/// return the decimal point of the current locale
|
||||
/// return the locale-dependent decimal point
|
||||
JSON_HEDLEY_PURE
|
||||
static char get_decimal_point() noexcept
|
||||
{
|
||||
const auto* loc = localeconv();
|
||||
@@ -10182,10 +10189,9 @@ class lexer : public lexer_base<BasicJsonType>
|
||||
token_type::value_float if number could be successfully scanned,
|
||||
token_type::parse_error otherwise
|
||||
|
||||
@note The scanner is independent of the current locale: token_buffer
|
||||
always holds `.`. Only the std::strtod fallback of convert_number()
|
||||
depends on the locale, and it looks up the decimal point right
|
||||
before converting (see convert_float_locale_aware()).
|
||||
@note The scanner is independent of the current locale. Internally, the
|
||||
locale's decimal point is used instead of `.` to work with the
|
||||
locale-dependent converters.
|
||||
*/
|
||||
token_type scan_number() // lgtm [cpp/use-of-goto] `goto` is used in this function to implement the number-parsing state machine described above. By design, any finite input will eventually reach the "done" state or return token_type::parse_error. In each intermediate state, 1 byte of the input is appended to the token_buffer vector, and only the already initialized variables token_buffer, number_type, and error_message are manipulated.
|
||||
{
|
||||
@@ -10274,7 +10280,7 @@ scan_number_zero:
|
||||
{
|
||||
case '.':
|
||||
{
|
||||
add(current);
|
||||
add(decimal_point_char);
|
||||
decimal_point_position = token_buffer.size() - 1;
|
||||
goto scan_number_decimal1;
|
||||
}
|
||||
@@ -10311,7 +10317,7 @@ scan_number_any1:
|
||||
|
||||
case '.':
|
||||
{
|
||||
add(current);
|
||||
add(decimal_point_char);
|
||||
decimal_point_position = token_buffer.size() - 1;
|
||||
goto scan_number_decimal1;
|
||||
}
|
||||
@@ -10553,9 +10559,9 @@ scan_number_done:
|
||||
|
||||
// Only a number below 1 can carry further insignificant zeros, and only
|
||||
// while the count stays at the limit does removing them change the
|
||||
// answer - so this loop is skipped for all but a few tokens. The
|
||||
// fraction is located through decimal_point_position rather than by
|
||||
// searching '.'.
|
||||
// answer - so this loop is skipped for all but a few tokens. Note
|
||||
// token_buffer holds the locale's decimal point, so the fraction is
|
||||
// located through decimal_point_position rather than by searching '.'.
|
||||
if (lead_zero != 0)
|
||||
{
|
||||
JSON_ASSERT(has_dot != 0); // an integer "0" cannot reach the limit
|
||||
@@ -10573,8 +10579,8 @@ scan_number_done:
|
||||
@brief convert the number text in token_buffer to its value and token type
|
||||
|
||||
The digit sequence in token_buffer has already been validated (by the
|
||||
scan_number() state machine or by the contiguous fast path) and holds '.'
|
||||
as decimal point, independent of the locale. Integers are parsed first and fall
|
||||
scan_number() state machine or by the contiguous fast path) and holds the
|
||||
locale decimal point in place of '.'. Integers are parsed first and fall
|
||||
back to floating point on overflow. This is shared so both scanners produce
|
||||
identical results.
|
||||
|
||||
@@ -10654,7 +10660,7 @@ scan_number_done:
|
||||
// integer conversion above overflowed. Prefer std::from_chars
|
||||
// (Eisel-Lemire, locale-independent, correctly rounded) when available;
|
||||
// otherwise the exact Clinger fast path (double only); otherwise the
|
||||
// locale-aware strtof/strtod/strtold.
|
||||
// locale-aware strtof/strtod.
|
||||
if (parse_float_from_chars(num_begin, num_end, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
@@ -10663,75 +10669,26 @@ scan_number_done:
|
||||
// extra pass over the token's bytes, which otherwise shows up on
|
||||
// high-precision inputs such as canada.json
|
||||
if (mantissa_fits_clinger(mantissa_end)
|
||||
&& parse_float_fast(num_begin, num_end, value_float))
|
||||
&& parse_float_fast(num_begin, num_end, decimal_point_char, value_float))
|
||||
{
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
convert_float_locale_aware();
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
// we checked the number format before
|
||||
JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
|
||||
|
||||
return token_type::value_float;
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief convert the float in token_buffer with strtof/strtod/strtold
|
||||
|
||||
These functions expect the decimal point of the *current* locale, so it is
|
||||
looked up right before the conversion instead of once when the lexer is
|
||||
constructed: a locale change in between (by a parser callback, a SAX
|
||||
handler, or another thread) must not truncate the value (#5198). The
|
||||
token has been validated before, so if the conversion stops early and the
|
||||
decimal point changed in the meantime, the locale changed between the
|
||||
lookup and the call, and the conversion is repeated with the new decimal
|
||||
point. If the decimal point did not change, a retry cannot succeed: the
|
||||
locale's decimal point is not a single character (e.g., the two-byte
|
||||
U+066B of ar_EG.UTF-8 or fa_IR.UTF-8) and cannot be substituted in place.
|
||||
The value strtod parsed up to that point is kept, as before this change.
|
||||
|
||||
Note that changing the locale in another thread *while* strtod runs is
|
||||
undefined behavior of the C library, which this function cannot prevent.
|
||||
*/
|
||||
void convert_float_locale_aware()
|
||||
{
|
||||
const bool has_dot = decimal_point_position != std::string::npos;
|
||||
char decimal_point = get_decimal_point();
|
||||
for (;;)
|
||||
{
|
||||
const bool substitute = has_dot && decimal_point != '.';
|
||||
if (substitute)
|
||||
{
|
||||
token_buffer[decimal_point_position] = static_cast<typename string_t::value_type>(decimal_point);
|
||||
}
|
||||
|
||||
char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
|
||||
strtof(value_float, token_buffer.data(), &endptr);
|
||||
|
||||
if (substitute)
|
||||
{
|
||||
// get_string() hands the token to the SAX interface with '.'
|
||||
token_buffer[decimal_point_position] = '.';
|
||||
}
|
||||
|
||||
if (JSON_HEDLEY_LIKELY(endptr == token_buffer.data() + token_buffer.size()))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// retry only if the locale changed; otherwise, this would loop forever
|
||||
const char current_decimal_point = get_decimal_point();
|
||||
if (current_decimal_point == decimal_point)
|
||||
{
|
||||
return;
|
||||
}
|
||||
decimal_point = current_decimal_point;
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief contiguous fast path for scanning a number
|
||||
|
||||
Parses the whole number token straight from the input buffer, avoiding the
|
||||
per-character get()/add() of scan_number(). On success it fills token_buffer
|
||||
(as scan_number() does) and
|
||||
(with the locale decimal point substituted, as scan_number() does) and
|
||||
returns the token type. On anything it does not fully recognize as a
|
||||
well-formed number it makes no state change and returns
|
||||
token_type::uninitialized, so the caller falls back to scan_number(), which
|
||||
@@ -10847,11 +10804,16 @@ scan_number_done:
|
||||
}
|
||||
#endif
|
||||
|
||||
// materialize the token exactly as scan_number() would. reset() already
|
||||
// cleared token_buffer, so append() fills it (assign() is avoided
|
||||
// because custom string_t types need not provide it)
|
||||
// materialize the token exactly as scan_number() would, substituting the
|
||||
// locale decimal point so convert_number()'s strtof fallback stays valid.
|
||||
// reset() already cleared token_buffer, so append() fills it (assign() is
|
||||
// avoided because custom string_t types need not provide it)
|
||||
token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), len);
|
||||
decimal_point_position = dot_index;
|
||||
if (dot_index != std::string::npos)
|
||||
{
|
||||
token_buffer[dot_index] = static_cast<typename string_t::value_type>(decimal_point_char);
|
||||
decimal_point_position = dot_index;
|
||||
}
|
||||
|
||||
ia.bulk_skip(len - 1);
|
||||
position.chars_read_total += (len - 1);
|
||||
@@ -11118,7 +11080,11 @@ scan_number_done:
|
||||
/// return current string value (implicitly resets the token; useful only once)
|
||||
string_t& get_string()
|
||||
{
|
||||
// a number token holds '.' regardless of the locale (#4084)
|
||||
// translate decimal points from locale back to '.' (#4084)
|
||||
if (decimal_point_char != '.' && decimal_point_position != std::string::npos)
|
||||
{
|
||||
token_buffer[decimal_point_position] = '.';
|
||||
}
|
||||
return token_buffer;
|
||||
}
|
||||
|
||||
@@ -11414,7 +11380,9 @@ scan_number_done:
|
||||
number_unsigned_t value_unsigned = 0;
|
||||
number_float_t value_float = 0;
|
||||
|
||||
/// the position of the decimal point in token_buffer
|
||||
/// the decimal point
|
||||
const char_int_type decimal_point_char = '.';
|
||||
/// the position of the decimal point in the input
|
||||
std::size_t decimal_point_position = std::string::npos;
|
||||
|
||||
/// whether the caller (e.g. accept()/json_sax_acceptor) only needs the
|
||||
|
||||
@@ -666,7 +666,7 @@ TEST_CASE("parse_float_fast declines what it cannot convert exactly")
|
||||
// always safe: the caller then falls back to a slower, exact conversion.
|
||||
const auto fast = [](const std::string & s, double & out)
|
||||
{
|
||||
return nlohmann::detail::parse_float_fast(s.data(), s.data() + s.size(), out);
|
||||
return nlohmann::detail::parse_float_fast(s.data(), s.data() + s.size(), '.', out);
|
||||
};
|
||||
double out = 0;
|
||||
|
||||
|
||||
@@ -12,12 +12,7 @@
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <array>
|
||||
#include <clocale>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
struct ParserImpl final: public nlohmann::json_sax<json>
|
||||
{
|
||||
@@ -180,208 +175,3 @@ TEST_CASE("locale-dependent test (LC_NUMERIC=de_DE)")
|
||||
MESSAGE("locale de_DE is not usable");
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// records the numbers of a flat array and switches LC_NUMERIC to the given
|
||||
// locale once the array opens - after the lexer was constructed, but before
|
||||
// any number in the array is lexed
|
||||
struct LocaleSwitchingSax final: public nlohmann::json_sax<json>
|
||||
{
|
||||
explicit LocaleSwitchingSax(const char* switch_to)
|
||||
: locale_after_open(switch_to)
|
||||
{}
|
||||
|
||||
bool null() override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool boolean(bool /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool number_integer(json::number_integer_t /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool number_unsigned(json::number_unsigned_t /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool number_float(json::number_float_t val, const json::string_t& s) override
|
||||
{
|
||||
values.push_back(val);
|
||||
strings.push_back(s);
|
||||
return true;
|
||||
}
|
||||
bool string(json::string_t& /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool binary(json::binary_t& /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool start_object(std::size_t /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool key(json::string_t& /*val*/) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool end_object() override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool start_array(std::size_t /*val*/) override
|
||||
{
|
||||
switched = std::setlocale(LC_NUMERIC, locale_after_open) != nullptr;
|
||||
return true;
|
||||
}
|
||||
bool end_array() override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
bool parse_error(std::size_t /*val*/, const std::string& /*val*/, const nlohmann::detail::exception& /*val*/) override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* locale_after_open;
|
||||
bool switched = false;
|
||||
std::vector<json::number_float_t> values;
|
||||
std::vector<json::string_t> strings;
|
||||
};
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("locale changes between lexer construction and number conversion (#5198)")
|
||||
{
|
||||
// The numbers are chosen so that the conversion also takes the strtod
|
||||
// fallback, which honors the locale that is current at conversion time:
|
||||
// too many significant digits for Clinger's fast path, an underflow that
|
||||
// std::from_chars rejects, and a plain value.
|
||||
const std::vector<std::string> numbers = {"3.14159265358979323846", "1.5e-400", "12.34", "-0.000123456789012345678"};
|
||||
std::string text = "[";
|
||||
for (const auto& n : numbers)
|
||||
{
|
||||
text += (text.size() == 1 ? "" : ",") + n;
|
||||
}
|
||||
text += "]";
|
||||
|
||||
using long_double_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, long double>;
|
||||
|
||||
// reference values, parsed without a locale switch
|
||||
REQUIRE(std::setlocale(LC_NUMERIC, "C") != nullptr);
|
||||
const json expected = json::parse(text);
|
||||
const long_double_json expected_ld = long_double_json::parse(text);
|
||||
|
||||
const std::array<std::pair<const char*, const char*>, 2> transitions =
|
||||
{
|
||||
{
|
||||
{"C", "de_DE"},
|
||||
{"de_DE", "C"}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto& transition : transitions)
|
||||
{
|
||||
CAPTURE(transition.first);
|
||||
CAPTURE(transition.second);
|
||||
|
||||
if (std::setlocale(LC_NUMERIC, transition.first) == nullptr)
|
||||
{
|
||||
MESSAGE("locale is not usable");
|
||||
continue;
|
||||
}
|
||||
|
||||
// SAX parsing
|
||||
{
|
||||
LocaleSwitchingSax sax(transition.second);
|
||||
CHECK(json::sax_parse(text, &sax));
|
||||
if (sax.switched)
|
||||
{
|
||||
CHECK(sax.values == expected.get<std::vector<json::number_float_t>>());
|
||||
CHECK(sax.strings == numbers);
|
||||
}
|
||||
}
|
||||
|
||||
// DOM parsing with a callback
|
||||
{
|
||||
bool switched = false;
|
||||
const auto cb = [&](int /*depth*/, json::parse_event_t event, json& /*parsed*/)
|
||||
{
|
||||
if (event == json::parse_event_t::array_start)
|
||||
{
|
||||
switched = std::setlocale(LC_NUMERIC, transition.second) != nullptr;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const json j = json::parse(text, cb);
|
||||
if (switched)
|
||||
{
|
||||
CHECK(j == expected);
|
||||
}
|
||||
}
|
||||
|
||||
// a long double goes through std::strtold unless std::from_chars supports it
|
||||
{
|
||||
bool switched = false;
|
||||
const auto cb = [&](int /*depth*/, long_double_json::parse_event_t event, long_double_json& /*parsed*/)
|
||||
{
|
||||
if (event == long_double_json::parse_event_t::array_start)
|
||||
{
|
||||
switched = std::setlocale(LC_NUMERIC, transition.second) != nullptr;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const long_double_json j = long_double_json::parse(text, cb);
|
||||
if (switched)
|
||||
{
|
||||
CHECK(j == expected_ld);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
}
|
||||
|
||||
TEST_CASE("locale with a multi-byte decimal point")
|
||||
{
|
||||
// Some locales use a decimal point that is not a single character, e.g.
|
||||
// U+066B ARABIC DECIMAL SEPARATOR (two bytes in UTF-8). It cannot be
|
||||
// substituted in place for '.', so the strtod fallback stops early. The
|
||||
// conversion must still terminate rather than retry forever.
|
||||
const std::array<const char*, 6> names = {{"ar_EG.UTF-8", "ar_SA.UTF-8", "fa_IR.UTF-8", "ps_AF.UTF-8", "ar_EG", "fa_IR"}};
|
||||
bool tested = false;
|
||||
for (const char* name : names)
|
||||
{
|
||||
if (std::setlocale(LC_NUMERIC, name) == nullptr)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const std::string decimal_point = std::localeconv()->decimal_point;
|
||||
if (decimal_point.size() < 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
CAPTURE(name);
|
||||
tested = true;
|
||||
|
||||
// too many significant digits for Clinger's fast path, and an underflow
|
||||
// that std::from_chars rejects: both reach the strtod fallback
|
||||
json j;
|
||||
CHECK_NOTHROW(j = json::parse("[3.14159265358979323846, 1.5e-400, -0.000123456789012345678]"));
|
||||
CHECK(j.is_array());
|
||||
CHECK(json::accept("3.14159265358979323846"));
|
||||
|
||||
// a value the locale-independent paths convert is not affected
|
||||
CHECK(json::parse("12.5") == 12.5);
|
||||
}
|
||||
if (!tested)
|
||||
{
|
||||
MESSAGE("no locale with a multi-byte decimal point is usable");
|
||||
}
|
||||
|
||||
std::setlocale(LC_NUMERIC, "C");
|
||||
}
|
||||
|
||||
+12
-11
@@ -2165,6 +2165,13 @@ TEST_CASE("MessagePack with std::byte")
|
||||
#endif
|
||||
|
||||
// the fake sizes below do not fit into a 32-bit std::size_t
|
||||
// with clang and libstdc++ 10, the std::filesystem::path conversion that
|
||||
// C++17 builds consider for every string type is ambiguous for a class
|
||||
// derived from std::string, so the string case is not tested there
|
||||
#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11)
|
||||
#define JSON_TEST_BEYOND_UINT32_STRING 1
|
||||
#endif
|
||||
|
||||
#if SIZE_MAX > UINT32_MAX
|
||||
template<typename T, typename A = std::allocator<T>>
|
||||
struct huge_array : std::vector<T, A>
|
||||
@@ -2262,11 +2269,12 @@ TEST_CASE("MessagePack Size above uint32 for object")
|
||||
object.fake_size = false;
|
||||
}
|
||||
|
||||
#ifdef JSON_TEST_BEYOND_UINT32_STRING
|
||||
struct huge_string : std::string
|
||||
{
|
||||
using std::string::string;
|
||||
|
||||
std::size_t size() const noexcept
|
||||
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
|
||||
{
|
||||
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
|
||||
}
|
||||
@@ -2287,20 +2295,20 @@ using huge_string_json = nlohmann::basic_json <
|
||||
|
||||
TEST_CASE("MessagePack Size above uint32 for string")
|
||||
{
|
||||
|
||||
huge_string_json j = "hello";
|
||||
const huge_string_json j = "hello";
|
||||
|
||||
CHECK_THROWS_WITH_AS(
|
||||
huge_string_json::to_msgpack(j),
|
||||
"[json.exception.out_of_range.412] MessagePack length 4294967296 exceeds maximum of 4294967295",
|
||||
json::out_of_range&);
|
||||
}
|
||||
#endif
|
||||
|
||||
struct huge_binary : std::vector<std::uint8_t>
|
||||
{
|
||||
using std::vector<std::uint8_t>::vector;
|
||||
|
||||
std::size_t size() const noexcept
|
||||
std::size_t size() const noexcept // NOLINT(readability-convert-member-functions-to-static)
|
||||
{
|
||||
return static_cast<std::size_t>(UINT32_MAX) + 1ULL;
|
||||
}
|
||||
@@ -2355,13 +2363,6 @@ class beyond_uint32_binary_t : public std::vector<std::uint8_t>
|
||||
}
|
||||
};
|
||||
|
||||
// with clang and libstdc++ 10, the std::filesystem::path conversion that
|
||||
// C++17 builds consider for every string type is ambiguous for a class
|
||||
// derived from std::string, so the string case is not tested there
|
||||
#if !(defined(__clang__) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11)
|
||||
#define JSON_TEST_BEYOND_UINT32_STRING 1
|
||||
#endif
|
||||
|
||||
#ifdef JSON_TEST_BEYOND_UINT32_STRING
|
||||
class beyond_uint32_string_t : public std::string
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user