lexer: use std::from_chars when available, fall back to strto* otherwise

`std::from_chars` is locale-independent and thus avoids the TOCTOU
issue #5198 when it's available.

This commit introduces a `from_chars` utility in the `detail` namespace
which merely wraps `std::from_chars` if it is available and emulates its
API and behavior using the `strto*` family of functions when it is not.
As of this commit, the `strto*`-based fallback is still locale-dependent
and only matches `std::from_chars` when the "C" locale is used.

Availability of `std::from_chars` for floating point numbers is varied.
For instance, as of libc++ 22, `std::from_chars` for `long double` isn't
supported and, consequently, the feature-test macro `__cpp_lib_to_chars`
is not set, while the overloads for `float` and `double` are available.
To avoid complicating matters further, the feature-test macro is taken
as the authoritative indicator for (full) `std::from_chars` support and
the fallback is used otherwise.

If `std::from_chars` is available, our wrapper additionally tweaks its
output in the case of floating-point out-of-range results. The behavior
in this case is poorly (under-)specified and existing implementations do
not agree. The wrapper's tweaks are intended to ensure consistent
behavior across all toolchains in this edge case. In practice, this
should only take effect when parsing out-of-range floats on libstdc++.

The unit tests for the `from_chars` helper are intentionally compiled in
both C++11 and C++17 modes. This ensures that the test expectations
align with the actual behavior we're trying to model. The tests are not
aiming to cover all of floating-point parsing comprehensively, as
`std::from_chars` is specified to behave mostly the same as `strto*`
and instead focuses on the edge cases where the two differ:
- leading whitespace is not tolerated
- `+` sign is not allowed (outside of the exponent)
- out-parameter is left unchanged in case of `invalid_argument` and
  integral `result_out_of_range` errors

Signed-off-by: Jonas Greitemann <jgreitemann@gmail.com>

include/nlohmann/detail/conversions/from_chars.hpp

@@ -0,0 +1,369 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <nlohmann/detail/macro_scope.hpp>
+
+#if JSON_HAS_FROM_CHARS
+    #include <algorithm> // find, find_if
+    #include <charconv> // from_chars, from_chars_result
+    #include <cmath> // isnan
+#else
+    #include <cctype> // isspace
+    #include <cerrno> // ERANGE
+    #include <clocale> // localeconv
+    #include <cstdlib> // strto*
+    #include <type_traits> // enable_if, is_unsigned
+#endif
+#include <limits> // numeric_limits<T>::[has_]infinity, quiet_NaN
+#include <system_error> // errc
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+constexpr char get_locale_independent_decimal_point() noexcept
+{
+    return '.';
+}
+
+#if JSON_HAS_FROM_CHARS
+
+//////////////////////////////////////////////
+// Delegate to std::from_chars if supported //
+//////////////////////////////////////////////
+
+/*!
+@brief Number parsing implementation details
+
+A traits class template that allows users of nlohmann::detail::from_chars to
+query details of the used implementation.
+
+@tparam T numeric type to parse, matching the `value` argument of from_chars
+*/
+template <typename T>
+struct from_chars_traits
+{
+    /// whether the from_chars in unaffected by the current global C locale
+    static constexpr bool is_locale_independent = true;
+
+    /// getter for the character used as decimal point by from_chars
+    static constexpr char(*get_decimal_point)() = &get_locale_independent_decimal_point;
+};
+
+using std::from_chars_result;
+
+/*!
+@brief Parse integer/floating-point numbers
+
+Parses the string [first, last) into the numeric type T.
+
+This implementation merely delegates to `std::from_chars` with one noteworthy
+difference: Different C++ standard library implementations do not agree on
+whether `value` should be set in the out-of-range case for floating-point
+numbers. Presently, libstdc++ leaves `value` unchanged whereas libc++ and MS STL
+set `value` to +/-0 or +/-inf which allows its users to distinguish between the
+various cases of over- and underflow. The C++ proposal [P4168][1] details this
+discrepancy and advocates to standardize the latter behavior.
+
+Since we need to be able to distinguish absolute underflow (+/-0) from absolute
+overflow (+/-inf), this implementation "corrects" the out-of-range behavior
+accordingly by estimating the effective exponent through manual string parsing.
+
+[1]: https://isocpp.org/files/papers/P4168R0.html
+
+@tparam T numeric type to parse
+@param[in] first points to the beginning of the parsed string (inclusive)
+@param[in] end points to the end of the parsed string (exclusive)
+@param[out] value references the variable to set the parsed number
+@return structure consisting of `ptr` and `ec` such that:
+        - If the string starting at `first` matches a number, `ptr` points to
+          the address immediately after the last character that matches.
+          * If the matched number can be represented by `T`, `value` is set and
+            `ec` is value-initialized.
+          * If the matched number cannot be represented by `T`, `ec` is set to
+            `std::errc::result_out_of_range` and depending on the type of `T`:
+            + `value` is left unchanged for integral `T`,
+            + `value` is set to +/-0 or +/-inf for floating-point `T`.
+        - If the string starting at `first` doesn't match a number, `ptr`
+          points to `first`, `ec` is `std::errc::invalid_argument`, and `value`
+          is left unchanged.
+*/
+template <typename T>
+JSON_HEDLEY_NON_NULL(1, 2)
+inline from_chars_result from_chars(const char* first, const char* last, T& value)
+{
+    // No implementation of std::from_chars will set its value argument to NaN,
+    // so by using NaN as the initial value, we can tell if it changed.
+    T inner_value = std::numeric_limits<T>::quiet_NaN();
+    auto res = std::from_chars(first, last, inner_value);
+
+    if constexpr (std::numeric_limits<T>::has_infinity)
+    {
+        if (res.ec == std::errc::result_out_of_range)
+        {
+            if (std::isnan(inner_value)) // inner_value was not set
+            {
+                // [first, res.ptr) matches a valid floating-point number that's
+                // out-of-range and our std::from_chars did not set `value`, so we
+                // need to distinguish the type of under-/overflow ourselves.
+                const char* mantissa_begin = *first == '-' ? first + 1 : first;
+                const char* mantissa_end = std::find_if(mantissa_begin, res.ptr, [](char c)
+                {
+                    return c == 'e' || c == 'E';
+                });
+                const char* decimal_point = std::find(mantissa_begin, mantissa_end, '.');
+                const char* significant_digit = std::find_if(mantissa_begin, mantissa_end, [](char d)
+                {
+                    return d >= '1' && d <= '9';
+                });
+
+                // Position of the significant digit relative to the decimal gives
+                // the order of magnitude of the mantissa.
+                auto effective_exponent = static_cast<int>(decimal_point - significant_digit);
+
+                // If the number includes an explicit exponent, add that to the the
+                // total exponent.
+                if (mantissa_end != res.ptr)
+                {
+                    const char* exponent_begin = *(mantissa_end + 1) == '+'
+                                                 ? mantissa_end + 2 // skip the exponent's + sign
+                                                 : mantissa_end + 1;
+                    int exponent = 0;
+                    auto exp_res = std::from_chars(exponent_begin, res.ptr, exponent);
+                    JSON_ASSERT(exp_res.ptr == res.ptr);
+
+                    if (exp_res.ec == std::errc::result_out_of_range)
+                    {
+                        // NOTE: Parentheses around function names mitigate min/max macro collision on Windows
+                        effective_exponent = *exponent_begin == '-'
+                                             ? (std::numeric_limits<int>::min)()
+                                             : (std::numeric_limits<int>::max)();
+                    }
+                    else
+                    {
+                        JSON_ASSERT(exp_res.ec == std::errc{});
+                        effective_exponent += exponent;
+                    }
+                }
+
+                // Set `value` to the sign-correct non-finite value based on the
+                // effective exponent.
+                value = (*first == '-' ? -1 : 1) * (effective_exponent < 0
+                                                    ? T{}
+                                                    : std::numeric_limits<T>::infinity());
+            }
+            else
+            {
+                value = inner_value;
+            }
+        }
+    }
+
+    if (res.ec == std::errc{})
+    {
+        value = inner_value;
+    }
+
+    return res;
+}
+
+#else
+
+//////////////////////////////////////////
+// Fallback implementation using strto* //
+//////////////////////////////////////////
+
+/*!
+@brief Query the decimal point used by the current C locale
+
+Note that calling this function while switching the global C locale from
+another thread is undefined behavior.
+*/
+JSON_HEDLEY_PURE
+inline char get_locale_decimal_point() noexcept
+{
+    const auto* loc = localeconv();
+    JSON_ASSERT(loc != nullptr);
+    return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
+}
+
+/*!
+@brief Number parsing implementation details
+
+A traits class template that allows users of nlohmann::detail::from_chars to
+query details of the used implementation.
+
+Each specialization provides the following static members:
+- is_locale_independent: whether the from_chars in unaffected by the current
+                         global C locale
+- get_decimal_point: getter for the character used as decimal point by from_chars
+- strto: dispatches to the C standard library strto* function for type T
+
+@tparam T numeric type to parse, matching the `value` argument of from_chars
+*/
+template <typename T>
+struct from_chars_traits;
+
+template <>
+struct from_chars_traits<long long> // NOLINT(runtime/int)
+{
+    static constexpr bool is_locale_independent = false;
+    static constexpr char(*get_decimal_point)() = &get_locale_decimal_point;
+    static long long strto(const char* str, char** endptr) noexcept
+    {
+        return std::strtoll(str, endptr, 10);
+    }
+};
+
+template <>
+struct from_chars_traits<unsigned long long> // NOLINT(runtime/int)
+{
+    static constexpr bool is_locale_independent = false;
+    static constexpr char(*get_decimal_point)() = &get_locale_decimal_point;
+    static unsigned long long strto(const char* str, char** endptr) noexcept
+    {
+        return std::strtoull(str, endptr, 10);
+    }
+};
+
+template <>
+struct from_chars_traits<float>
+{
+    static constexpr bool is_locale_independent = false;
+    static constexpr char(*get_decimal_point)() = &get_locale_decimal_point;
+    static constexpr float(*strto)(const char*, char**) = &std::strtof;
+};
+
+template <>
+struct from_chars_traits<double>
+{
+    static constexpr bool is_locale_independent = false;
+    static constexpr char(*get_decimal_point)() = &get_locale_decimal_point;
+    static constexpr double(*strto)(const char*, char**) = &std::strtod;
+};
+
+template <>
+struct from_chars_traits<long double>
+{
+    static constexpr bool is_locale_independent = false;
+    static constexpr char(*get_decimal_point)() = &get_locale_decimal_point;
+    static constexpr long double(*strto)(const char*, char**) = &std::strtold;
+};
+
+template <typename T>
+constexpr typename std::enable_if<std::numeric_limits<T>::has_infinity, bool>::type is_out_of_range_value(T value) noexcept
+{
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfloat-equal"
+#endif
+    return value == T {} || value == std::numeric_limits<T>::infinity() || value == -std::numeric_limits<T>::infinity();
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+}
+
+template <typename T>
+constexpr typename std::enable_if < !std::numeric_limits<T>::has_infinity, bool >::type is_out_of_range_value(T value) noexcept
+{
+    // NOTE: Parentheses around function names mitigate min/max macro collision on Windows
+    return value == (std::numeric_limits<T>::max)() || value == (std::numeric_limits<T>::min)();
+}
+
+struct from_chars_result
+{
+    const char* ptr;
+    std::errc ec;
+};
+
+/*!
+@brief Parse integer/floating-point numbers
+
+Parses the string [first, last) into the numeric type T.
+
+This implementation uses the C standard library functions strto* to emulate the
+behavior of `std::from_chars` on platforms where it is not (fully) supported.
+
+Regarding the under-/overflow behavior, this implementation adopts the behavior
+proposed by [P4168][1] and implemented by libc++ and MS STL of setting `value`
+to the corresponding non-finite floating-point value in case of an out-of-range
+result.
+
+[1]: https://isocpp.org/files/papers/P4168R0.html
+
+@pre Unlike std::from_chars, this implementation requires the string to be
+     null-terminated, such that `last` dereferences into a NUL byte.
+
+@tparam T numeric type to parse
+@param[in] first points to the beginning of the parsed string (inclusive)
+@param[in] end points to the end of the parsed string (exclusive)
+@param[out] value references the variable to set the parsed number
+@return structure consisting of `ptr` and `ec` such that:
+        - If the string starting at `first` matches a number, `ptr` points to
+          the address immediately after the last character that matches.
+          * If the matched number can be represented by `T`, `value` is set and
+            `ec` is value-initialized.
+          * If the matched number cannot be represented by `T`, `ec` is set to
+            `std::errc::result_out_of_range` and depending on the type of `T`:
+            + `value` is left unchanged for integral `T`,
+            + `value` is set to +/-0 or +/-inf for floating-point `T`.
+        - If the string starting at `first` doesn't match a number, `ptr`
+          points to `first`, `ec` is `std::errc::invalid_argument`, and `value`
+          is left unchanged.
+*/
+template <typename T>
+JSON_HEDLEY_NON_NULL(1, 2)
+inline from_chars_result from_chars(const char* first, const char* last, T& value)
+{
+    JSON_ASSERT(*last == '\0');
+
+    // Unlike strto*, from_chars does not accept leading whitespace or + signs
+    if (first == last || *first == '+'
+            || (std::is_unsigned<T>::value && *first == '-')
+            || std::isspace(*first) != 0)
+    {
+        return {first, std::errc::invalid_argument};
+    }
+
+    errno = 0;
+    char* ptr = nullptr; // NOLINT(misc-const-correctness)
+    T result = from_chars_traits<T>::strto(first, &ptr);
+
+    if (ptr == first)
+    {
+        return {ptr, std::errc::invalid_argument};
+    }
+
+    // Upon under-/overflow, strto* returns a marginal value and sets errno.
+    // Note that it is NOT sufficient to just check errno: strto* only clears
+    // errno if the parsed string actually parses into 0/[U]LLONG_MIN/-_MAX
+    // and this result was returned without indicating an under-/overflow.
+    // Otherwise, errno may not be relied upon to indicate the *absence* of
+    // an out-of-range error.
+    if (is_out_of_range_value(result) && errno == ERANGE)
+    {
+        if (std::numeric_limits<T>::has_infinity)
+        {
+            // ONLY for floating-point types, set `value` to the same non-finite
+            // result returned by strto* to allow users to distinguish between
+            // different types of under-/overflow.
+            value = result;
+        }
+        return {ptr, std::errc::result_out_of_range};
+    }
+
+    value = result;
+    return {ptr, std::errc{}};
+}
+
+#endif
+
+}  // namespace detail
+NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/input/lexer.hpp

@@ -9,15 +9,15 @@
 #pragma once
 
 #include <array> // array
-#include <clocale> // localeconv
 #include <cstddef> // size_t
 #include <cstdio> // snprintf
-#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull
 #include <initializer_list> // initializer_list
 #include <string> // char_traits, string
+#include <system_error> // errc
 #include <utility> // move
 #include <vector> // vector
 
+#include <nlohmann/detail/conversions/from_chars.hpp>
 #include <nlohmann/detail/input/input_adapters.hpp>
 #include <nlohmann/detail/input/position_t.hpp>
 #include <nlohmann/detail/macro_scope.hpp>
@@ -152,7 +152,7 @@ class lexer : public lexer_base<BasicJsonType>
     explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept
         : ia(std::move(adapter))
         , ignore_comments(ignore_comments_)
-        , decimal_point_char(static_cast<char_int_type>(get_decimal_point()))
+        , decimal_point_char(static_cast<char_int_type>(from_chars_traits<number_float_t>::get_decimal_point()))
     {}
 
     // deleted because of pointer members
@@ -163,19 +163,6 @@ class lexer : public lexer_base<BasicJsonType>
     ~lexer() = default;
 
   private:
-    /////////////////////
-    // locales
-    /////////////////////
-
-    /// return the locale-dependent decimal point
-    JSON_HEDLEY_PURE
-    static char get_decimal_point() noexcept
-    {
-        const auto* loc = localeconv();
-        JSON_ASSERT(loc != nullptr);
-        return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point);
-    }
-
     /////////////////////
     // scan functions
     /////////////////////
@@ -941,24 +928,6 @@ class lexer : public lexer_base<BasicJsonType>
         }
     }
 
-    JSON_HEDLEY_NON_NULL(2)
-    static void strtof(float& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtof(str, endptr);
-    }
-
-    JSON_HEDLEY_NON_NULL(2)
-    static void strtof(double& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtod(str, endptr);
-    }
-
-    JSON_HEDLEY_NON_NULL(2)
-    static void strtof(long double& f, const char* str, char** endptr) noexcept
-    {
-        f = std::strtold(str, endptr);
-    }
-
     /*!
     @brief scan a number literal
 
@@ -1279,19 +1248,18 @@ scan_number_done:
         // we are done scanning a number)
         unget();
 
-        char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg)
-        errno = 0;
-
         // try to parse integers first and fall back to floats
         if (number_type == token_type::value_unsigned)
         {
-            const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
+            unsigned long long x{}; // NOLINT(runtime/int)
+            const auto res = ::nlohmann::detail::from_chars(token_buffer.data(), token_buffer.data() + token_buffer.size(), x);
 
             // we checked the number format before
-            JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+            JSON_ASSERT(res.ptr == token_buffer.data() + token_buffer.size());
 
-            if (errno != ERANGE)
+            if (res.ec != std::errc::result_out_of_range)
             {
+                JSON_ASSERT(res.ec == std::errc{});
                 value_unsigned = static_cast<number_unsigned_t>(x);
                 if (value_unsigned == x)
                 {
@@ -1301,13 +1269,15 @@ scan_number_done:
         }
         else if (number_type == token_type::value_integer)
         {
-            const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
+            long long x{}; // NOLINT(runtime/int)
+            const auto res = ::nlohmann::detail::from_chars(token_buffer.data(), token_buffer.data() + token_buffer.size(), x);
 
             // we checked the number format before
-            JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+            JSON_ASSERT(res.ptr == token_buffer.data() + token_buffer.size());
 
-            if (errno != ERANGE)
+            if (res.ec != std::errc::result_out_of_range)
             {
+                JSON_ASSERT(res.ec == std::errc{});
                 value_integer = static_cast<number_integer_t>(x);
                 if (value_integer == x)
                 {
@@ -1318,10 +1288,11 @@ scan_number_done:
 
         // this code is reached if we parse a floating-point number or if an
         // integer conversion above failed
-        strtof(value_float, token_buffer.data(), &endptr);
+        const auto res = ::nlohmann::detail::from_chars(token_buffer.data(), token_buffer.data() + token_buffer.size(), value_float);
 
         // we checked the number format before
-        JSON_ASSERT(endptr == token_buffer.data() + token_buffer.size());
+        JSON_ASSERT(res.ptr == token_buffer.data() + token_buffer.size());
+        JSON_ASSERT(res.ec != std::errc::invalid_argument);
 
         return token_type::value_float;
     }

include/nlohmann/detail/macro_scope.hpp

@@ -124,6 +124,20 @@
     #define JSON_HAS_FILESYSTEM 0
 #endif
 
+#ifndef JSON_HAS_FROM_CHARS
+    #if defined(JSON_HAS_CPP_17) && defined(__cpp_lib_to_chars)
+        // The std::from_chars<float> implementation in libstdc++ 11 gets some of the corner cases wrong;
+        // starting with libstdc++ 12, these are fixed by switching the implementation to fast_float.
+        #if defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12
+            #define JSON_HAS_FROM_CHARS 0
+        #else
+            #define JSON_HAS_FROM_CHARS 1
+        #endif
+    #else
+        #define JSON_HAS_FROM_CHARS 0
+    #endif
+#endif
+
 #ifndef JSON_HAS_THREE_WAY_COMPARISON
     #if defined(__cpp_impl_three_way_comparison) && __cpp_impl_three_way_comparison >= 201907L \
         && defined(__cpp_lib_three_way_comparison) && __cpp_lib_three_way_comparison >= 201907L

include/nlohmann/detail/macro_unscope.hpp

@@ -39,6 +39,7 @@
     #undef JSON_HAS_CPP_26
     #undef JSON_HAS_FILESYSTEM
     #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
+    #undef JSON_HAS_FROM_CHARS
     #undef JSON_HAS_THREE_WAY_COMPARISON
     #undef JSON_HAS_RANGES
     #undef JSON_HAS_STD_FORMAT