Merge branch 'claude/parser-stream-performance-8cebe9' into perf/lexer-number-scan

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/lexer.hpp

@@ -1625,9 +1625,12 @@ scan_number_done:
         const std::size_t avail = rem + 1;
 
         // validate + classify the number extent (mirrors scan_number()'s grammar)
+        // and accumulate the mantissa digits on the way, so that tokens with up
+        // to 19 digits need no second pass for their value
         std::size_t i = 0;
         std::size_t dot_index = std::string::npos;
         token_type number_type = token_type::value_unsigned;
+        std::uint64_t w = 0;
         if (data[0] == '-')
         {
             number_type = token_type::value_integer;
@@ -1637,22 +1640,21 @@ scan_number_done:
                 return token_type::uninitialized;
             }
         }
+        const std::size_t integer_begin = i;
         if (data[i] == '0')
         {
             ++i;
         }
         else if (data[i] >= '1' && data[i] <= '9')
         {
-            ++i;
-            while (i < avail && data[i] >= '0' && data[i] <= '9')
-            {
-                ++i;
-            }
+            i = accumulate_digits(data, i, avail, w);
         }
         else
         {
             return token_type::uninitialized;
         }
+        std::size_t digit_count = i - integer_begin;
+        std::size_t fraction_digits = 0;
         if (i < avail && data[i] == '.')
         {
             number_type = token_type::value_float;
@@ -1662,29 +1664,39 @@ scan_number_done:
             {
                 return token_type::uninitialized;
             }
-            while (i < avail && data[i] >= '0' && data[i] <= '9')
-            {
-                ++i;
-            }
+            const std::size_t fraction_begin = i;
+            i = accumulate_digits(data, i, avail, w);
+            fraction_digits = i - fraction_begin;
+            digit_count += fraction_digits;
         }
         // the mantissa ends here, whether or not an exponent part follows
         const std::size_t mantissa_end = i;
+        std::int64_t exponent = 0;
         if (i < avail && (data[i] == 'e' || data[i] == 'E'))
         {
             number_type = token_type::value_float;
             ++i;
+            bool negative_exponent = false;
             if (i < avail && (data[i] == '+' || data[i] == '-'))
             {
+                negative_exponent = data[i] == '-';
                 ++i;
             }
             if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
             {
                 return token_type::uninitialized;
             }
+            // saturated far beyond every range, see parse_float_exponent()
+            constexpr std::int64_t saturation = 100000000000000000; // 10^17
             while (i < avail && data[i] >= '0' && data[i] <= '9')
             {
+                if (exponent < saturation)
+                {
+                    exponent = (exponent * 10) + (data[i] - '0');
+                }
                 ++i;
             }
+            exponent = negative_exponent ? -exponent : exponent;
         }
         const std::size_t len = i;
         if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -1705,7 +1717,10 @@ scan_number_done:
 #if !JSON_DIAGNOSTIC_POSITIONS
         if (number_type != token_type::value_float)
         {
-            const token_type integer_result = convert_integer(number_type, data, data + len);
+            // up to 18 digits always fit into std::uint64_t and std::int64_t
+            const token_type integer_result = (digit_count <= 18)
+                                              ? store_integer(number_type, w)
+                                              : convert_integer(number_type, data, data + len);
             if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
             {
                 bulk_consume(data + 1, len - 1);
@@ -1728,6 +1743,20 @@ scan_number_done:
         bulk_consume(data + 1, len - 1);
         read_number_terminator(std::integral_constant<bool, bulk_window> {});
 
+        // a float whose digits all fit into w converts from the accumulated
+        // value; longer ones (and other float formats) take the general path
+        if (number_type == token_type::value_float && digit_count <= 19)
+        {
+            float_significand s;
+            s.w = w;
+            s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
+            s.negative = data[0] == '-';
+            if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
+            {
+                return token_type::value_float;
+            }
+        }
+
         return convert_number(number_type, mantissa_end);
     }
 
@@ -1756,6 +1785,35 @@ scan_number_done:
     /// the input is not handed back, so the terminator can stay unread
     void read_number_terminator(std::false_type /*window*/) const noexcept {}
 
+    /// binary32/binary64: the value of a float token from its accumulated
+    /// significand (see scan_number_bulk_contiguous())
+    bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
+    {
+        value_float = decimal_to_float<number_float_t>(s);
+        return true;
+    }
+
+    /// other formats (e.g. x87 long double): not instantiated for them, as
+    /// decimal_to_float() only supports binary32/binary64
+    static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
+    {
+        return false;
+    }
+
+    /// the value of an integer token with at most 18 digits, accumulated in @a w
+    token_type store_integer(token_type number_type, std::uint64_t w) noexcept
+    {
+        if (number_type == token_type::value_unsigned)
+        {
+            value_unsigned = static_cast<number_unsigned_t>(w);
+            // reject values that do not round-trip into a narrower type
+            return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
+        }
+        const std::int64_t x = -static_cast<std::int64_t>(w);
+        value_integer = static_cast<number_integer_t>(x);
+        return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
+    }
+
     /// contiguous input: try the number fast path, else the byte-path scanner
     token_type scan_number_dispatch(std::true_type /*bulk*/)
     {

include/nlohmann/detail/input/number_parse.hpp

@@ -275,13 +275,53 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
     return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
 }
 
-/// whether [first, last) contains a digit other than '0'
-inline bool has_nonzero_digit(const char* first, const char* last) noexcept
+/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
+/// digits (after fast_float)
+JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
 {
-    return std::any_of(first, last, [](char c) noexcept
+    return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
+}
+
+/*!
+@brief skip the ASCII digits starting at data[i] and accumulate their value
+
+Eight digits are taken at a time while at least eight bytes remain. @a w wraps
+around after 19 digits; the caller only uses it for shorter digit runs.
+
+@return the index of the first byte at or after @a i that is not a digit
+*/
+JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
+{
+    while (avail - i >= 8)
     {
-        return c != '0';
-    });
+        const std::uint64_t v = read_eight_bytes(data + i);
+        if (!is_eight_digits(v))
+        {
+            break;
+        }
+        w = (w * 100000000u) + parse_eight_digits(v);
+        i += 8;
+    }
+    while (i < avail && data[i] >= '0' && data[i] <= '9')
+    {
+        w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
+        ++i;
+    }
+    return i;
+}
+
+/// whether [first, last) contains a digit other than '0'; a plain loop, as
+/// GCC calls std::any_of out of line even for the usual empty range
+JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
+{
+    for (; first != last; ++first)
+    {
+        if (*first != '0')
+        {
+            return true;
+        }
+    }
+    return false;
 }
 
 /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -327,7 +367,7 @@ truncated is set.
                                    std::string::npos if there is none
 @param[in] mantissa_end            index of the 'e'/'E', or the token length
 */
-inline float_significand parse_float_significand(const char* first, const char* last,
+JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
         std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
 {
     float_significand s;

single_include/nlohmann/json.hpp

@@ -10316,13 +10316,53 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex
     return static_cast<std::uint32_t>(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u);
 }
 
-/// whether [first, last) contains a digit other than '0'
-inline bool has_nonzero_digit(const char* first, const char* last) noexcept
+/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII
+/// digits (after fast_float)
+JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept
 {
-    return std::any_of(first, last, [](char c) noexcept
+    return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u;
+}
+
+/*!
+@brief skip the ASCII digits starting at data[i] and accumulate their value
+
+Eight digits are taken at a time while at least eight bytes remain. @a w wraps
+around after 19 digits; the caller only uses it for shorter digit runs.
+
+@return the index of the first byte at or after @a i that is not a digit
+*/
+JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept
+{
+    while (avail - i >= 8)
     {
-        return c != '0';
-    });
+        const std::uint64_t v = read_eight_bytes(data + i);
+        if (!is_eight_digits(v))
+        {
+            break;
+        }
+        w = (w * 100000000u) + parse_eight_digits(v);
+        i += 8;
+    }
+    while (i < avail && data[i] >= '0' && data[i] <= '9')
+    {
+        w = (w * 10u) + static_cast<std::uint64_t>(data[i] - '0');
+        ++i;
+    }
+    return i;
+}
+
+/// whether [first, last) contains a digit other than '0'; a plain loop, as
+/// GCC calls std::any_of out of line even for the usual empty range
+JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept
+{
+    for (; first != last; ++first)
+    {
+        if (*first != '0')
+        {
+            return true;
+        }
+    }
+    return false;
 }
 
 /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last),
@@ -10368,7 +10408,7 @@ truncated is set.
                                    std::string::npos if there is none
 @param[in] mantissa_end            index of the 'e'/'E', or the token length
 */
-inline float_significand parse_float_significand(const char* first, const char* last,
+JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last,
         std::size_t decimal_point_position, std::size_t mantissa_end) noexcept
 {
     float_significand s;
@@ -13071,9 +13111,12 @@ scan_number_done:
         const std::size_t avail = rem + 1;
 
         // validate + classify the number extent (mirrors scan_number()'s grammar)
+        // and accumulate the mantissa digits on the way, so that tokens with up
+        // to 19 digits need no second pass for their value
         std::size_t i = 0;
         std::size_t dot_index = std::string::npos;
         token_type number_type = token_type::value_unsigned;
+        std::uint64_t w = 0;
         if (data[0] == '-')
         {
             number_type = token_type::value_integer;
@@ -13083,22 +13126,21 @@ scan_number_done:
                 return token_type::uninitialized;
             }
         }
+        const std::size_t integer_begin = i;
         if (data[i] == '0')
         {
             ++i;
         }
         else if (data[i] >= '1' && data[i] <= '9')
         {
-            ++i;
-            while (i < avail && data[i] >= '0' && data[i] <= '9')
-            {
-                ++i;
-            }
+            i = accumulate_digits(data, i, avail, w);
         }
         else
         {
             return token_type::uninitialized;
         }
+        std::size_t digit_count = i - integer_begin;
+        std::size_t fraction_digits = 0;
         if (i < avail && data[i] == '.')
         {
             number_type = token_type::value_float;
@@ -13108,29 +13150,39 @@ scan_number_done:
             {
                 return token_type::uninitialized;
             }
-            while (i < avail && data[i] >= '0' && data[i] <= '9')
-            {
-                ++i;
-            }
+            const std::size_t fraction_begin = i;
+            i = accumulate_digits(data, i, avail, w);
+            fraction_digits = i - fraction_begin;
+            digit_count += fraction_digits;
         }
         // the mantissa ends here, whether or not an exponent part follows
         const std::size_t mantissa_end = i;
+        std::int64_t exponent = 0;
         if (i < avail && (data[i] == 'e' || data[i] == 'E'))
         {
             number_type = token_type::value_float;
             ++i;
+            bool negative_exponent = false;
             if (i < avail && (data[i] == '+' || data[i] == '-'))
             {
+                negative_exponent = data[i] == '-';
                 ++i;
             }
             if (i >= avail || !(data[i] >= '0' && data[i] <= '9'))
             {
                 return token_type::uninitialized;
             }
+            // saturated far beyond every range, see parse_float_exponent()
+            constexpr std::int64_t saturation = 100000000000000000; // 10^17
             while (i < avail && data[i] >= '0' && data[i] <= '9')
             {
+                if (exponent < saturation)
+                {
+                    exponent = (exponent * 10) + (data[i] - '0');
+                }
                 ++i;
             }
+            exponent = negative_exponent ? -exponent : exponent;
         }
         const std::size_t len = i;
         if (number_may_continue(len == avail, std::integral_constant<bool, bulk_window> {}))
@@ -13151,7 +13203,10 @@ scan_number_done:
 #if !JSON_DIAGNOSTIC_POSITIONS
         if (number_type != token_type::value_float)
         {
-            const token_type integer_result = convert_integer(number_type, data, data + len);
+            // up to 18 digits always fit into std::uint64_t and std::int64_t
+            const token_type integer_result = (digit_count <= 18)
+                                              ? store_integer(number_type, w)
+                                              : convert_integer(number_type, data, data + len);
             if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized))
             {
                 bulk_consume(data + 1, len - 1);
@@ -13174,6 +13229,20 @@ scan_number_done:
         bulk_consume(data + 1, len - 1);
         read_number_terminator(std::integral_constant<bool, bulk_window> {});
 
+        // a float whose digits all fit into w converts from the accumulated
+        // value; longer ones (and other float formats) take the general path
+        if (number_type == token_type::value_float && digit_count <= 19)
+        {
+            float_significand s;
+            s.w = w;
+            s.exponent = exponent - static_cast<std::int64_t>(fraction_digits);
+            s.negative = data[0] == '-';
+            if (convert_accumulated_float(s, std::integral_constant<bool, has_native_float_format<number_float_t>::value> {}))
+            {
+                return token_type::value_float;
+            }
+        }
+
         return convert_number(number_type, mantissa_end);
     }
 
@@ -13202,6 +13271,35 @@ scan_number_done:
     /// the input is not handed back, so the terminator can stay unread
     void read_number_terminator(std::false_type /*window*/) const noexcept {}
 
+    /// binary32/binary64: the value of a float token from its accumulated
+    /// significand (see scan_number_bulk_contiguous())
+    bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept
+    {
+        value_float = decimal_to_float<number_float_t>(s);
+        return true;
+    }
+
+    /// other formats (e.g. x87 long double): not instantiated for them, as
+    /// decimal_to_float() only supports binary32/binary64
+    static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept
+    {
+        return false;
+    }
+
+    /// the value of an integer token with at most 18 digits, accumulated in @a w
+    token_type store_integer(token_type number_type, std::uint64_t w) noexcept
+    {
+        if (number_type == token_type::value_unsigned)
+        {
+            value_unsigned = static_cast<number_unsigned_t>(w);
+            // reject values that do not round-trip into a narrower type
+            return (static_cast<std::uint64_t>(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized;
+        }
+        const std::int64_t x = -static_cast<std::int64_t>(w);
+        value_integer = static_cast<number_integer_t>(x);
+        return (static_cast<std::int64_t>(value_integer) == x) ? token_type::value_integer : token_type::uninitialized;
+    }
+
     /// contiguous input: try the number fast path, else the byte-path scanner
     token_type scan_number_dispatch(std::true_type /*bulk*/)
     {

tests/src/unit-class_lexer.cpp

@@ -15,9 +15,12 @@ using nlohmann::json;
 #include <array> // array
 #include <cstdint> // uint32_t, uint64_t
 #include <cstdio> // snprintf
+#include <cmath> // isfinite
 #include <cstdlib> // strtod
 #include <cstring> // memcpy
 #include <limits> // numeric_limits
+#include <initializer_list> // initializer_list
+#include <list> // list
 #include <map> // map
 #include <random> // mt19937
 #include <sstream> // stringstream
@@ -332,6 +335,109 @@ TEST_CASE("lexer number fast path")
         }
     }
 
+    SECTION("digits accumulated while scanning")
+    {
+        // The fast path accumulates the digits while it validates them, eight
+        // at a time where possible, and converts tokens with at most 19
+        // digits (floats) or 18 digits (integers) from that value. Every digit
+        // count from 1 to 25, in the integer and in the fraction part, with
+        // and without sign and exponent, must give the byte path's result. A
+        // std::list has no contiguous storage, so parsing it takes the byte
+        // path.
+        // type and value, or the error (values beyond the range of double
+        // are rejected with out_of_range.406 on both paths)
+        const auto outcome = [](const std::string & doc, bool byte_path) -> std::string
+        {
+            JSON_TRY
+            {
+                const std::list<char> chars(doc.begin(), doc.end());
+                const json j = byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc);
+                return std::string(j[0].type_name()) + '|' + j.dump();
+            }
+            JSON_CATCH (const json::exception & e)
+            {
+                return e.what();
+            }
+        };
+
+        // the concatenation of @a parts
+        const auto join = [](std::initializer_list<std::string> parts)
+        {
+            std::string result;
+            for (const auto& part : parts)
+            {
+                result += part;
+            }
+            return result;
+        };
+
+        std::vector<std::string> numbers;
+        for (std::size_t digits = 1; digits <= 25; ++digits)
+        {
+            std::string run;
+            for (std::size_t i = 0; i < digits; ++i)
+            {
+                run += static_cast<char>('1' + static_cast<char>((i * 7) % 9));
+            }
+            const std::string nines(digits, '9');
+            for (const char* sign_text :
+                    {"", "-"
+                    })
+            {
+                const std::string sign = sign_text;
+                numbers.push_back(join({sign, run}));
+                numbers.push_back(join({sign, nines}));
+                numbers.push_back(join({sign, "1", std::string(digits - 1, '0')}));
+                numbers.push_back(join({sign, "0.", run}));
+                numbers.push_back(join({sign, "0.", std::string(digits, '0'), "1"}));
+                numbers.push_back(join({sign, run, ".", run}));
+                numbers.push_back(join({sign, run, "e-", std::to_string(digits)}));
+                numbers.push_back(join({sign, "1.", run, "E+", std::to_string(digits * 13)}));
+            }
+        }
+        // exponents beyond every range, and zeros
+        numbers.insert(numbers.end(), {"1e400", "1e-400", "-1e99999999999999999999", "1e-99999999999999999999",
+                                       "0e99999999999999999999", "0.0e-5", "-0.000", "123456789e-0000000000000000000000330"
+                                      });
+
+        for (const auto& n : numbers)
+        {
+            CAPTURE(n)
+            const std::string doc = "[" + n + "]";
+            const std::string a = outcome(doc, false);
+            CHECK(a == outcome(doc, true));
+            const double expected = std::strtod(n.c_str(), nullptr);
+            if (a.compare(0, 6, "number") == 0 && std::isfinite(expected) && n.find_first_of(".eE") != std::string::npos)
+            {
+                CHECK(json::parse(doc)[0].get<double>() == expected);
+            }
+        }
+    }
+
+    SECTION("integers that do not fit a narrower type become floats")
+    {
+        // up to 18 digits, the fast path stores the accumulated value without
+        // an overflow check; a narrower number_integer_t/number_unsigned_t
+        // must still see the value rejected and converted as a float
+        using json32 = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int32_t, std::uint32_t, float>;
+        for (const char* n :
+                {"4294967295", "4294967296", "2147483647", "-2147483648", "-2147483649", "99999999999", "-99999999999"
+                })
+        {
+            CAPTURE(n)
+            const std::string doc = std::string("[") + n + "]";
+            const std::list<char> chars(doc.begin(), doc.end());
+            const json32 a = json32::parse(doc);
+            const json32 b = json32::parse(chars.begin(), chars.end());
+            CHECK(a[0].type() == b[0].type());
+            CHECK(a == b);
+        }
+        CHECK(json32::parse("[4294967295]")[0].is_number_unsigned());
+        CHECK(json32::parse("[4294967296]")[0].is_number_float());
+        CHECK(json32::parse("[-2147483648]")[0].is_number_integer());
+        CHECK(json32::parse("[-2147483649]")[0].is_number_float());
+    }
+
     SECTION("token type classification")
     {
         CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));