From 32ca00cdfab7a6e5c1bcbc9b48def97091eba337 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 15 Oct 2026 17:24:44 +0200 Subject: [PATCH] Accumulate number digits while scanning them The contiguous number fast path validated a token byte by byte, copied it into token_buffer, and then converted it in out-of-line functions that scanned the digits a second time: parse_float_native() called parse_float_significand(), which returned its result through memory and called std::any_of (out of line with GCC) twice per number, and parse_integer_signed() divided by ten on every digit for its overflow check. scan_number_bulk_contiguous() now accumulates the significand while it validates the digits, eight at a time where at least eight bytes remain (is_eight_digits() and parse_eight_digits(), after fast_float). Floats with at most 19 digits go straight to decimal_to_float(); integers with at most 18 digits are stored without an overflow check (a narrower number type still rejects values that do not round-trip). Longer tokens take the existing path. has_nonzero_digit() is a plain loop and parse_float_significand() is always inlined, which helps that path. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 78 +++++++++-- .../nlohmann/detail/input/number_parse.hpp | 52 ++++++- single_include/nlohmann/json.hpp | 130 +++++++++++++++--- tests/src/unit-class_lexer.cpp | 106 ++++++++++++++ 4 files changed, 334 insertions(+), 32 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 9bc20cdd5..0f10af759 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -1625,9 +1625,12 @@ scan_number_done: const std::size_t avail = rem + 1; // validate + classify the number extent (mirrors scan_number()'s grammar) + // and accumulate the mantissa digits on the way, so that tokens with up + // to 19 digits need no second pass for their value std::size_t i = 0; std::size_t dot_index = std::string::npos; token_type number_type = token_type::value_unsigned; + std::uint64_t w = 0; if (data[0] == '-') { number_type = token_type::value_integer; @@ -1637,22 +1640,21 @@ scan_number_done: return token_type::uninitialized; } } + const std::size_t integer_begin = i; if (data[i] == '0') { ++i; } else if (data[i] >= '1' && data[i] <= '9') { - ++i; - while (i < avail && data[i] >= '0' && data[i] <= '9') - { - ++i; - } + i = accumulate_digits(data, i, avail, w); } else { return token_type::uninitialized; } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; if (i < avail && data[i] == '.') { number_type = token_type::value_float; @@ -1662,29 +1664,39 @@ scan_number_done: { return token_type::uninitialized; } - while (i < avail && data[i] >= '0' && data[i] <= '9') - { - ++i; - } + const std::size_t fraction_begin = i; + i = accumulate_digits(data, i, avail, w); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; } // the mantissa ends here, whether or not an exponent part follows const std::size_t mantissa_end = i; + std::int64_t exponent = 0; if (i < avail && (data[i] == 'e' || data[i] == 'E')) { number_type = token_type::value_float; ++i; + bool negative_exponent = false; if (i < avail && (data[i] == '+' || data[i] == '-')) { + negative_exponent = data[i] == '-'; ++i; } if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) { return token_type::uninitialized; } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 while (i < avail && data[i] >= '0' && data[i] <= '9') { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } ++i; } + exponent = negative_exponent ? -exponent : exponent; } const std::size_t len = i; if (number_may_continue(len == avail, std::integral_constant {})) @@ -1705,7 +1717,10 @@ scan_number_done: #if !JSON_DIAGNOSTIC_POSITIONS if (number_type != token_type::value_float) { - const token_type integer_result = convert_integer(number_type, data, data + len); + // up to 18 digits always fit into std::uint64_t and std::int64_t + const token_type integer_result = (digit_count <= 18) + ? store_integer(number_type, w) + : convert_integer(number_type, data, data + len); if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { bulk_consume(data + 1, len - 1); @@ -1728,6 +1743,20 @@ scan_number_done: bulk_consume(data + 1, len - 1); read_number_terminator(std::integral_constant {}); + // a float whose digits all fit into w converts from the accumulated + // value; longer ones (and other float formats) take the general path + if (number_type == token_type::value_float && digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = data[0] == '-'; + if (convert_accumulated_float(s, std::integral_constant::value> {})) + { + return token_type::value_float; + } + } + return convert_number(number_type, mantissa_end); } @@ -1756,6 +1785,35 @@ scan_number_done: /// the input is not handed back, so the terminator can stay unread void read_number_terminator(std::false_type /*window*/) const noexcept {} + /// binary32/binary64: the value of a float token from its accumulated + /// significand (see scan_number_bulk_contiguous()) + bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept + { + value_float = decimal_to_float(s); + return true; + } + + /// other formats (e.g. x87 long double): not instantiated for them, as + /// decimal_to_float() only supports binary32/binary64 + static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept + { + return false; + } + + /// the value of an integer token with at most 18 digits, accumulated in @a w + token_type store_integer(token_type number_type, std::uint64_t w) noexcept + { + if (number_type == token_type::value_unsigned) + { + value_unsigned = static_cast(w); + // reject values that do not round-trip into a narrower type + return (static_cast(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized; + } + const std::int64_t x = -static_cast(w); + value_integer = static_cast(x); + return (static_cast(value_integer) == x) ? token_type::value_integer : token_type::uninitialized; + } + /// contiguous input: try the number fast path, else the byte-path scanner token_type scan_number_dispatch(std::true_type /*bulk*/) { diff --git a/include/nlohmann/detail/input/number_parse.hpp b/include/nlohmann/detail/input/number_parse.hpp index e6e5bd58b..bb845cc1a 100644 --- a/include/nlohmann/detail/input/number_parse.hpp +++ b/include/nlohmann/detail/input/number_parse.hpp @@ -259,13 +259,53 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } -/// whether [first, last) contains a digit other than '0' -inline bool has_nonzero_digit(const char* first, const char* last) noexcept +/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII +/// digits (after fast_float) +JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept { - return std::any_of(first, last, [](char c) noexcept + return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; +} + +/*! +@brief skip the ASCII digits starting at data[i] and accumulate their value + +Eight digits are taken at a time while at least eight bytes remain. @a w wraps +around after 19 digits; the caller only uses it for shorter digit runs. + +@return the index of the first byte at or after @a i that is not a digit +*/ +JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept +{ + while (avail - i >= 8) { - return c != '0'; - }); + const std::uint64_t v = read_eight_bytes(data + i); + if (!is_eight_digits(v)) + { + break; + } + w = (w * 100000000u) + parse_eight_digits(v); + i += 8; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + w = (w * 10u) + static_cast(data[i] - '0'); + ++i; + } + return i; +} + +/// whether [first, last) contains a digit other than '0'; a plain loop, as +/// GCC calls std::any_of out of line even for the usual empty range +JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + for (; first != last; ++first) + { + if (*first != '0') + { + return true; + } + } + return false; } /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), @@ -311,7 +351,7 @@ truncated is set. std::string::npos if there is none @param[in] mantissa_end index of the 'e'/'E', or the token length */ -inline float_significand parse_float_significand(const char* first, const char* last, +JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept { float_significand s; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index f437cc5d5..a8498bd71 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9996,13 +9996,53 @@ JSON_HEDLEY_ALWAYS_INLINE std::uint32_t parse_eight_digits(std::uint64_t v) noex return static_cast(((v & 0x0000FFFF0000FFFFu) * 42949672960001u) >> 32u); } -/// whether [first, last) contains a digit other than '0' -inline bool has_nonzero_digit(const char* first, const char* last) noexcept +/// whether the eight bytes in @a v (see read_eight_bytes()) are all ASCII +/// digits (after fast_float) +JSON_HEDLEY_ALWAYS_INLINE bool is_eight_digits(std::uint64_t v) noexcept { - return std::any_of(first, last, [](char c) noexcept + return ((v & 0xF0F0F0F0F0F0F0F0u) | (((v + 0x0606060606060606u) & 0xF0F0F0F0F0F0F0F0u) >> 4u)) == 0x3333333333333333u; +} + +/*! +@brief skip the ASCII digits starting at data[i] and accumulate their value + +Eight digits are taken at a time while at least eight bytes remain. @a w wraps +around after 19 digits; the caller only uses it for shorter digit runs. + +@return the index of the first byte at or after @a i that is not a digit +*/ +JSON_HEDLEY_ALWAYS_INLINE std::size_t accumulate_digits(const char* data, std::size_t i, std::size_t avail, std::uint64_t& w) noexcept +{ + while (avail - i >= 8) { - return c != '0'; - }); + const std::uint64_t v = read_eight_bytes(data + i); + if (!is_eight_digits(v)) + { + break; + } + w = (w * 100000000u) + parse_eight_digits(v); + i += 8; + } + while (i < avail && data[i] >= '0' && data[i] <= '9') + { + w = (w * 10u) + static_cast(data[i] - '0'); + ++i; + } + return i; +} + +/// whether [first, last) contains a digit other than '0'; a plain loop, as +/// GCC calls std::any_of out of line even for the usual empty range +JSON_HEDLEY_ALWAYS_INLINE bool has_nonzero_digit(const char* first, const char* last) noexcept +{ + for (; first != last; ++first) + { + if (*first != '0') + { + return true; + } + } + return false; } /// the value of the validated exponent digits [+-]?[0-9]+ in [first, last), @@ -10048,7 +10088,7 @@ truncated is set. std::string::npos if there is none @param[in] mantissa_end index of the 'e'/'E', or the token length */ -inline float_significand parse_float_significand(const char* first, const char* last, +JSON_HEDLEY_ALWAYS_INLINE float_significand parse_float_significand(const char* first, const char* last, std::size_t decimal_point_position, std::size_t mantissa_end) noexcept { float_significand s; @@ -12751,9 +12791,12 @@ scan_number_done: const std::size_t avail = rem + 1; // validate + classify the number extent (mirrors scan_number()'s grammar) + // and accumulate the mantissa digits on the way, so that tokens with up + // to 19 digits need no second pass for their value std::size_t i = 0; std::size_t dot_index = std::string::npos; token_type number_type = token_type::value_unsigned; + std::uint64_t w = 0; if (data[0] == '-') { number_type = token_type::value_integer; @@ -12763,22 +12806,21 @@ scan_number_done: return token_type::uninitialized; } } + const std::size_t integer_begin = i; if (data[i] == '0') { ++i; } else if (data[i] >= '1' && data[i] <= '9') { - ++i; - while (i < avail && data[i] >= '0' && data[i] <= '9') - { - ++i; - } + i = accumulate_digits(data, i, avail, w); } else { return token_type::uninitialized; } + std::size_t digit_count = i - integer_begin; + std::size_t fraction_digits = 0; if (i < avail && data[i] == '.') { number_type = token_type::value_float; @@ -12788,29 +12830,39 @@ scan_number_done: { return token_type::uninitialized; } - while (i < avail && data[i] >= '0' && data[i] <= '9') - { - ++i; - } + const std::size_t fraction_begin = i; + i = accumulate_digits(data, i, avail, w); + fraction_digits = i - fraction_begin; + digit_count += fraction_digits; } // the mantissa ends here, whether or not an exponent part follows const std::size_t mantissa_end = i; + std::int64_t exponent = 0; if (i < avail && (data[i] == 'e' || data[i] == 'E')) { number_type = token_type::value_float; ++i; + bool negative_exponent = false; if (i < avail && (data[i] == '+' || data[i] == '-')) { + negative_exponent = data[i] == '-'; ++i; } if (i >= avail || !(data[i] >= '0' && data[i] <= '9')) { return token_type::uninitialized; } + // saturated far beyond every range, see parse_float_exponent() + constexpr std::int64_t saturation = 100000000000000000; // 10^17 while (i < avail && data[i] >= '0' && data[i] <= '9') { + if (exponent < saturation) + { + exponent = (exponent * 10) + (data[i] - '0'); + } ++i; } + exponent = negative_exponent ? -exponent : exponent; } const std::size_t len = i; if (number_may_continue(len == avail, std::integral_constant {})) @@ -12831,7 +12883,10 @@ scan_number_done: #if !JSON_DIAGNOSTIC_POSITIONS if (number_type != token_type::value_float) { - const token_type integer_result = convert_integer(number_type, data, data + len); + // up to 18 digits always fit into std::uint64_t and std::int64_t + const token_type integer_result = (digit_count <= 18) + ? store_integer(number_type, w) + : convert_integer(number_type, data, data + len); if (JSON_HEDLEY_LIKELY(integer_result != token_type::uninitialized)) { bulk_consume(data + 1, len - 1); @@ -12854,6 +12909,20 @@ scan_number_done: bulk_consume(data + 1, len - 1); read_number_terminator(std::integral_constant {}); + // a float whose digits all fit into w converts from the accumulated + // value; longer ones (and other float formats) take the general path + if (number_type == token_type::value_float && digit_count <= 19) + { + float_significand s; + s.w = w; + s.exponent = exponent - static_cast(fraction_digits); + s.negative = data[0] == '-'; + if (convert_accumulated_float(s, std::integral_constant::value> {})) + { + return token_type::value_float; + } + } + return convert_number(number_type, mantissa_end); } @@ -12882,6 +12951,35 @@ scan_number_done: /// the input is not handed back, so the terminator can stay unread void read_number_terminator(std::false_type /*window*/) const noexcept {} + /// binary32/binary64: the value of a float token from its accumulated + /// significand (see scan_number_bulk_contiguous()) + bool convert_accumulated_float(const float_significand& s, std::true_type /*native*/) noexcept + { + value_float = decimal_to_float(s); + return true; + } + + /// other formats (e.g. x87 long double): not instantiated for them, as + /// decimal_to_float() only supports binary32/binary64 + static bool convert_accumulated_float(const float_significand& /*s*/, std::false_type /*native*/) noexcept + { + return false; + } + + /// the value of an integer token with at most 18 digits, accumulated in @a w + token_type store_integer(token_type number_type, std::uint64_t w) noexcept + { + if (number_type == token_type::value_unsigned) + { + value_unsigned = static_cast(w); + // reject values that do not round-trip into a narrower type + return (static_cast(value_unsigned) == w) ? token_type::value_unsigned : token_type::uninitialized; + } + const std::int64_t x = -static_cast(w); + value_integer = static_cast(x); + return (static_cast(value_integer) == x) ? token_type::value_integer : token_type::uninitialized; + } + /// contiguous input: try the number fast path, else the byte-path scanner token_type scan_number_dispatch(std::true_type /*bulk*/) { diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 05cd2bf1d..91c835db3 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -15,9 +15,12 @@ using nlohmann::json; #include // array #include // uint32_t, uint64_t #include // snprintf +#include // isfinite #include // strtod #include // memcpy #include // numeric_limits +#include // initializer_list +#include // list #include // map #include // mt19937 #include // stringstream @@ -332,6 +335,109 @@ TEST_CASE("lexer number fast path") } } + SECTION("digits accumulated while scanning") + { + // The fast path accumulates the digits while it validates them, eight + // at a time where possible, and converts tokens with at most 19 + // digits (floats) or 18 digits (integers) from that value. Every digit + // count from 1 to 25, in the integer and in the fraction part, with + // and without sign and exponent, must give the byte path's result. A + // std::list has no contiguous storage, so parsing it takes the byte + // path. + // type and value, or the error (values beyond the range of double + // are rejected with out_of_range.406 on both paths) + const auto outcome = [](const std::string & doc, bool byte_path) -> std::string + { + JSON_TRY + { + const std::list chars(doc.begin(), doc.end()); + const json j = byte_path ? json::parse(chars.begin(), chars.end()) : json::parse(doc); + return std::string(j[0].type_name()) + '|' + j.dump(); + } + JSON_CATCH (const json::exception & e) + { + return e.what(); + } + }; + + // the concatenation of @a parts + const auto join = [](std::initializer_list parts) + { + std::string result; + for (const auto& part : parts) + { + result += part; + } + return result; + }; + + std::vector numbers; + for (std::size_t digits = 1; digits <= 25; ++digits) + { + std::string run; + for (std::size_t i = 0; i < digits; ++i) + { + run += static_cast('1' + static_cast((i * 7) % 9)); + } + const std::string nines(digits, '9'); + for (const char* sign_text : + {"", "-" + }) + { + const std::string sign = sign_text; + numbers.push_back(join({sign, run})); + numbers.push_back(join({sign, nines})); + numbers.push_back(join({sign, "1", std::string(digits - 1, '0')})); + numbers.push_back(join({sign, "0.", run})); + numbers.push_back(join({sign, "0.", std::string(digits, '0'), "1"})); + numbers.push_back(join({sign, run, ".", run})); + numbers.push_back(join({sign, run, "e-", std::to_string(digits)})); + numbers.push_back(join({sign, "1.", run, "E+", std::to_string(digits * 13)})); + } + } + // exponents beyond every range, and zeros + numbers.insert(numbers.end(), {"1e400", "1e-400", "-1e99999999999999999999", "1e-99999999999999999999", + "0e99999999999999999999", "0.0e-5", "-0.000", "123456789e-0000000000000000000000330" + }); + + for (const auto& n : numbers) + { + CAPTURE(n) + const std::string doc = "[" + n + "]"; + const std::string a = outcome(doc, false); + CHECK(a == outcome(doc, true)); + const double expected = std::strtod(n.c_str(), nullptr); + if (a.compare(0, 6, "number") == 0 && std::isfinite(expected) && n.find_first_of(".eE") != std::string::npos) + { + CHECK(json::parse(doc)[0].get() == expected); + } + } + } + + SECTION("integers that do not fit a narrower type become floats") + { + // up to 18 digits, the fast path stores the accumulated value without + // an overflow check; a narrower number_integer_t/number_unsigned_t + // must still see the value rejected and converted as a float + using json32 = nlohmann::basic_json; + for (const char* n : + {"4294967295", "4294967296", "2147483647", "-2147483648", "-2147483649", "99999999999", "-99999999999" + }) + { + CAPTURE(n) + const std::string doc = std::string("[") + n + "]"; + const std::list chars(doc.begin(), doc.end()); + const json32 a = json32::parse(doc); + const json32 b = json32::parse(chars.begin(), chars.end()); + CHECK(a[0].type() == b[0].type()); + CHECK(a == b); + } + CHECK(json32::parse("[4294967295]")[0].is_number_unsigned()); + CHECK(json32::parse("[4294967296]")[0].is_number_float()); + CHECK(json32::parse("[-2147483648]")[0].is_number_integer()); + CHECK(json32::parse("[-2147483649]")[0].is_number_float()); + } + SECTION("token type classification") { CHECK((scan_string("0") == json::lexer::token_type::value_unsigned));