diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 641d93fa4..948c12dfd 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -149,10 +149,11 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) , decimal_point_char(static_cast(get_decimal_point())) + , discard_number_values(discard_number_values_) {} // deleted because of pointer members @@ -1279,6 +1280,38 @@ scan_number_done: // we are done scanning a number) unget(); + // If the caller does not need the converted value (only whether the + // input is syntactically valid; see json_sax_acceptor/accept()), an + // unsigned/integer token can be reported without calling + // strtoull()/strtoll() at all, *provided* we can already tell from + // the digit count alone that the conversion cannot overflow 64 bits. + // Such tokens are always finite and are accepted unconditionally by + // the parser regardless of their actual value (parser::sax_parse_internal() + // never checks finiteness for value_unsigned/value_integer), so the + // classification below is all that is needed. + // + // A decimal number with up to 18 digits is always representable in + // both std::uint64_t and std::int64_t (18 nines is ~1e18, well below + // both UINT64_MAX ~1.8e19 and INT64_MAX ~9.2e18), so strtoull()/strtoll() + // could not have set errno to ERANGE for it. Numbers with more digits + // (rare in practice) fall through to the exact code below, unchanged, + // so their handling -- including reclassification to value_float when + // the value overflows 64 bits, and rejection when it is not even + // finite as a double -- is bit-for-bit identical to before this + // optimization. + if (discard_number_values) + { + constexpr std::size_t safe_digit_count = 18; + if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) + { + return token_type::value_unsigned; + } + if (number_type == token_type::value_integer && token_buffer.size() - 1 <= safe_digit_count) + { + return token_type::value_integer; + } + } + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) errno = 0; @@ -1754,6 +1787,13 @@ scan_number_done: const char_int_type decimal_point_char = '.'; /// the position of the decimal point in the input std::size_t decimal_point_position = std::string::npos; + + /// whether the caller (e.g. accept()/json_sax_acceptor) only needs the + /// token classification and never looks at the converted numeric value; + /// when set, scan_number() may skip strtoull()/strtoll() for + /// value_unsigned/value_integer tokens whose digit count guarantees they + /// fit into 64 bits (see scan_number()) + const bool discard_number_values = false; }; } // namespace detail diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index ecc78eb72..a45ee4a0a 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -72,9 +72,10 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false) + const bool ignore_trailing_commas_ = false, + const bool discard_number_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments) + , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index be4ccb90f..6f9876478 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -164,11 +164,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false + const bool ignore_trailing_commas = false, + const bool discard_number_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); } private: @@ -4133,7 +4134,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief check if the input is valid JSON (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -4144,7 +4145,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT @@ -4153,7 +4154,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief generate SAX events diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 31bfb869d..90cda4105 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7932,10 +7932,11 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) , decimal_point_char(static_cast(get_decimal_point())) + , discard_number_values(discard_number_values_) {} // deleted because of pointer members @@ -9062,6 +9063,38 @@ scan_number_done: // we are done scanning a number) unget(); + // If the caller does not need the converted value (only whether the + // input is syntactically valid; see json_sax_acceptor/accept()), an + // unsigned/integer token can be reported without calling + // strtoull()/strtoll() at all, *provided* we can already tell from + // the digit count alone that the conversion cannot overflow 64 bits. + // Such tokens are always finite and are accepted unconditionally by + // the parser regardless of their actual value (parser::sax_parse_internal() + // never checks finiteness for value_unsigned/value_integer), so the + // classification below is all that is needed. + // + // A decimal number with up to 18 digits is always representable in + // both std::uint64_t and std::int64_t (18 nines is ~1e18, well below + // both UINT64_MAX ~1.8e19 and INT64_MAX ~9.2e18), so strtoull()/strtoll() + // could not have set errno to ERANGE for it. Numbers with more digits + // (rare in practice) fall through to the exact code below, unchanged, + // so their handling -- including reclassification to value_float when + // the value overflows 64 bits, and rejection when it is not even + // finite as a double -- is bit-for-bit identical to before this + // optimization. + if (discard_number_values) + { + constexpr std::size_t safe_digit_count = 18; + if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) + { + return token_type::value_unsigned; + } + if (number_type == token_type::value_integer && token_buffer.size() - 1 <= safe_digit_count) + { + return token_type::value_integer; + } + } + char* endptr = nullptr; // NOLINT(misc-const-correctness,cppcoreguidelines-pro-type-vararg,hicpp-vararg) errno = 0; @@ -9537,6 +9570,13 @@ scan_number_done: const char_int_type decimal_point_char = '.'; /// the position of the decimal point in the input std::size_t decimal_point_position = std::string::npos; + + /// whether the caller (e.g. accept()/json_sax_acceptor) only needs the + /// token classification and never looks at the converted numeric value; + /// when set, scan_number() may skip strtoull()/strtoll() for + /// value_unsigned/value_integer tokens whose digit count guarantees they + /// fit into 64 bits (see scan_number()) + const bool discard_number_values = false; }; } // namespace detail @@ -14043,9 +14083,10 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false) + const bool ignore_trailing_commas_ = false, + const bool discard_number_values_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments) + , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -21592,11 +21633,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false + const bool ignore_trailing_commas = false, + const bool discard_number_values = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); } private: @@ -25561,7 +25603,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief check if the input is valid JSON (iterator pair, or iterator+sentinel pair for C++20 ranges support) @@ -25572,7 +25614,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT @@ -25581,7 +25623,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool ignore_comments = false, const bool ignore_trailing_commas = false) { - return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); + return parser(i.get(), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); } /// @brief generate SAX events diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 8b3ea660e..344621f78 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -930,6 +930,94 @@ TEST_CASE("parser class") CHECK(accept_helper("+1") == false); CHECK(accept_helper("+0") == false); } + + SECTION("issue #5411 - skip conversion when accept() does not need the numeric value") + { + // lexer::scan_number() may skip strtoull()/strtoll() for + // value_unsigned/value_integer tokens when the caller (e.g. + // json::accept()) does not need the converted value, as long + // as the digit count alone guarantees no 64-bit overflow (see + // the "safe_digit_count" fast path in scan_number()). This + // differential test checks that json::accept() (which enables + // the fast path) and json::parse() (which never does) always + // agree, over a corpus that exercises both the fast path + // (<=18 digits) and the untouched, exact fallback path (>=19 + // digits) -- including reclassification of huge digit-only + // integers to a (possibly non-finite) floating-point value. + const std::vector> cases = + { + // normal small/large integers, both signs + {"0", true}, {"1", true}, {"-1", true}, {"42", true}, {"-42", true}, + {"123456789", true}, {"-123456789", true}, + + // digit-count boundary around the 18-digit safe cutoff (both signs) + {std::string(17, '9'), true}, + {std::string(18, '9'), true}, + {std::string(19, '9'), true}, + {std::string(20, '9'), true}, + {"-" + std::string(17, '9'), true}, + {"-" + std::string(18, '9'), true}, + {"-" + std::string(19, '9'), true}, + {"-" + std::string(20, '9'), true}, + + // 64-bit boundaries + {"9223372036854775807", true}, // INT64_MAX + {"-9223372036854775808", true}, // INT64_MIN + {"18446744073709551615", true}, // UINT64_MAX + {"18446744073709551616", true}, // UINT64_MAX + 1 (overflows uint64_t, finite double) + + // the 28-digit example from the issue: overflows uint64_t + // but is finite as a double, so the scanner reclassifies + // it to value_float and it is accepted + {"9999999999999999999999999999", true}, + + // huge digit-only integers that overflow even a double -> rejected + {std::string(309, '9'), false}, + {std::string(400, '9'), false}, + {"1" + std::string(400, '0'), false}, + + // 1e999 / 1e400 style overflow -> rejected + {"1e999", false}, + {"1e400", false}, + {"-1e999", false}, + {"1E999", false}, + + // values straddling DBL_MAX + {"1.7976931348623157e308", true}, // <= DBL_MAX, finite + {"1.7976931348623159e308", false}, // > DBL_MAX, overflows to inf + + // a mix of other valid/invalid numeric syntax + {"3.14159", true}, + {"-0.0", true}, + {"1.0e10", true}, + {"01", false}, + {"-", false}, + {"1.", false}, + {"1e", false}, + {"+1", false}, + }; + + for (const auto& c : cases) + { + const std::string& number = c.first; + const bool expected = c.second; + CAPTURE(number) + CAPTURE(expected) + + // accept() takes the fast path (skips conversion when possible) + CHECK(json::accept(number) == expected); + + // parse() always performs the full conversion; it must agree + json j; + CHECK_NOTHROW(json::parser(nlohmann::detail::input_adapter(number), nullptr, false).parse(true, j)); + CHECK(!j.is_discarded() == expected); + + // wrap in an array so get_token() is exercised beyond the + // very first (constructor-time) scan as well + const std::string wrapped = "[" + number + "," + number + "]"; + CHECK(json::accept(wrapped) == expected); + } + } } }