From 14913168dd155444e424a3100976395060b56fe1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 31 Oct 2026 20:22:03 +0100 Subject: [PATCH] accept(): its own lexer instead of a run-time choice in scan() The lexer chose between the copying and the checking string scan at run time in scan(). Both scans were then inlined into the scan() of parse() as well, which made json::parse() of jeopardy.json indented by 4 2 % slower on Apple M1 (same instruction count). Whether token values are discarded is now a template parameter (DiscardValues) of lexer and parser, false by default; only accept() instantiates the parser with true. parse() gets a lexer with the copying scan only, and without the run-time checks of the number paths. Apple M1, A/B against develop, 6 code layouts: parse() of indented jeopardy 1.020 -> 1.003, twitter 1.006 -> 1.005; accept() twitter 1.012 -> 1.002, tree-pretty 1.016 -> 1.004, mesh 1.000 -> 0.940, twitter_min 0.990 -> 1.009, citm_catalog 0.937 -> 0.950, gsoc-2018 0.838 -> 0.836. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 32 +++++++++------ include/nlohmann/detail/input/parser.hpp | 9 ++-- include/nlohmann/json.hpp | 11 ++--- single_include/nlohmann/json.hpp | 52 ++++++++++++++---------- 4 files changed, 60 insertions(+), 44 deletions(-) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index d9e9a726d..66cb14b4f 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -209,8 +209,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -268,10 +273,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_values(discard_values_) { // the token string of an error before the first token is everything // read so far @@ -1458,7 +1462,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -1709,7 +1713,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -2463,11 +2467,9 @@ scan_number_done: // string case '\"': - // accept() (discard_values) checks strings without copying them - // into token_buffer. The choice is made once per string: a - // check inside the string loop made number-heavy documents up - // to 3 % slower (mesh.json, Apple M1) - return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {}); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -2552,13 +2554,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// values of number and string tokens; set only by accept(), which parses + /// values of number and string tokens; DiscardValues, set only by accept(), which parses /// through json_sax_acceptor. When set, convert_number() skips converting /// integer tokens whose digit count guarantees that they fit into 64 bits /// (see there), floats are converted only when they may overflow (see /// convert_accumulated_float()), and strings are checked without being /// copied into token_buffer (see scan()) - const bool discard_values = false; + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index e49c81a03..023327166 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -72,14 +72,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -88,10 +88,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 15510ac16..7bed813df 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -150,7 +150,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -183,12 +183,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -3117,7 +3116,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3fb14819f..68ef29728 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13065,8 +13065,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -13124,10 +13129,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_values(discard_values_) { // the token string of an error before the first token is everything // read so far @@ -14314,7 +14318,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -14565,7 +14569,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -15319,11 +15323,9 @@ scan_number_done: // string case '\"': - // accept() (discard_values) checks strings without copying them - // into token_buffer. The choice is made once per string: a - // check inside the string loop made number-heavy documents up - // to 3 % slower (mesh.json, Apple M1) - return discard_values ? scan_string(std::false_type {}) : scan_string(std::true_type {}); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -15408,13 +15410,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// values of number and string tokens; set only by accept(), which parses + /// values of number and string tokens; DiscardValues, set only by accept(), which parses /// through json_sax_acceptor. When set, convert_number() skips converting /// integer tokens whose digit count guarantees that they fit into 64 bits /// (see there), floats are converted only when they may overflow (see /// convert_accumulated_float()), and strings are checked without being /// copied into token_buffer (see scan()) - const bool discard_values = false; + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail @@ -22606,14 +22614,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -22622,10 +22630,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -36988,7 +36995,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -37021,12 +37028,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -39955,7 +39961,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter