diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 14d14f42a..66cb14b4f 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -209,8 +209,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -268,10 +273,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) { // the token string of an error before the first token is everything // read so far @@ -388,6 +392,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -396,17 +402,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -419,14 +426,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -444,7 +453,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -488,7 +497,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -502,13 +512,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -520,7 +535,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -542,35 +557,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -638,10 +653,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -824,7 +836,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -860,7 +872,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -870,7 +882,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -894,7 +906,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -904,7 +916,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -914,7 +926,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -926,7 +938,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -936,7 +948,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -1427,7 +1439,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -1442,7 +1454,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -1450,7 +1462,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -1701,7 +1713,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -2033,6 +2045,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -2427,7 +2467,9 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -2512,11 +2554,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; DiscardValues, set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 5bf8f1b6b..023327166 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -72,14 +72,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -88,10 +88,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 4d4f0a1cd..7bed813df 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -150,7 +150,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -183,12 +183,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -3117,7 +3116,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 84720a816..4bca3a3a5 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13095,8 +13095,13 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/) @brief lexical analysis This class organizes the lexical analysis during JSON deserialization. + +DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only +accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of +parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower +on Apple M1. */ -template +template class lexer : public lexer_base { using number_integer_t = typename BasicJsonType::number_integer_t; @@ -13154,10 +13159,9 @@ class lexer : public lexer_base public: using token_type = typename lexer_base::token_type; - explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept + explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept : ia(std::move(adapter)) , ignore_comments(ignore_comments_) - , discard_number_values(discard_number_values_) { // the token string of an error before the first token is everything // read so far @@ -13274,6 +13278,8 @@ class lexer : public lexer_base checks if it is inside the range. If a violation was detected, set up an error message and return false. Otherwise, return true. + @param[in] keep whether the bytes are added to token_buffer (false for + accept(), see scan()) @param[in] ranges list of integers; interpreted as list of pairs of inclusive lower and upper bound, respectively @@ -13282,17 +13288,18 @@ class lexer : public lexer_base @return true if and only if no range violation was detected */ - bool next_byte_in_range(std::initializer_list ranges) + template + bool next_byte_in_range(std::integral_constant keep, std::initializer_list ranges) { JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6); - add(current); + add_string_char(keep, current); for (auto range = ranges.begin(); range != ranges.end(); ++range) { get(); if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions) { - add(current); + add_string_char(keep, current); } else { @@ -13305,14 +13312,16 @@ class lexer : public lexer_base } /// contiguous input: bulk-append the run of ordinary characters and complete - /// well-formed UTF-8 sequences starting at the current read position, then + /// well-formed UTF-8 sequences starting at the current read position (only + /// skip it if @a keep is false), then /// read the first byte that needs individual handling (the closing quote, an /// escape, a control character, or an ill-formed UTF-8 byte) like get(). /// That byte is taken from the buffer that was just scanned. Through get(), /// Apple Clang merged this read with the other get() calls of the scan /// function, which load the read position again right after it was stored /// (3% of json::accept() on jeopardy.json on Apple M1). - char_int_type get_after_string_run(std::true_type /*bulk*/) + template + char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant keep) { // a pending unget must be consumed through the normal path first if (next_unget) @@ -13330,7 +13339,7 @@ class lexer : public lexer_base const std::size_t pos = lexer_string_bulk_run(data, remaining); if (pos != 0) { - token_buffer.append(reinterpret_cast(data), pos); + append_string_run(keep, data, pos); bulk_consume(reinterpret_cast(data), pos); } if (pos == remaining) @@ -13374,7 +13383,8 @@ class lexer : public lexer_base } /// streaming input: no bulk fast path - char_int_type get_after_string_run(std::false_type /*bulk*/) + template + char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant /*keep*/) { return get(); } @@ -13388,13 +13398,18 @@ class lexer : public lexer_base may contain \0 bytes), and token_buffer.size() is the number of bytes in the string. + @param[in] keep whether the string is copied into token_buffer; if false + (accept(), see scan()), it is only checked, with the same + errors, and token_buffer stays empty + @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @note In case of errors, variable error_message contains a textual description. */ - token_type scan_string() + template + token_type scan_string(std::integral_constant keep) { // reset token_buffer (ignore opening quote) reset(); @@ -13406,7 +13421,7 @@ class lexer : public lexer_base { // bulk-consume ordinary characters from contiguous input, then // get the next character, the one that ends the run - switch (get_after_string_run(std::integral_constant())) + switch (get_after_string_run(std::integral_constant(), keep)) { // end of file while parsing the string case char_traits::eof(): @@ -13428,35 +13443,35 @@ class lexer : public lexer_base { // quotation mark case '\"': - add('\"'); + add_string_char(keep, '\"'); break; // reverse solidus case '\\': - add('\\'); + add_string_char(keep, '\\'); break; // solidus case '/': - add('/'); + add_string_char(keep, '/'); break; // backspace case 'b': - add('\b'); + add_string_char(keep, '\b'); break; // form feed case 'f': - add('\f'); + add_string_char(keep, '\f'); break; // line feed case 'n': - add('\n'); + add_string_char(keep, '\n'); break; // carriage return case 'r': - add('\r'); + add_string_char(keep, '\r'); break; // tab case 't': - add('\t'); + add_string_char(keep, '\t'); break; // unicode escapes @@ -13524,10 +13539,7 @@ class lexer : public lexer_base JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF); // translate codepoint into bytes - encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) - { - add(static_cast(byte)); - }); + add_codepoint(keep, codepoint); break; } @@ -13710,7 +13722,7 @@ class lexer : public lexer_base case 0x7E: case 0x7F: { - add(current); + add_string_char(keep, current); break; } @@ -13746,7 +13758,7 @@ class lexer : public lexer_base case 0xDE: case 0xDF: { - if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF}))) + if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF}))) { return token_type::parse_error; } @@ -13756,7 +13768,7 @@ class lexer : public lexer_base // U+0800..U+0FFF: bytes E0 A0..BF 80..BF case 0xE0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13780,7 +13792,7 @@ class lexer : public lexer_base case 0xEE: case 0xEF: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13790,7 +13802,7 @@ class lexer : public lexer_base // U+D000..U+D7FF: bytes ED 80..9F 80..BF case 0xED: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13800,7 +13812,7 @@ class lexer : public lexer_base // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF case 0xF0: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13812,7 +13824,7 @@ class lexer : public lexer_base case 0xF2: case 0xF3: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -13822,7 +13834,7 @@ class lexer : public lexer_base // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF case 0xF4: { - if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) { return token_type::parse_error; } @@ -14313,7 +14325,7 @@ scan_number_done: token_type convert_number(token_type number_type, std::size_t mantissa_end) { // accept() only needs to know whether the input is valid, so it sets - // discard_number_values (see json.hpp), and an integer token whose + // discard_values (see json.hpp), and an integer token whose // digit count shows that it fits is reported without calling // convert_integer(). A number with up to 18 digits always fits into // both std::uint64_t and std::int64_t (18 nines is about 1e18, below @@ -14328,7 +14340,7 @@ scan_number_done: // json_sax_acceptor, whose number callbacks discard their argument and // return true, and the parser rejects neither integers nor finite // floats. value_unsigned/value_integer are left unset here, so a caller - // that reads the converted value must not set discard_number_values. + // that reads the converted value must not set discard_values. // // On contiguous input, scan_number_bulk_contiguous() converts integer // tokens itself and does not pass them to this function, unless @@ -14336,7 +14348,7 @@ scan_number_done: // reached for input without bulk access (e.g. streams), with // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous() // falls back to scan_number(). - if (discard_number_values) + if (discard_values()) { constexpr std::size_t safe_digit_count = 18; if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count) @@ -14587,7 +14599,7 @@ scan_number_done: // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a // value that underflows is a finite number as well. Only a token that // may overflow has to be converted to find out. - if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits::max_exponent10)) + if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits::max_exponent10)) { value_float = 0; return true; @@ -14919,6 +14931,34 @@ scan_number_done: token_buffer.push_back(static_cast(c)); } + /// add a character of a string token; accept() checks the string but does + /// not keep its value (@a keep is false) + void add_string_char(std::true_type /*keep*/, char_int_type c) + { + token_buffer.push_back(static_cast(c)); + } + + static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {} + + /// add the UTF-8 bytes of a code point from a \\u escape to a string token + void add_codepoint(std::true_type /*keep*/, int codepoint) + { + encode_utf8(static_cast(codepoint), [this](std::uint32_t byte) + { + add(static_cast(byte)); + }); + } + + static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {} + + /// append a run of ordinary string characters (see get_after_string_run()) + void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n) + { + token_buffer.append(reinterpret_cast(data), n); + } + + static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {} + public: ///////////////////// // value getters @@ -15313,7 +15353,9 @@ scan_number_done: // string case '\"': - return scan_string(); + // accept() (DiscardValues) checks strings without copying them + // into token_buffer + return scan_string(keep_strings {}); // number case '-': @@ -15398,11 +15440,19 @@ scan_number_done: std::size_t decimal_point_position = std::string::npos; /// whether the caller only needs the token types and never looks at the - /// converted numeric values; set only by accept(), which parses through - /// json_sax_acceptor. When set, convert_number() skips converting integer - /// tokens whose digit count guarantees that they fit into 64 bits (see - /// there) - const bool discard_number_values = false; + /// values of number and string tokens; DiscardValues, set only by accept(), which parses + /// through json_sax_acceptor. When set, convert_number() skips converting + /// integer tokens whose digit count guarantees that they fit into 64 bits + /// (see there), floats are converted only when they may overflow (see + /// convert_accumulated_float()), and strings are checked without being + /// copied into token_buffer (see scan()) + static bool discard_values() noexcept + { + return DiscardValues; + } + + /// the tag of scan_string(): whether string tokens are kept + using keep_strings = typename std::conditional::type; }; } // namespace detail @@ -23023,14 +23073,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read is reported as SAX events. */ -template +template class parser { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; using string_t = typename BasicJsonType::string_t; - using lexer_t = lexer; + using lexer_t = lexer; using token_type = typename lexer_t::token_type; public: @@ -23039,10 +23089,9 @@ class parser parser_callback_t cb = nullptr, const bool allow_exceptions_ = true, const bool ignore_comments = false, - const bool ignore_trailing_commas_ = false, - const bool discard_number_values_ = false) + const bool ignore_trailing_commas_ = false) : callback(std::move(cb)) - , m_lexer(std::move(adapter), ignore_comments, discard_number_values_) + , m_lexer(std::move(adapter), ignore_comments) , allow_exceptions(allow_exceptions_) , ignore_trailing_commas(ignore_trailing_commas_) { @@ -37439,7 +37488,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // can be restored when json_pointer backwards compatibility is removed // friend ::nlohmann::json_pointer; - template + template friend class ::nlohmann::detail::parser; friend ::nlohmann::detail::serializer; template @@ -37472,12 +37521,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::parser_callback_tcb = nullptr, const bool allow_exceptions = true, const bool ignore_comments = false, - const bool ignore_trailing_commas = false, - const bool discard_number_values = false + const bool ignore_trailing_commas = false ) { return ::nlohmann::detail::parser(std::move(adapter), - std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values); + std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas); } private: @@ -40406,7 +40454,9 @@ public: const bool ignore_comments, const bool ignore_trailing_commas) { - return parser(std::forward(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true); + // the parser and lexer that discard the values of tokens (see lexer) + return ::nlohmann::detail::parser::type, true>(std::forward(adapter), + nullptr, false, ignore_comments, ignore_trailing_commas).accept(true); } /// generate SAX events from the input behind an already created adapter diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 090ab4b41..987e25858 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2982,6 +2982,67 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit") } } +TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them") +{ + // accept() scans strings and object keys with a separate instantiation of + // the string scanner that does not copy them into the token buffer; it + // must accept and reject exactly what parse() does + const std::vector pieces = + { + "", "abc", R"(\"\\\/\b\f\n\r\t)", R"(\u0041\u00e4\u20AC)", "\\ud83d\\ude00", "\\uD83D\\uDE00", + "\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\", + "\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80", + "\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0') + }; + + for (std::size_t run = 0; run <= 70; ++run) + { + for (const std::string& piece : pieces) + { + // a run of ordinary characters puts the piece at every offset of the bulk scanners + const std::string str = std::string(run, 'a') + piece; + std::string opened = "\""; + opened += str; + std::string closed = opened; + closed += '"'; + std::vector texts; + texts.push_back(closed); + texts.emplace_back("["); + texts.back() += closed; + texts.back() += ']'; + texts.emplace_back("{"); + texts.back() += closed; + texts.back() += ":1}"; + texts.emplace_back("{\"k\":"); + texts.back() += closed; + texts.back() += ','; + texts.back() += closed; + texts.back() += ":["; + texts.back() += closed; + texts.back() += "]}"; + texts.emplace_back("["); + texts.back() += opened; + for (const std::string& text : texts) + { + CAPTURE(text) + for (int options = 0; options < 4; ++options) + { + const bool ignore_comments = (options & 1) != 0; + const bool ignore_trailing_commas = (options & 2) != 0; + const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded(); + CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses); + + std::istringstream stream(text); + CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses); + + const std::list chars(text.begin(), text.end()); + CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses); + } + } + } + } +} + namespace { // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example