diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 4c5f24a7b..8a98ee728 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -370,8 +370,10 @@ class json_sax_dom_parser case value_t::string: { - // include the length of the quotes, which is 2 - v.start_position = v.end_position - v.m_data.m_value.string->size() - 2; + // escape sequences make the token longer than the value it + // parses to, so the start position cannot be derived from + // the value; use the offset the lexer recorded instead + v.start_position = m_lexer_ref->get_token_start_position(); break; } @@ -769,8 +771,10 @@ class json_sax_dom_callback_parser case value_t::string: { - // include the length of the quotes, which is 2 - v.start_position = v.end_position - v.m_data.m_value.string->size() - 2; + // escape sequences make the token longer than the value it + // parses to, so the start position cannot be derived from + // the value; use the offset the lexer recorded instead + v.start_position = m_lexer_ref->get_token_start_position(); break; } diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 6318994e6..641d93fa4 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -1357,6 +1357,11 @@ scan_number_done: token_buffer.clear(); decimal_point_position = std::string::npos; +#if JSON_DIAGNOSTIC_POSITIONS + // the first character of the token has already been read, hence the -1 + token_start_position = position.chars_read_total - 1; +#endif + note_token_start(std::integral_constant {}); } @@ -1519,6 +1524,15 @@ scan_number_done: return position; } +#if JSON_DIAGNOSTIC_POSITIONS + /// return the offset of the first character of the last read token; unlike + /// the token's parsed value, this accounts for escape sequences + constexpr std::size_t get_token_start_position() const noexcept + { + return token_start_position; + } +#endif + /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -1719,6 +1733,12 @@ scan_number_done: /// the last read token on error for seekable adapters (see collect_token_chars) std::size_t token_string_start = 0; +#if JSON_DIAGNOSTIC_POSITIONS + /// start offset of the current token within the input, used to report + /// diagnostic positions (see reset()) + std::size_t token_start_position = 0; +#endif + /// buffer for variable-length tokens (numbers, strings) string_t token_buffer {}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index c1b178038..fa6199da1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9075,6 +9075,11 @@ scan_number_done: token_buffer.clear(); decimal_point_position = std::string::npos; +#if JSON_DIAGNOSTIC_POSITIONS + // the first character of the token has already been read, hence the -1 + token_start_position = position.chars_read_total - 1; +#endif + note_token_start(std::integral_constant {}); } @@ -9237,6 +9242,15 @@ scan_number_done: return position; } +#if JSON_DIAGNOSTIC_POSITIONS + /// return the offset of the first character of the last read token; unlike + /// the token's parsed value, this accounts for escape sequences + constexpr std::size_t get_token_start_position() const noexcept + { + return token_start_position; + } +#endif + /// seekable adapter: rebuild the last read token from the input on demand const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const { @@ -9437,6 +9451,12 @@ scan_number_done: /// the last read token on error for seekable adapters (see collect_token_chars) std::size_t token_string_start = 0; +#if JSON_DIAGNOSTIC_POSITIONS + /// start offset of the current token within the input, used to report + /// diagnostic positions (see reset()) + std::size_t token_start_position = 0; +#endif + /// buffer for variable-length tokens (numbers, strings) string_t token_buffer {}; @@ -9813,8 +9833,10 @@ class json_sax_dom_parser case value_t::string: { - // include the length of the quotes, which is 2 - v.start_position = v.end_position - v.m_data.m_value.string->size() - 2; + // escape sequences make the token longer than the value it + // parses to, so the start position cannot be derived from + // the value; use the offset the lexer recorded instead + v.start_position = m_lexer_ref->get_token_start_position(); break; } @@ -10212,8 +10234,10 @@ class json_sax_dom_callback_parser case value_t::string: { - // include the length of the quotes, which is 2 - v.start_position = v.end_position - v.m_data.m_value.string->size() - 2; + // escape sequences make the token longer than the value it + // parses to, so the start position cannot be derived from + // the value; use the offset the lexer recorded instead + v.start_position = m_lexer_ref->get_token_start_position(); break; } diff --git a/tests/src/unit-diagnostic-positions.cpp b/tests/src/unit-diagnostic-positions.cpp index 59c21dc59..ad9527540 100644 --- a/tests/src/unit-diagnostic-positions.cpp +++ b/tests/src/unit-diagnostic-positions.cpp @@ -38,6 +38,36 @@ TEST_CASE("Better diagnostics with positions") "[json.exception.type_error.302] type must be number, but is string", json::type_error); } + SECTION("positions of strings containing escape sequences") + { + // escape sequences make the token longer than the string it parses to, + // so the positions must not be derived from the parsed value's length + const auto check = [](const std::string & text, const std::string & token) + { + CAPTURE(text) + CAPTURE(token) + const json j = json::parse(text); + const json& v = j.at("a"); + CHECK(text.substr(v.start_pos(), v.end_pos() - v.start_pos()) == token); + }; + + check(R"({"a":"plain"})", R"("plain")"); + check(R"({"a":"tab\there"})", R"("tab\there")"); + check(R"({"a":"\n\n\n\n\n\n"})", R"("\n\n\n\n\n\n")"); + check(R"({"a":"\""})", R"("\"")"); + check(R"({"a":"\\"})", R"("\\")"); + check(R"({"a":"é"})", R"("é")"); + check(R"({"a":"🌞"})", R"("🌞")"); + check("{\"a\":\"\xc3\xa9\"}", "\"\xc3\xa9\""); // multi-byte UTF-8, no escapes + + // a string at the root, where an escape would otherwise push the + // reported start position past the opening quote + const std::string root = R"("a\tb")"; + const json j = json::parse(root); + CHECK(j.start_pos() == 0); + CHECK(j.end_pos() == root.size()); + } + SECTION("JSON patch add to primitive parent (#4292)") { // the JSON Patch "add" target /foo/bar/baz has a string parent