Fix start_pos() for strings containing escape sequences (#5361)

The diagnostic position of a string value was derived by subtracting the
parsed value's length from the end position. Escape sequences make the
source token longer than the value it parses to, so the reported start
position landed inside the string, one byte off per escape sequence:

    input: {"a":"\n\n\n\n\n\n"}
      start_pos() == 11, so the reported range covered  n\n\n\n"
      instead of the documented "\n\n\n\n\n\n"

This contradicts the documented behavior of start_pos(), which is the
position of the opening quote, and it also corrupted the "(bytes N-M)"
part of JSON_DIAGNOSTICS exception messages. Strings with multi-byte
UTF-8 but no escapes were unaffected, which is why this went unnoticed.

Record the offset of the token in the lexer when it starts scanning and
use that, instead of reconstructing it from the parsed value. Booleans,
null and numbers already reported correct positions and are unchanged.

The new lexer member and accessor are compiled only when
JSON_DIAGNOSTIC_POSITIONS is enabled, which is already part of the ABI
tag, so the default build is unaffected.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/json_sax.hpp

@@ -370,8 +370,10 @@ class json_sax_dom_parser
 
                 case value_t::string:
                 {
-                    // include the length of the quotes, which is 2
-                    v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
+                    // escape sequences make the token longer than the value it
+                    // parses to, so the start position cannot be derived from
+                    // the value; use the offset the lexer recorded instead
+                    v.start_position = m_lexer_ref->get_token_start_position();
                     break;
                 }
 
@@ -769,8 +771,10 @@ class json_sax_dom_callback_parser
 
                 case value_t::string:
                 {
-                    // include the length of the quotes, which is 2
-                    v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
+                    // escape sequences make the token longer than the value it
+                    // parses to, so the start position cannot be derived from
+                    // the value; use the offset the lexer recorded instead
+                    v.start_position = m_lexer_ref->get_token_start_position();
                     break;
                 }
 

include/nlohmann/detail/input/lexer.hpp

@@ -1357,6 +1357,11 @@ scan_number_done:
         token_buffer.clear();
         decimal_point_position = std::string::npos;
 
+#if JSON_DIAGNOSTIC_POSITIONS
+        // the first character of the token has already been read, hence the -1
+        token_start_position = position.chars_read_total - 1;
+#endif
+
         note_token_start(std::integral_constant<bool, lazy_token_string> {});
     }
 
@@ -1519,6 +1524,15 @@ scan_number_done:
         return position;
     }
 
+#if JSON_DIAGNOSTIC_POSITIONS
+    /// return the offset of the first character of the last read token; unlike
+    /// the token's parsed value, this accounts for escape sequences
+    constexpr std::size_t get_token_start_position() const noexcept
+    {
+        return token_start_position;
+    }
+#endif
+
     /// seekable adapter: rebuild the last read token from the input on demand
     const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
     {
@@ -1719,6 +1733,12 @@ scan_number_done:
     /// the last read token on error for seekable adapters (see collect_token_chars)
     std::size_t token_string_start = 0;
 
+#if JSON_DIAGNOSTIC_POSITIONS
+    /// start offset of the current token within the input, used to report
+    /// diagnostic positions (see reset())
+    std::size_t token_start_position = 0;
+#endif
+
     /// buffer for variable-length tokens (numbers, strings)
     string_t token_buffer {};
 

single_include/nlohmann/json.hpp

@@ -9075,6 +9075,11 @@ scan_number_done:
         token_buffer.clear();
         decimal_point_position = std::string::npos;
 
+#if JSON_DIAGNOSTIC_POSITIONS
+        // the first character of the token has already been read, hence the -1
+        token_start_position = position.chars_read_total - 1;
+#endif
+
         note_token_start(std::integral_constant<bool, lazy_token_string> {});
     }
 
@@ -9237,6 +9242,15 @@ scan_number_done:
         return position;
     }
 
+#if JSON_DIAGNOSTIC_POSITIONS
+    /// return the offset of the first character of the last read token; unlike
+    /// the token's parsed value, this accounts for escape sequences
+    constexpr std::size_t get_token_start_position() const noexcept
+    {
+        return token_start_position;
+    }
+#endif
+
     /// seekable adapter: rebuild the last read token from the input on demand
     const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
     {
@@ -9437,6 +9451,12 @@ scan_number_done:
     /// the last read token on error for seekable adapters (see collect_token_chars)
     std::size_t token_string_start = 0;
 
+#if JSON_DIAGNOSTIC_POSITIONS
+    /// start offset of the current token within the input, used to report
+    /// diagnostic positions (see reset())
+    std::size_t token_start_position = 0;
+#endif
+
     /// buffer for variable-length tokens (numbers, strings)
     string_t token_buffer {};
 
@@ -9813,8 +9833,10 @@ class json_sax_dom_parser
 
                 case value_t::string:
                 {
-                    // include the length of the quotes, which is 2
-                    v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
+                    // escape sequences make the token longer than the value it
+                    // parses to, so the start position cannot be derived from
+                    // the value; use the offset the lexer recorded instead
+                    v.start_position = m_lexer_ref->get_token_start_position();
                     break;
                 }
 
@@ -10212,8 +10234,10 @@ class json_sax_dom_callback_parser
 
                 case value_t::string:
                 {
-                    // include the length of the quotes, which is 2
-                    v.start_position = v.end_position - v.m_data.m_value.string->size() - 2;
+                    // escape sequences make the token longer than the value it
+                    // parses to, so the start position cannot be derived from
+                    // the value; use the offset the lexer recorded instead
+                    v.start_position = m_lexer_ref->get_token_start_position();
                     break;
                 }
 

tests/src/unit-diagnostic-positions.cpp

@@ -38,6 +38,36 @@ TEST_CASE("Better diagnostics with positions")
                              "[json.exception.type_error.302] type must be number, but is string", json::type_error);
     }
 
+    SECTION("positions of strings containing escape sequences")
+    {
+        // escape sequences make the token longer than the string it parses to,
+        // so the positions must not be derived from the parsed value's length
+        const auto check = [](const std::string & text, const std::string & token)
+        {
+            CAPTURE(text)
+            CAPTURE(token)
+            const json j = json::parse(text);
+            const json& v = j.at("a");
+            CHECK(text.substr(v.start_pos(), v.end_pos() - v.start_pos()) == token);
+        };
+
+        check(R"({"a":"plain"})", R"("plain")");
+        check(R"({"a":"tab\there"})", R"("tab\there")");
+        check(R"({"a":"\n\n\n\n\n\n"})", R"("\n\n\n\n\n\n")");
+        check(R"({"a":"\""})", R"("\"")");
+        check(R"({"a":"\\"})", R"("\\")");
+        check(R"({"a":"é"})", R"("é")");
+        check(R"({"a":"🌞"})", R"("🌞")");
+        check("{\"a\":\"\xc3\xa9\"}", "\"\xc3\xa9\"");  // multi-byte UTF-8, no escapes
+
+        // a string at the root, where an escape would otherwise push the
+        // reported start position past the opening quote
+        const std::string root = R"("a\tb")";
+        const json j = json::parse(root);
+        CHECK(j.start_pos() == 0);
+        CHECK(j.end_pos() == root.size());
+    }
+
     SECTION("JSON patch add to primitive parent (#4292)")
     {
         // the JSON Patch "add" target /foo/bar/baz has a string parent