Restore the column when ungetting a newline

The contiguous number fast path never reads the character that terminates
a number token, while scan_number() reads it and then ungets it. When that
character is a newline, get() has already cleared chars_read_current_line,
and unget() could only restore lines_read - leaving the column at 0. The
two paths therefore reported different columns for the same document:

    json::parse("[01\n]")      -> line 1, column 3
    json::parse(stringstream)  -> line 1, column 0

Remember the column the newline was read at so unget() can restore it.
Both paths now report the position the offending token actually starts at,
which also fixes the pre-existing column-0 artifact for streaming input.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/lexer.hpp

@@ -1615,6 +1615,9 @@ scan_number_done:
         if (current == '\n')
         {
             ++position.lines_read;
+            // remember the column the newline was read at: chars_read_current_line
+            // is about to be cleared, and a matching unget() cannot reconstruct it
+            chars_read_before_newline = position.chars_read_current_line;
             position.chars_read_current_line = 0;
         }
 
@@ -1648,12 +1651,20 @@ scan_number_done:
         --position.chars_read_total;
 
         // in case we "unget" a newline, we have to also decrement the lines_read
+        // and restore the column that get() cleared when it saw the newline;
+        // chars_read_current_line == 0 can only mean the last get() read one
         if (position.chars_read_current_line == 0)
         {
             if (position.lines_read > 0)
             {
                 --position.lines_read;
             }
+
+            // chars_read_before_newline counts the newline itself, which is the
+            // character being ungotten, hence the -1
+            position.chars_read_current_line = (chars_read_before_newline > 0)
+                                               ? chars_read_before_newline - 1
+                                               : 0;
         }
         else
         {
@@ -1927,6 +1938,10 @@ scan_number_done:
     /// the start position of the current token
     position_t position {};
 
+    /// the value chars_read_current_line had when the last newline was read, so
+    /// that unget() can restore the column instead of leaving it at 0
+    std::size_t chars_read_before_newline = 0;
+
     /// raw input token string for error messages; only populated for streaming
     /// adapters (seekable adapters reconstruct it lazily via token_string_start)
     std::vector<char_type> token_string {};

tests/src/unit-class_lexer.cpp

@@ -299,4 +299,60 @@ TEST_CASE("lexer number fast path")
             CHECK_FALSE(json::accept(ss));
         }
     }
+
+    SECTION("error positions match the streaming path")
+    {
+        // Rejecting identically is not enough: the fast path must also report the
+        // error at the same position as the byte path. A number directly followed
+        // by a newline is the interesting case, because the byte path reaches the
+        // newline (which resets the column) and then ungets it.
+        // returns the parse_error message, or "" if the document parsed
+        const auto contiguous_error = [](const std::string & doc)
+        {
+            try
+            {
+                const json j = json::parse(doc);
+                static_cast<void>(j);
+            }
+            catch (const json::parse_error& e)
+            {
+                return std::string(e.what());
+            }
+            return std::string();
+        };
+        const auto streaming_error = [](const std::string & doc)
+        {
+            try
+            {
+                std::stringstream ss(doc);
+                const json j = json::parse(ss);
+                static_cast<void>(j);
+            }
+            catch (const json::parse_error& e)
+            {
+                return std::string(e.what());
+            }
+            return std::string();
+        };
+
+        for (const char* bad :
+                {"[01\n]", "[00\n]", "[-01\n]", "{1\n}", "[1\n2]", "[1.2.3\n]",
+                 "[1 \n2]", "[\n1\n2]", "1\n2", "[01\r\n]", "[1e\n]", "[-\n]"
+                })
+        {
+            CAPTURE(bad);
+            const std::string doc = bad;
+            const std::string contiguous_what = contiguous_error(doc);
+
+            CHECK_FALSE(contiguous_what.empty());
+            CHECK(contiguous_what == streaming_error(doc));
+        }
+
+        // the column must be the one the offending token actually starts at,
+        // not the 0 that an unget() across the newline used to leave behind
+        CHECK_THROWS_WITH_AS(json::parse("[01\n]"),
+                             "[json.exception.parse_error.101] parse error at line 1, column 3: "
+                             "syntax error while parsing array - unexpected number literal; expected ']'",
+                             json::parse_error&);
+    }
 }