From ddd9c5b0be81cbe588fbba2300182a6e7d8692d6 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 19 Aug 2026 20:44:07 +0200 Subject: [PATCH] Restore the column when ungetting a newline The contiguous number fast path never reads the character that terminates a number token, while scan_number() reads it and then ungets it. When that character is a newline, get() has already cleared chars_read_current_line, and unget() could only restore lines_read - leaving the column at 0. The two paths therefore reported different columns for the same document: json::parse("[01\n]") -> line 1, column 3 json::parse(stringstream) -> line 1, column 0 Remember the column the newline was read at so unget() can restore it. Both paths now report the position the offending token actually starts at, which also fixes the pre-existing column-0 artifact for streaming input. Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/lexer.hpp | 15 +++++++ tests/src/unit-class_lexer.cpp | 56 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index d1bcf5d97..8574b911a 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -1615,6 +1615,9 @@ scan_number_done: if (current == '\n') { ++position.lines_read; + // remember the column the newline was read at: chars_read_current_line + // is about to be cleared, and a matching unget() cannot reconstruct it + chars_read_before_newline = position.chars_read_current_line; position.chars_read_current_line = 0; } @@ -1648,12 +1651,20 @@ scan_number_done: --position.chars_read_total; // in case we "unget" a newline, we have to also decrement the lines_read + // and restore the column that get() cleared when it saw the newline; + // chars_read_current_line == 0 can only mean the last get() read one if (position.chars_read_current_line == 0) { if (position.lines_read > 0) { --position.lines_read; } + + // chars_read_before_newline counts the newline itself, which is the + // character being ungotten, hence the -1 + position.chars_read_current_line = (chars_read_before_newline > 0) + ? chars_read_before_newline - 1 + : 0; } else { @@ -1927,6 +1938,10 @@ scan_number_done: /// the start position of the current token position_t position {}; + /// the value chars_read_current_line had when the last newline was read, so + /// that unget() can restore the column instead of leaving it at 0 + std::size_t chars_read_before_newline = 0; + /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 490397740..0dd456b8c 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -299,4 +299,60 @@ TEST_CASE("lexer number fast path") CHECK_FALSE(json::accept(ss)); } } + + SECTION("error positions match the streaming path") + { + // Rejecting identically is not enough: the fast path must also report the + // error at the same position as the byte path. A number directly followed + // by a newline is the interesting case, because the byte path reaches the + // newline (which resets the column) and then ungets it. + // returns the parse_error message, or "" if the document parsed + const auto contiguous_error = [](const std::string & doc) + { + try + { + const json j = json::parse(doc); + static_cast(j); + } + catch (const json::parse_error& e) + { + return std::string(e.what()); + } + return std::string(); + }; + const auto streaming_error = [](const std::string & doc) + { + try + { + std::stringstream ss(doc); + const json j = json::parse(ss); + static_cast(j); + } + catch (const json::parse_error& e) + { + return std::string(e.what()); + } + return std::string(); + }; + + for (const char* bad : + {"[01\n]", "[00\n]", "[-01\n]", "{1\n}", "[1\n2]", "[1.2.3\n]", + "[1 \n2]", "[\n1\n2]", "1\n2", "[01\r\n]", "[1e\n]", "[-\n]" + }) + { + CAPTURE(bad); + const std::string doc = bad; + const std::string contiguous_what = contiguous_error(doc); + + CHECK_FALSE(contiguous_what.empty()); + CHECK(contiguous_what == streaming_error(doc)); + } + + // the column must be the one the offending token actually starts at, + // not the 0 that an unget() across the newline used to leave behind + CHECK_THROWS_WITH_AS(json::parse("[01\n]"), + "[json.exception.parse_error.101] parse error at line 1, column 3: " + "syntax error while parsing array - unexpected number literal; expected ']'", + json::parse_error&); + } }