diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index be58b1110..5a41b0c40 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -161,8 +161,18 @@ class iterator_input_adapter public: using char_type = typename std::iterator_traits::value_type; + // Whether the lexer may reconstruct already-consumed input on demand (for + // diagnostics) instead of copying every scanned character eagerly. This is + // only sound for multi-pass, randomly-addressable byte input: the iterator + // must be random-access (so the consumed prefix can be revisited in O(1)) + // and each element must map 1:1 to an input byte (wide inputs are wrapped + // in wide_string_input_adapter, which does not expose this). + static constexpr bool supports_seek = + std::is_same::iterator_category, std::random_access_iterator_tag>::value + && sizeof(char_type) == 1; + iterator_input_adapter(IteratorType first, IteratorType last) - : current(std::move(first)), end(std::move(last)) + : begin(first), current(std::move(first)), end(std::move(last)) {} typename char_traits::int_type get_character() @@ -177,6 +187,22 @@ class iterator_input_adapter return char_traits::eof(); } + // number of characters consumed from the input so far + std::size_t get_consumed_count() const + { + return static_cast(std::distance(begin, current)); + } + + // append the already-consumed characters in the half-open range + // [first_index, last_index) to @a out; only valid when supports_seek + template + void copy_consumed_range(std::size_t first_index, std::size_t last_index, ContainerType& out) const + { + const auto from = std::next(begin, static_cast::difference_type>(first_index)); + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + out.insert(out.end(), from, to); + } + // for general iterators, we cannot really do something better than falling back to processing the range one-by-one template std::size_t get_elements(T* dest, std::size_t count = 1) @@ -198,6 +224,7 @@ class iterator_input_adapter } private: + IteratorType begin; IteratorType current; IteratorType end; diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 4742d889b..6318994e6 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -103,6 +103,28 @@ class lexer_base } } }; + +// Detect whether an input adapter can reconstruct already-consumed input on +// demand (see iterator_input_adapter::supports_seek). Adapters that do not +// expose the flag - e.g. file, stream, wide-string, and user-defined adapters - +// are treated as non-seekable streaming input, for which the lexer keeps +// copying every scanned character eagerly. The value is read via tag dispatch +// on is_detected so the flag is only referenced for adapters that provide it. +template +using detect_supports_seek = decltype(InputAdapterType::supports_seek); + +template +constexpr bool input_adapter_supports_seek(std::true_type /*detected*/) +{ + return InputAdapterType::supports_seek; +} + +template +constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -118,6 +140,12 @@ class lexer : public lexer_base using char_type = typename InputAdapterType::char_type; using char_int_type = typename char_traits::int_type; + /// whether the last read token can be reconstructed from the input adapter + /// on demand (in error paths) instead of being copied on every scanned + /// character; see input_adapter_supports_seek + static constexpr bool lazy_token_string = + input_adapter_supports_seek(is_detected {}); + public: using token_type = typename lexer_base::token_type; @@ -1327,8 +1355,24 @@ scan_number_done: void reset() noexcept { token_buffer.clear(); - token_string.clear(); decimal_point_position = std::string::npos; + + note_token_start(std::integral_constant {}); + } + + /// seekable adapter: remember where the current token starts so it can be + /// reconstructed from the input on error; current has already been + /// consumed, hence the -1 + void note_token_start(std::true_type /*lazy*/) noexcept + { + token_string_start = ia.get_consumed_count() - 1; + } + + /// streaming adapter: start copying the token eagerly, beginning with the + /// already-read first character + void note_token_start(std::false_type /*lazy*/) noexcept + { + token_string.clear(); token_string.push_back(char_traits::to_char_type(current)); } @@ -1357,10 +1401,9 @@ scan_number_done: current = ia.get_character(); } - if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) - { - token_string.push_back(char_traits::to_char_type(current)); - } + // seekable adapters reconstruct the token lazily on error (see + // get_token_string), so the eager per-character copy is skipped + capture_char(std::integral_constant {}); if (current == '\n') { @@ -1371,6 +1414,18 @@ scan_number_done: return current; } + /// seekable adapter: nothing to capture, the token is rebuilt on error + void capture_char(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: copy the scanned character into token_string + void capture_char(std::false_type /*lazy*/) + { + if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) + { + token_string.push_back(char_traits::to_char_type(current)); + } + } + /*! @brief unget current character (read it again on next get) @@ -1398,6 +1453,15 @@ scan_number_done: --position.chars_read_current_line; } + uncapture_char(std::integral_constant {}); + } + + /// seekable adapter: nothing was captured, so nothing to undo + void uncapture_char(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: drop the character copied by the matching get() + void uncapture_char(std::false_type /*lazy*/) + { if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) { JSON_ASSERT(!token_string.empty()); @@ -1455,14 +1519,38 @@ scan_number_done: return position; } + /// seekable adapter: rebuild the last read token from the input on demand + const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const + { + // a pending unget of a real (non-EOF) character means that character + // was consumed from the input but is not part of the token; EOF is + // never consumed, so it must not be subtracted (mirrors unget()) + const bool pending_real_unget = next_unget && current != char_traits::eof(); + const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u); + if (JSON_HEDLEY_LIKELY(stop >= token_string_start)) + { + ia.copy_consumed_range(token_string_start, stop, out); + } + return out; + } + + /// streaming adapter: the token was copied eagerly while scanning + const std::vector& collect_token_chars(std::vector& /*out*/, std::false_type /*lazy*/) const + { + return token_string; + } + /// return the last read token (for errors only). Will never contain EOF /// (an arbitrary value that is not a valid char value, often -1), because /// 255 may legitimately occur. May contain NUL, which should be escaped. std::string get_token_string() const { + std::vector reconstructed; + const std::vector& chars = collect_token_chars(reconstructed, std::integral_constant {}); + // escape control characters std::string result; - for (const auto c : token_string) + for (const auto c : chars) { if (static_cast(c) <= '\x1F') { @@ -1623,9 +1711,14 @@ scan_number_done: /// the start position of the current token position_t position {}; - /// raw input token string (for error messages) + /// raw input token string for error messages; only populated for streaming + /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; + /// start offset of the current token within the input, used to reconstruct + /// the last read token on error for seekable adapters (see collect_token_chars) + std::size_t token_string_start = 0; + /// buffer for variable-length tokens (numbers, strings) string_t token_buffer {}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1a2b3d357..badcb4f47 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -6993,8 +6993,18 @@ class iterator_input_adapter public: using char_type = typename std::iterator_traits::value_type; + // Whether the lexer may reconstruct already-consumed input on demand (for + // diagnostics) instead of copying every scanned character eagerly. This is + // only sound for multi-pass, randomly-addressable byte input: the iterator + // must be random-access (so the consumed prefix can be revisited in O(1)) + // and each element must map 1:1 to an input byte (wide inputs are wrapped + // in wide_string_input_adapter, which does not expose this). + static constexpr bool supports_seek = + std::is_same::iterator_category, std::random_access_iterator_tag>::value + && sizeof(char_type) == 1; + iterator_input_adapter(IteratorType first, IteratorType last) - : current(std::move(first)), end(std::move(last)) + : begin(first), current(std::move(first)), end(std::move(last)) {} typename char_traits::int_type get_character() @@ -7009,6 +7019,22 @@ class iterator_input_adapter return char_traits::eof(); } + // number of characters consumed from the input so far + std::size_t get_consumed_count() const + { + return static_cast(std::distance(begin, current)); + } + + // append the already-consumed characters in the half-open range + // [first_index, last_index) to @a out; only valid when supports_seek + template + void copy_consumed_range(std::size_t first_index, std::size_t last_index, ContainerType& out) const + { + const auto from = std::next(begin, static_cast::difference_type>(first_index)); + const auto to = std::next(begin, static_cast::difference_type>(last_index)); + out.insert(out.end(), from, to); + } + // for general iterators, we cannot really do something better than falling back to processing the range one-by-one template std::size_t get_elements(T* dest, std::size_t count = 1) @@ -7030,6 +7056,7 @@ class iterator_input_adapter } private: + IteratorType begin; IteratorType current; IteratorType end; @@ -7510,6 +7537,28 @@ class lexer_base } } }; + +// Detect whether an input adapter can reconstruct already-consumed input on +// demand (see iterator_input_adapter::supports_seek). Adapters that do not +// expose the flag - e.g. file, stream, wide-string, and user-defined adapters - +// are treated as non-seekable streaming input, for which the lexer keeps +// copying every scanned character eagerly. The value is read via tag dispatch +// on is_detected so the flag is only referenced for adapters that provide it. +template +using detect_supports_seek = decltype(InputAdapterType::supports_seek); + +template +constexpr bool input_adapter_supports_seek(std::true_type /*detected*/) +{ + return InputAdapterType::supports_seek; +} + +template +constexpr bool input_adapter_supports_seek(std::false_type /*detected*/) +{ + return false; +} + /*! @brief lexical analysis @@ -7525,6 +7574,12 @@ class lexer : public lexer_base using char_type = typename InputAdapterType::char_type; using char_int_type = typename char_traits::int_type; + /// whether the last read token can be reconstructed from the input adapter + /// on demand (in error paths) instead of being copied on every scanned + /// character; see input_adapter_supports_seek + static constexpr bool lazy_token_string = + input_adapter_supports_seek(is_detected {}); + public: using token_type = typename lexer_base::token_type; @@ -8734,8 +8789,24 @@ scan_number_done: void reset() noexcept { token_buffer.clear(); - token_string.clear(); decimal_point_position = std::string::npos; + + note_token_start(std::integral_constant {}); + } + + /// seekable adapter: remember where the current token starts so it can be + /// reconstructed from the input on error; current has already been + /// consumed, hence the -1 + void note_token_start(std::true_type /*lazy*/) noexcept + { + token_string_start = ia.get_consumed_count() - 1; + } + + /// streaming adapter: start copying the token eagerly, beginning with the + /// already-read first character + void note_token_start(std::false_type /*lazy*/) noexcept + { + token_string.clear(); token_string.push_back(char_traits::to_char_type(current)); } @@ -8764,10 +8835,9 @@ scan_number_done: current = ia.get_character(); } - if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) - { - token_string.push_back(char_traits::to_char_type(current)); - } + // seekable adapters reconstruct the token lazily on error (see + // get_token_string), so the eager per-character copy is skipped + capture_char(std::integral_constant {}); if (current == '\n') { @@ -8778,6 +8848,18 @@ scan_number_done: return current; } + /// seekable adapter: nothing to capture, the token is rebuilt on error + void capture_char(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: copy the scanned character into token_string + void capture_char(std::false_type /*lazy*/) + { + if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) + { + token_string.push_back(char_traits::to_char_type(current)); + } + } + /*! @brief unget current character (read it again on next get) @@ -8805,6 +8887,15 @@ scan_number_done: --position.chars_read_current_line; } + uncapture_char(std::integral_constant {}); + } + + /// seekable adapter: nothing was captured, so nothing to undo + void uncapture_char(std::true_type /*lazy*/) const noexcept {} + + /// streaming adapter: drop the character copied by the matching get() + void uncapture_char(std::false_type /*lazy*/) + { if (JSON_HEDLEY_LIKELY(current != char_traits::eof())) { JSON_ASSERT(!token_string.empty()); @@ -8862,14 +8953,38 @@ scan_number_done: return position; } + /// seekable adapter: rebuild the last read token from the input on demand + const std::vector& collect_token_chars(std::vector& out, std::true_type /*lazy*/) const + { + // a pending unget of a real (non-EOF) character means that character + // was consumed from the input but is not part of the token; EOF is + // never consumed, so it must not be subtracted (mirrors unget()) + const bool pending_real_unget = next_unget && current != char_traits::eof(); + const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u); + if (JSON_HEDLEY_LIKELY(stop >= token_string_start)) + { + ia.copy_consumed_range(token_string_start, stop, out); + } + return out; + } + + /// streaming adapter: the token was copied eagerly while scanning + const std::vector& collect_token_chars(std::vector& /*out*/, std::false_type /*lazy*/) const + { + return token_string; + } + /// return the last read token (for errors only). Will never contain EOF /// (an arbitrary value that is not a valid char value, often -1), because /// 255 may legitimately occur. May contain NUL, which should be escaped. std::string get_token_string() const { + std::vector reconstructed; + const std::vector& chars = collect_token_chars(reconstructed, std::integral_constant {}); + // escape control characters std::string result; - for (const auto c : token_string) + for (const auto c : chars) { if (static_cast(c) <= '\x1F') { @@ -9030,9 +9145,14 @@ scan_number_done: /// the start position of the current token position_t position {}; - /// raw input token string (for error messages) + /// raw input token string for error messages; only populated for streaming + /// adapters (seekable adapters reconstruct it lazily via token_string_start) std::vector token_string {}; + /// start offset of the current token within the input, used to reconstruct + /// the last read token on error for seekable adapters (see collect_token_chars) + std::size_t token_string_start = 0; + /// buffer for variable-length tokens (numbers, strings) string_t token_buffer {}; diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 4ce7c4346..f4c28b267 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -16,6 +16,12 @@ using nlohmann::json; #endif #include +#include +#include +#include +#include +#include +#include namespace { @@ -1725,3 +1731,110 @@ TEST_CASE("parser class") CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*'", json::parse_error); } } + +// this test relies on parse errors being thrown, so it is skipped when +// exceptions are disabled (json::parse aborts instead of throwing there) +#if !defined(JSON_NOEXCEPTION) +namespace +{ +// Return the exception message from parsing @a input, or a "" +// sentinel if the parse unexpectedly succeeds. json::parse is nodiscard, so the +// result is consumed (via size()) to keep -Wunused-result / -Werror happy. +template +std::string parse_error_message(InputType&& input) +{ + try + { + const json j = json::parse(std::forward(input)); + return ""; + } + catch (const json::exception& e) + { + return e.what(); + } +} + +template +std::string parse_error_message_range(IteratorType first, IteratorType last) +{ + try + { + const json j = json::parse(first, last); + return ""; + } + catch (const json::exception& e) + { + return e.what(); + } +} +} // namespace + +TEST_CASE("last-read diagnostics are identical across input adapters") +{ + // The lexer reconstructs the "last read" token lazily for seekable adapters + // (contiguous byte input) and copies it eagerly for streaming adapters. + // Both strategies must yield byte-for-byte identical error messages. + + // a selection of malformed inputs that exercise different token kinds, + // whitespace/structural accumulation, number overflow, and control-char + // escaping in the reconstructed "last read" token + const std::vector inputs = + { + "[1,2,x]", + " \n @", + "{\"a\": }", + "1.18973e+4932", + "\"\t\"", + "tru", + "[1 2]", + "\xEF\xBB\xBF nul", + }; + + for (const auto& s : inputs) + { + CAPTURE(s); + + // reference: contiguous std::string -> seekable (lazy) path + const std::string reference = parse_error_message(s); + // every input is malformed, so parsing must fail (error messages start + // with '['; the success sentinel returned above starts with '<') + CHECK(reference.front() == '['); + + // const char* -> also seekable + CHECK(parse_error_message(s.c_str()) == reference); + + // std::vector iterators -> seekable (random-access) + { + const std::vector v(s.begin(), s.end()); + CHECK(parse_error_message_range(v.begin(), v.end()) == reference); + } + + // std::list iterators -> non-seekable (bidirectional) eager path + { + const std::list l(s.begin(), s.end()); + CHECK(parse_error_message_range(l.begin(), l.end()) == reference); + } + + // std::istringstream -> non-seekable streaming eager path + { + std::istringstream ss(s); + CHECK(parse_error_message(ss) == reference); + } + + // wide strings -> wide_string_input_adapter eager path; only comparable + // for ASCII input, as non-ASCII bytes are transcoded to different UTF-8 + const bool is_ascii = std::all_of(s.begin(), s.end(), [](char c) + { + return static_cast(c) < 0x80; + }); + if (is_ascii) + { + const std::u16string w16(s.begin(), s.end()); + CHECK(parse_error_message(w16) == reference); + + const std::u32string w32(s.begin(), s.end()); + CHECK(parse_error_message(w32) == reference); + } + } +} +#endif // !defined(JSON_NOEXCEPTION)