diff --git a/docs/mkdocs/docs/home/exceptions.md b/docs/mkdocs/docs/home/exceptions.md index 2f50bd6b4..c91e4fc36 100644 --- a/docs/mkdocs/docs/home/exceptions.md +++ b/docs/mkdocs/docs/home/exceptions.md @@ -140,6 +140,8 @@ Exceptions have ids 1xx. This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member `byte` indicates the error position. +The message ends with the last read token (`last read: '...'`), with control characters written as ``. A token of more than 8192 characters is shown by its first and its last 4096 characters, separated by `...`. + !!! failure "Example message" Input ended prematurely: diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index b4c998259..bc491222a 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -110,87 +110,164 @@ inline const char* empty_input_message() noexcept return "attempting to parse an empty input; check that your input string or stream contains the expected JSON"; } -//////////////////// -// input adapters // -//////////////////// - -#ifndef JSON_NO_IO /*! +@brief the characters of the last read token, kept for error messages + A growable array of characters for the part of a token that an input adapter -copied out of its window: unlike std::vector::push_back(), appending a -character takes no reference to a local, and is a few instructions when there -is room. +copied out of its window, or that the lexer copies while it scans input it +cannot read again: unlike std::vector::push_back(), appending a character +takes no reference to a local, and is a few instructions when there is room. + +The token is only needed for error messages, which show its first head() +characters and its last head() characters (see lexer::get_token_string). The +buffer therefore keeps the first head() and at least the last tail() +characters: when it is full and holds more than limit() characters, it drops +the characters in between instead of growing. That happens where it would +otherwise reallocate, so appending stays as cheap as before, and a long string +or a long run of whitespace read from a stream is not kept a second time. */ -class token_carry_buffer +template +class token_text_buffer { public: - token_carry_buffer() = default; - token_carry_buffer(const token_carry_buffer&) = delete; - token_carry_buffer& operator=(const token_carry_buffer&) = delete; - token_carry_buffer(token_carry_buffer&& rhs) noexcept + static constexpr std::size_t head() noexcept + { + return 4096; + } + /// a little more than head(), as the last character may be handed back + static constexpr std::size_t tail() noexcept + { + return head() + 16; + } + static constexpr std::size_t limit() noexcept + { + return head() + (2 * tail()); + } + + token_text_buffer() = default; + token_text_buffer(const token_text_buffer&) = delete; + token_text_buffer& operator=(const token_text_buffer&) = delete; + token_text_buffer(token_text_buffer&& rhs) noexcept : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end) { rhs.m_end = rhs.m_capacity_end = nullptr; } - token_carry_buffer& operator=(token_carry_buffer&&) = delete; - ~token_carry_buffer() = default; + token_text_buffer& operator=(token_text_buffer&& rhs) noexcept + { + m_data = std::move(rhs.m_data); + m_end = rhs.m_end; + m_capacity_end = rhs.m_capacity_end; + rhs.m_end = rhs.m_capacity_end = nullptr; + return *this; + } + ~token_text_buffer() = default; void clear() noexcept { m_end = m_data.get(); } - void push_back(char c) + bool empty() const noexcept + { + return m_end == m_data.get(); + } + + void push_back(CharType c) { if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end)) { - grow(1); + make_room(1); } *m_end++ = c; } - void append(const char* first, const char* last) + /// drop the last character + void pop_back() noexcept + { + JSON_ASSERT(!empty()); + --m_end; + } + + void append(const CharType* first, const CharType* last) { const auto n = static_cast(last - first); const auto room = static_cast(m_capacity_end - m_end); - if (n > room) + if (JSON_HEDLEY_UNLIKELY(n > room)) { - grow(n); + append_long(first, n); + return; } if (n != 0) { - std::memcpy(m_end, first, n); + std::memcpy(m_end, first, n * sizeof(CharType)); } m_end += n; } /// replace the contents of @a out - void copy_to(std::vector& out) const + void copy_to(std::vector& out) const { out.assign(m_data.get(), m_end); } private: - /// make room for @a extra more characters - JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra) + /// append @a n characters that do not fit, in pieces of at most tail() + /// characters, so that dropping the middle keeps the last tail() of them + JSON_HEDLEY_NEVER_INLINE void append_long(const CharType* first, std::size_t n) { - const auto size = static_cast(m_end - m_data.get()); - const auto capacity = static_cast(m_capacity_end - m_data.get()); - const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast(32) : 2 * capacity, size + extra); - std::unique_ptr data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - if (size != 0) + while (n != 0) { - std::memcpy(data.get(), m_data.get(), size); + const std::size_t piece = n < tail() ? n : tail(); + make_room(piece); + std::memcpy(m_end, first, piece * sizeof(CharType)); + m_end += piece; + first += piece; + n -= piece; } - m_data = std::move(data); - m_end = m_data.get() + size; - m_capacity_end = m_data.get() + new_capacity; } - std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) - char* m_end = nullptr; - char* m_capacity_end = nullptr; + /// make room for @a extra (at most tail()) more characters: drop the + /// middle of a long token instead of growing beyond limit() + JSON_HEDLEY_NEVER_INLINE void make_room(std::size_t extra) + { + // nothing is allocated before the first character + std::size_t size = m_data ? static_cast(m_end - m_data.get()) : 0; + if (size + extra > limit() && size > head() + tail()) + { + std::memmove(m_data.get() + head(), m_end - tail(), tail() * sizeof(CharType)); + m_end = m_data.get() + head() + tail(); + size = head() + tail(); + } + const std::size_t capacity = m_data ? static_cast(m_capacity_end - m_data.get()) : 0; + if (size + extra > capacity) + { + // never more than limit(): from there on, the middle is dropped + const std::size_t doubled = capacity < 32 ? static_cast(32) : 2 * capacity; + const std::size_t new_capacity = (std::max)(doubled < limit() ? doubled : limit(), size + extra); + std::unique_ptr data(new CharType[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + if (size != 0) + { + std::memcpy(data.get(), m_data.get(), size * sizeof(CharType)); + } + m_data = std::move(data); + m_end = m_data.get() + size; + m_capacity_end = m_data.get() + new_capacity; + } + } + + std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + CharType* m_end = nullptr; + CharType* m_capacity_end = nullptr; }; + +/// the part of a token that an input adapter copied out of its window +using token_carry_buffer = token_text_buffer; + +//////////////////// +// input adapters // +//////////////////// + +#ifndef JSON_NO_IO /*! Input adapter for stdio file access. The file is locked for the lifetime of the adapter and read with the unlocked stdio functions (see diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index 1e1d4e8af..54420867b 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -494,7 +494,10 @@ class lexer : public lexer_base /// streaming adapter: capture the characters void bulk_track_token(const char* data, std::size_t n, std::false_type /*lazy*/) { - token_string.insert(token_string.end(), data, data + n); + for (std::size_t i = 0; i < n; ++i) + { + token_string.push_back(static_cast(data[i])); + } } /// streaming input: no bulk fast path @@ -2237,15 +2240,28 @@ scan_number_done: const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u); if (JSON_HEDLEY_LIKELY(stop >= token_string_start)) { - ia.copy_consumed_range(token_string_start, stop, out); + // only the beginning and the end of a long token are shown (see + // get_token_string), so copy only these + constexpr std::size_t head = token_text_buffer::head(); + constexpr std::size_t tail = token_text_buffer::tail(); + if (stop - token_string_start > head + tail) + { + ia.copy_consumed_range(token_string_start, token_string_start + head, out); + ia.copy_consumed_range(stop - tail, stop, out); + } + else + { + ia.copy_consumed_range(token_string_start, stop, out); + } } return out; } /// streaming adapter: the token was copied eagerly while scanning - const std::vector& collect_token_chars(std::vector& /*out*/, std::false_type /*lazy*/) const + const std::vector& collect_token_chars(std::vector& out, std::false_type /*lazy*/) const { - return token_string; + token_string.copy_to(out); + return out; } /// return the last read token (for errors only). Will never contain EOF @@ -2256,10 +2272,32 @@ scan_number_done: std::vector reconstructed; const std::vector& chars = collect_token_chars(reconstructed, std::integral_constant {}); - // escape control characters + // a long token is shown by its first and its last head() characters; + // the buffers that keep it while it is read keep no more than these + // and a few more (see token_text_buffer) + constexpr std::size_t shown = token_text_buffer::head(); std::string result; - for (const auto c : chars) + if (chars.size() > 2 * shown) { + append_escaped(result, chars.begin(), chars.begin() + static_cast(shown)); + result += "..."; + append_escaped(result, chars.end() - static_cast(shown), chars.end()); + } + else + { + append_escaped(result, chars.begin(), chars.end()); + } + return result; + } + + private: + /// append the characters [first, last) to @a result, escaping control characters + template + static void append_escaped(std::string& result, Iterator first, Iterator last) + { + for (; first != last; ++first) + { + const auto c = *first; if (static_cast(c) <= '\x1F') { // escape control characters @@ -2273,10 +2311,10 @@ scan_number_done: result.push_back(static_cast(c)); } } - - return result; } + public: + /// return syntax error message JSON_HEDLEY_RETURNS_NON_NULL constexpr const char* get_error_message() const noexcept @@ -2525,7 +2563,7 @@ scan_number_done: /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) - std::vector token_string {}; + token_text_buffer token_string {}; /// start offset of the current token within the input, used to reconstruct /// the last read token on error for seekable adapters (see collect_token_chars) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 102eb94ad..c7631d71d 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9323,87 +9323,164 @@ inline const char* empty_input_message() noexcept return "attempting to parse an empty input; check that your input string or stream contains the expected JSON"; } -//////////////////// -// input adapters // -//////////////////// - -#ifndef JSON_NO_IO /*! +@brief the characters of the last read token, kept for error messages + A growable array of characters for the part of a token that an input adapter -copied out of its window: unlike std::vector::push_back(), appending a -character takes no reference to a local, and is a few instructions when there -is room. +copied out of its window, or that the lexer copies while it scans input it +cannot read again: unlike std::vector::push_back(), appending a character +takes no reference to a local, and is a few instructions when there is room. + +The token is only needed for error messages, which show its first head() +characters and its last head() characters (see lexer::get_token_string). The +buffer therefore keeps the first head() and at least the last tail() +characters: when it is full and holds more than limit() characters, it drops +the characters in between instead of growing. That happens where it would +otherwise reallocate, so appending stays as cheap as before, and a long string +or a long run of whitespace read from a stream is not kept a second time. */ -class token_carry_buffer +template +class token_text_buffer { public: - token_carry_buffer() = default; - token_carry_buffer(const token_carry_buffer&) = delete; - token_carry_buffer& operator=(const token_carry_buffer&) = delete; - token_carry_buffer(token_carry_buffer&& rhs) noexcept + static constexpr std::size_t head() noexcept + { + return 4096; + } + /// a little more than head(), as the last character may be handed back + static constexpr std::size_t tail() noexcept + { + return head() + 16; + } + static constexpr std::size_t limit() noexcept + { + return head() + (2 * tail()); + } + + token_text_buffer() = default; + token_text_buffer(const token_text_buffer&) = delete; + token_text_buffer& operator=(const token_text_buffer&) = delete; + token_text_buffer(token_text_buffer&& rhs) noexcept : m_data(std::move(rhs.m_data)), m_end(rhs.m_end), m_capacity_end(rhs.m_capacity_end) { rhs.m_end = rhs.m_capacity_end = nullptr; } - token_carry_buffer& operator=(token_carry_buffer&&) = delete; - ~token_carry_buffer() = default; + token_text_buffer& operator=(token_text_buffer&& rhs) noexcept + { + m_data = std::move(rhs.m_data); + m_end = rhs.m_end; + m_capacity_end = rhs.m_capacity_end; + rhs.m_end = rhs.m_capacity_end = nullptr; + return *this; + } + ~token_text_buffer() = default; void clear() noexcept { m_end = m_data.get(); } - void push_back(char c) + bool empty() const noexcept + { + return m_end == m_data.get(); + } + + void push_back(CharType c) { if (JSON_HEDLEY_UNLIKELY(m_end == m_capacity_end)) { - grow(1); + make_room(1); } *m_end++ = c; } - void append(const char* first, const char* last) + /// drop the last character + void pop_back() noexcept + { + JSON_ASSERT(!empty()); + --m_end; + } + + void append(const CharType* first, const CharType* last) { const auto n = static_cast(last - first); const auto room = static_cast(m_capacity_end - m_end); - if (n > room) + if (JSON_HEDLEY_UNLIKELY(n > room)) { - grow(n); + append_long(first, n); + return; } if (n != 0) { - std::memcpy(m_end, first, n); + std::memcpy(m_end, first, n * sizeof(CharType)); } m_end += n; } /// replace the contents of @a out - void copy_to(std::vector& out) const + void copy_to(std::vector& out) const { out.assign(m_data.get(), m_end); } private: - /// make room for @a extra more characters - JSON_HEDLEY_NEVER_INLINE void grow(std::size_t extra) + /// append @a n characters that do not fit, in pieces of at most tail() + /// characters, so that dropping the middle keeps the last tail() of them + JSON_HEDLEY_NEVER_INLINE void append_long(const CharType* first, std::size_t n) { - const auto size = static_cast(m_end - m_data.get()); - const auto capacity = static_cast(m_capacity_end - m_data.get()); - const std::size_t new_capacity = (std::max)(capacity < 32 ? static_cast(32) : 2 * capacity, size + extra); - std::unique_ptr data(new char[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) - if (size != 0) + while (n != 0) { - std::memcpy(data.get(), m_data.get(), size); + const std::size_t piece = n < tail() ? n : tail(); + make_room(piece); + std::memcpy(m_end, first, piece * sizeof(CharType)); + m_end += piece; + first += piece; + n -= piece; } - m_data = std::move(data); - m_end = m_data.get() + size; - m_capacity_end = m_data.get() + new_capacity; } - std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) - char* m_end = nullptr; - char* m_capacity_end = nullptr; + /// make room for @a extra (at most tail()) more characters: drop the + /// middle of a long token instead of growing beyond limit() + JSON_HEDLEY_NEVER_INLINE void make_room(std::size_t extra) + { + // nothing is allocated before the first character + std::size_t size = m_data ? static_cast(m_end - m_data.get()) : 0; + if (size + extra > limit() && size > head() + tail()) + { + std::memmove(m_data.get() + head(), m_end - tail(), tail() * sizeof(CharType)); + m_end = m_data.get() + head() + tail(); + size = head() + tail(); + } + const std::size_t capacity = m_data ? static_cast(m_capacity_end - m_data.get()) : 0; + if (size + extra > capacity) + { + // never more than limit(): from there on, the middle is dropped + const std::size_t doubled = capacity < 32 ? static_cast(32) : 2 * capacity; + const std::size_t new_capacity = (std::max)(doubled < limit() ? doubled : limit(), size + extra); + std::unique_ptr data(new CharType[new_capacity]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays) + if (size != 0) + { + std::memcpy(data.get(), m_data.get(), size * sizeof(CharType)); + } + m_data = std::move(data); + m_end = m_data.get() + size; + m_capacity_end = m_data.get() + new_capacity; + } + } + + std::unique_ptr m_data{}; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays,readability-redundant-member-init) + CharType* m_end = nullptr; + CharType* m_capacity_end = nullptr; }; + +/// the part of a token that an input adapter copied out of its window +using token_carry_buffer = token_text_buffer; + +//////////////////// +// input adapters // +//////////////////// + +#ifndef JSON_NO_IO /*! Input adapter for stdio file access. The file is locked for the lifetime of the adapter and read with the unlocked stdio functions (see @@ -13898,7 +13975,10 @@ class lexer : public lexer_base /// streaming adapter: capture the characters void bulk_track_token(const char* data, std::size_t n, std::false_type /*lazy*/) { - token_string.insert(token_string.end(), data, data + n); + for (std::size_t i = 0; i < n; ++i) + { + token_string.push_back(static_cast(data[i])); + } } /// streaming input: no bulk fast path @@ -15641,15 +15721,28 @@ scan_number_done: const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u); if (JSON_HEDLEY_LIKELY(stop >= token_string_start)) { - ia.copy_consumed_range(token_string_start, stop, out); + // only the beginning and the end of a long token are shown (see + // get_token_string), so copy only these + constexpr std::size_t head = token_text_buffer::head(); + constexpr std::size_t tail = token_text_buffer::tail(); + if (stop - token_string_start > head + tail) + { + ia.copy_consumed_range(token_string_start, token_string_start + head, out); + ia.copy_consumed_range(stop - tail, stop, out); + } + else + { + ia.copy_consumed_range(token_string_start, stop, out); + } } return out; } /// streaming adapter: the token was copied eagerly while scanning - const std::vector& collect_token_chars(std::vector& /*out*/, std::false_type /*lazy*/) const + const std::vector& collect_token_chars(std::vector& out, std::false_type /*lazy*/) const { - return token_string; + token_string.copy_to(out); + return out; } /// return the last read token (for errors only). Will never contain EOF @@ -15660,10 +15753,32 @@ scan_number_done: std::vector reconstructed; const std::vector& chars = collect_token_chars(reconstructed, std::integral_constant {}); - // escape control characters + // a long token is shown by its first and its last head() characters; + // the buffers that keep it while it is read keep no more than these + // and a few more (see token_text_buffer) + constexpr std::size_t shown = token_text_buffer::head(); std::string result; - for (const auto c : chars) + if (chars.size() > 2 * shown) { + append_escaped(result, chars.begin(), chars.begin() + static_cast(shown)); + result += "..."; + append_escaped(result, chars.end() - static_cast(shown), chars.end()); + } + else + { + append_escaped(result, chars.begin(), chars.end()); + } + return result; + } + + private: + /// append the characters [first, last) to @a result, escaping control characters + template + static void append_escaped(std::string& result, Iterator first, Iterator last) + { + for (; first != last; ++first) + { + const auto c = *first; if (static_cast(c) <= '\x1F') { // escape control characters @@ -15677,10 +15792,10 @@ scan_number_done: result.push_back(static_cast(c)); } } - - return result; } + public: + /// return syntax error message JSON_HEDLEY_RETURNS_NON_NULL constexpr const char* get_error_message() const noexcept @@ -15929,7 +16044,7 @@ scan_number_done: /// raw input token string for error messages; only populated for streaming /// adapters (seekable adapters reconstruct it lazily via token_string_start) - std::vector token_string {}; + token_text_buffer token_string {}; /// start offset of the current token within the input, used to reconstruct /// the last read token on error for seekable adapters (see collect_token_chars) diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index 616999896..0f176df15 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -12,6 +12,7 @@ #include using nlohmann::json; +#include // min #include // array #include // toupper #include // uint32_t, uint64_t @@ -2233,3 +2234,123 @@ TEST_CASE("serialization of short strings") CHECK(json("caf\xC3\xA9").dump() == "\"caf\xC3\xA9\""); CHECK(json("caf\xC3\xA9").dump(-1, ' ', true) == "\"caf\\u00e9\""); } + +TEST_CASE("token_text_buffer keeps the beginning and the end of a long token") +{ + using buffer_t = nlohmann::detail::token_text_buffer; + constexpr std::size_t head = buffer_t::head(); + constexpr std::size_t tail = buffer_t::tail(); + + // the characters 0, 1, ..., 250, 0, 1, ... + const auto at = [](std::size_t i) + { + return static_cast(i % 251); + }; + // whether out holds the first head and the last tail characters of a + // token of n characters (and maybe more characters of its middle) + const auto keeps_ends = [&](const std::vector& out, std::size_t n) + { + if (out.size() < head + tail || out.size() > buffer_t::limit()) + { + return false; + } + for (std::size_t i = 0; i < head; ++i) + { + if (out[i] != at(i)) + { + return false; + } + } + for (std::size_t i = 0; i < tail; ++i) + { + if (out[out.size() - tail + i] != at(n - tail + i)) + { + return false; + } + } + return true; + }; + + SECTION("a short token is kept as it is") + { + buffer_t buffer; + CHECK(buffer.empty()); + for (std::size_t i = 0; i < head + tail; ++i) + { + buffer.push_back(at(i)); + } + std::vector out; + buffer.copy_to(out); + REQUIRE(out.size() == head + tail); + bool same = true; + for (std::size_t i = 0; i < out.size(); ++i) + { + same = same && out[i] == at(i); + } + CHECK(same); + } + + SECTION("characters added one by one") + { + for (const std::size_t n : + { + head + tail + 1, buffer_t::limit(), buffer_t::limit() + 1, static_cast(100000) + }) + { + CAPTURE(n) + buffer_t buffer; + for (std::size_t i = 0; i < n; ++i) + { + buffer.push_back(at(i)); + } + std::vector out; + buffer.copy_to(out); + CHECK(keeps_ends(out, n)); + + // the last character can be handed back + buffer.pop_back(); + buffer.copy_to(out); + CHECK(keeps_ends(out, n - 1)); + } + } + + SECTION("characters added in pieces of different sizes") + { + std::vector input(200000); + for (std::size_t i = 0; i < input.size(); ++i) + { + input[i] = at(i); + } + for (const std::size_t piece : + { + static_cast(1), static_cast(100), tail - 1, tail, tail + 1, static_cast(65536), input.size() + }) + { + CAPTURE(piece) + buffer_t buffer; + for (std::size_t pos = 0; pos < input.size(); pos += piece) + { + const std::size_t n = (std::min)(piece, input.size() - pos); + buffer.append(input.data() + pos, input.data() + pos + n); + } + std::vector out; + buffer.copy_to(out); + CHECK(keeps_ends(out, input.size())); + } + } + + SECTION("clear starts a new token") + { + buffer_t buffer; + for (std::size_t i = 0; i < 100000; ++i) + { + buffer.push_back('x'); + } + buffer.clear(); + CHECK(buffer.empty()); + buffer.push_back('a'); + std::vector out; + buffer.copy_to(out); + CHECK(out == std::vector {'a'}); + } +} diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 412d0dcb3..b015a87d0 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -3269,3 +3269,116 @@ TEST_CASE("a plain function is accepted as parser callback for containers") CHECK((nlohmann::detail::can_compare_ne::value)); CHECK((nlohmann::detail::can_compare_ne::value)); } + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("parse error message of a long token") +{ + // the message of a parse error shows the last read token; a long token is + // shown by its first and its last 4096 characters, and the message is the + // same for every kind of input + const auto message = [](const std::string & text, int kind) -> std::string + { + json _; + try + { + if (kind == 0) + { + _ = json::parse(text); + } + else if (kind == 1) + { + std::istringstream stream(text); + _ = json::parse(stream); + } + else if (kind == 2) + { + FILE* const file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory) + REQUIRE(file != nullptr); + CHECK(std::fwrite(text.data(), 1, text.size(), file) == text.size()); + CHECK(std::fseek(file, 0, SEEK_SET) == 0); + try + { + _ = json::parse(file); + } + catch (...) + { + std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory) + throw; + } + std::fclose(file); // NOLINT(cert-err33-c,cppcoreguidelines-owning-memory) + } + else + { + const std::list chars(text.begin(), text.end()); + _ = json::parse(chars.begin(), chars.end()); + } + } + catch (const json::parse_error& e) + { + return e.what(); + } + return "no error"; + }; + + SECTION("a short token is shown completely") + { + const std::string text = "[\"" + std::string(8000, 'a') + "\x01\"]"; + const std::string expected = "last read: '\"" + std::string(8000, 'a') + "'"; + for (int kind = 0; kind < 4; ++kind) + { + CAPTURE(kind) + const std::string m = message(text, kind); + CHECK(m.size() > expected.size()); + CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0); + } + } + + SECTION("a long string") + { + const std::string text = "[\"" + std::string(100000, 'a') + "b\x01\"]"; + // the token is '"', 100,000 times 'a', 'b', and the control character + const std::string expected = "last read: '\"" + std::string(4095, 'a') + "..." + std::string(4094, 'a') + "b'"; + for (int kind = 0; kind < 4; ++kind) + { + CAPTURE(kind) + const std::string m = message(text, kind); + CHECK(m.size() > expected.size()); + CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0); + } + } + + SECTION("a long run of whitespace after a number") + { + // the token begins with the number and ends with the invalid character + const std::string text = "[1," + std::string(100000, '\n') + "?]"; + std::string expected = "last read: '1,"; + for (int i = 0; i < 4094; ++i) + { + expected += ""; + } + expected += "..."; + for (int i = 0; i < 4095; ++i) + { + expected += ""; + } + expected += "?'"; + for (int kind = 0; kind < 4; ++kind) + { + CAPTURE(kind) + const std::string m = message(text, kind); + CHECK(m.size() > expected.size()); + CHECK(m.compare(m.size() - expected.size(), expected.size(), expected) == 0); + } + } + + SECTION("valid input with a long run of whitespace") + { + const std::string text = "1" + std::string(100000, ' '); + for (int kind = 0; kind < 4; ++kind) + { + CAPTURE(kind) + CHECK(message(text, kind) == "no error"); + } + } +} +#endif