Reconstruct lexer diagnostics lazily for seekable input (#120) (#5234)

`lexer::get()` copied every scanned character into `token_string` on the whole successful-parse hot path, yet that buffer is consumed only by `get_token_string()` when rendering the "last read" fragment of a parse error. On well-formed input the per-byte copy (plus the `unget()` pop) is pure overhead that is always discarded. For seekable input adapters - random-access, single-byte iterators such as those backing `std::string`, `const char*`, and `std::vector<char>` - the offending token is now reconstructed on demand from the input when an error is reported, using a saved start offset, and the eager copy is skipped. Streaming adapters (file, istream, wide-string, and user-defined adapters) keep the eager copy; the strategy is chosen at compile time via `input_adapter_supports_seek`, so adapters without the capability are unaffected. Error messages are byte-for-byte identical across all adapters, verified by a new parity regression test. Microbenchmark (4 MB mixed JSON, parsed from a std::string): ~149 -> ~160 MB/s, about +8%. Signed-off-by: Niels Lohmann <mail@nlohmann.me> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-07 11:05:09 +00:00 · 2026-07-05 10:42:48 +02:00
parent c034480c22
commit eed1587000
4 changed files with 369 additions and 16 deletions
@@ -161,8 +161,18 @@ class iterator_input_adapter
  public:
    using char_type = typename std::iterator_traits<IteratorType>::value_type;

+    // Whether the lexer may reconstruct already-consumed input on demand (for
+    // diagnostics) instead of copying every scanned character eagerly. This is
+    // only sound for multi-pass, randomly-addressable byte input: the iterator
+    // must be random-access (so the consumed prefix can be revisited in O(1))
+    // and each element must map 1:1 to an input byte (wide inputs are wrapped
+    // in wide_string_input_adapter, which does not expose this).
+    static constexpr bool supports_seek =
+        std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value
+        && sizeof(char_type) == 1;
+
    iterator_input_adapter(IteratorType first, IteratorType last)
-        : current(std::move(first)), end(std::move(last))
+        : begin(first), current(std::move(first)), end(std::move(last))
    {}

    typename char_traits<char_type>::int_type get_character()
@@ -177,6 +187,22 @@ class iterator_input_adapter
        return char_traits<char_type>::eof();
    }

+    // number of characters consumed from the input so far
+    std::size_t get_consumed_count() const
+    {
+        return static_cast<std::size_t>(std::distance(begin, current));
+    }
+
+    // append the already-consumed characters in the half-open range
+    // [first_index, last_index) to @a out; only valid when supports_seek
+    template<typename ContainerType>
+    void copy_consumed_range(std::size_t first_index, std::size_t last_index, ContainerType& out) const
+    {
+        const auto from = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(first_index));
+        const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
+        out.insert(out.end(), from, to);
+    }
+
    // for general iterators, we cannot really do something better than falling back to processing the range one-by-one
    template<class T>
    std::size_t get_elements(T* dest, std::size_t count = 1)
@@ -198,6 +224,7 @@ class iterator_input_adapter
    }

  private:
+    IteratorType begin;
    IteratorType current;
    IteratorType end;

@@ -103,6 +103,28 @@ class lexer_base
        }
    }
 };
+
+// Detect whether an input adapter can reconstruct already-consumed input on
+// demand (see iterator_input_adapter::supports_seek). Adapters that do not
+// expose the flag - e.g. file, stream, wide-string, and user-defined adapters -
+// are treated as non-seekable streaming input, for which the lexer keeps
+// copying every scanned character eagerly. The value is read via tag dispatch
+// on is_detected so the flag is only referenced for adapters that provide it.
+template<typename InputAdapterType>
+using detect_supports_seek = decltype(InputAdapterType::supports_seek);
+
+template<typename InputAdapterType>
+constexpr bool input_adapter_supports_seek(std::true_type /*detected*/)
+{
+    return InputAdapterType::supports_seek;
+}
+
+template<typename InputAdapterType>
+constexpr bool input_adapter_supports_seek(std::false_type /*detected*/)
+{
+    return false;
+}
+
 /*!
@brief lexical analysis

@@ -118,6 +140,12 @@ class lexer : public lexer_base<BasicJsonType>
    using char_type = typename InputAdapterType::char_type;
    using char_int_type = typename char_traits<char_type>::int_type;

+    /// whether the last read token can be reconstructed from the input adapter
+    /// on demand (in error paths) instead of being copied on every scanned
+    /// character; see input_adapter_supports_seek
+    static constexpr bool lazy_token_string =
+        input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
+
  public:
    using token_type = typename lexer_base<BasicJsonType>::token_type;

@@ -1327,8 +1355,24 @@ scan_number_done:
    void reset() noexcept
    {
        token_buffer.clear();
-        token_string.clear();
        decimal_point_position = std::string::npos;
+
+        note_token_start(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+    /// seekable adapter: remember where the current token starts so it can be
+    /// reconstructed from the input on error; current has already been
+    /// consumed, hence the -1
+    void note_token_start(std::true_type /*lazy*/) noexcept
+    {
+        token_string_start = ia.get_consumed_count() - 1;
+    }
+
+    /// streaming adapter: start copying the token eagerly, beginning with the
+    /// already-read first character
+    void note_token_start(std::false_type /*lazy*/) noexcept
+    {
+        token_string.clear();
        token_string.push_back(char_traits<char_type>::to_char_type(current));
    }

@@ -1357,10 +1401,9 @@ scan_number_done:
            current = ia.get_character();
        }

-        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
-        {
-            token_string.push_back(char_traits<char_type>::to_char_type(current));
-        }
+        // seekable adapters reconstruct the token lazily on error (see
+        // get_token_string), so the eager per-character copy is skipped
+        capture_char(std::integral_constant<bool, lazy_token_string> {});

        if (current == '\n')
        {
@@ -1371,6 +1414,18 @@ scan_number_done:
        return current;
    }

+    /// seekable adapter: nothing to capture, the token is rebuilt on error
+    void capture_char(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: copy the scanned character into token_string
+    void capture_char(std::false_type /*lazy*/)
+    {
+        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
+        {
+            token_string.push_back(char_traits<char_type>::to_char_type(current));
+        }
+    }
+
    /*!
    @brief unget current character (read it again on next get)

@@ -1398,6 +1453,15 @@ scan_number_done:
            --position.chars_read_current_line;
        }

+        uncapture_char(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+    /// seekable adapter: nothing was captured, so nothing to undo
+    void uncapture_char(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: drop the character copied by the matching get()
+    void uncapture_char(std::false_type /*lazy*/)
+    {
        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
        {
            JSON_ASSERT(!token_string.empty());
@@ -1455,14 +1519,38 @@ scan_number_done:
        return position;
    }

+    /// seekable adapter: rebuild the last read token from the input on demand
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
+    {
+        // a pending unget of a real (non-EOF) character means that character
+        // was consumed from the input but is not part of the token; EOF is
+        // never consumed, so it must not be subtracted (mirrors unget())
+        const bool pending_real_unget = next_unget && current != char_traits<char_type>::eof();
+        const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u);
+        if (JSON_HEDLEY_LIKELY(stop >= token_string_start))
+        {
+            ia.copy_consumed_range(token_string_start, stop, out);
+        }
+        return out;
+    }
+
+    /// streaming adapter: the token was copied eagerly while scanning
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& /*out*/, std::false_type /*lazy*/) const
+    {
+        return token_string;
+    }
+
    /// return the last read token (for errors only).  Will never contain EOF
    /// (an arbitrary value that is not a valid char value, often -1), because
    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
    std::string get_token_string() const
    {
+        std::vector<char_type> reconstructed;
+        const std::vector<char_type>& chars = collect_token_chars(reconstructed, std::integral_constant<bool, lazy_token_string> {});
+
        // escape control characters
        std::string result;
-        for (const auto c : token_string)
+        for (const auto c : chars)
        {
            if (static_cast<unsigned char>(c) <= '\x1F')
            {
@@ -1623,9 +1711,14 @@ scan_number_done:
    /// the start position of the current token
    position_t position {};

-    /// raw input token string (for error messages)
+    /// raw input token string for error messages; only populated for streaming
+    /// adapters (seekable adapters reconstruct it lazily via token_string_start)
    std::vector<char_type> token_string {};

+    /// start offset of the current token within the input, used to reconstruct
+    /// the last read token on error for seekable adapters (see collect_token_chars)
+    std::size_t token_string_start = 0;
+
    /// buffer for variable-length tokens (numbers, strings)
    string_t token_buffer {};

@@ -6993,8 +6993,18 @@ class iterator_input_adapter
  public:
    using char_type = typename std::iterator_traits<IteratorType>::value_type;

+    // Whether the lexer may reconstruct already-consumed input on demand (for
+    // diagnostics) instead of copying every scanned character eagerly. This is
+    // only sound for multi-pass, randomly-addressable byte input: the iterator
+    // must be random-access (so the consumed prefix can be revisited in O(1))
+    // and each element must map 1:1 to an input byte (wide inputs are wrapped
+    // in wide_string_input_adapter, which does not expose this).
+    static constexpr bool supports_seek =
+        std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value
+        && sizeof(char_type) == 1;
+
    iterator_input_adapter(IteratorType first, IteratorType last)
-        : current(std::move(first)), end(std::move(last))
+        : begin(first), current(std::move(first)), end(std::move(last))
    {}

    typename char_traits<char_type>::int_type get_character()
@@ -7009,6 +7019,22 @@ class iterator_input_adapter
        return char_traits<char_type>::eof();
    }

+    // number of characters consumed from the input so far
+    std::size_t get_consumed_count() const
+    {
+        return static_cast<std::size_t>(std::distance(begin, current));
+    }
+
+    // append the already-consumed characters in the half-open range
+    // [first_index, last_index) to @a out; only valid when supports_seek
+    template<typename ContainerType>
+    void copy_consumed_range(std::size_t first_index, std::size_t last_index, ContainerType& out) const
+    {
+        const auto from = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(first_index));
+        const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
+        out.insert(out.end(), from, to);
+    }
+
    // for general iterators, we cannot really do something better than falling back to processing the range one-by-one
    template<class T>
    std::size_t get_elements(T* dest, std::size_t count = 1)
@@ -7030,6 +7056,7 @@ class iterator_input_adapter
    }

  private:
+    IteratorType begin;
    IteratorType current;
    IteratorType end;

@@ -7510,6 +7537,28 @@ class lexer_base
        }
    }
 };
+
+// Detect whether an input adapter can reconstruct already-consumed input on
+// demand (see iterator_input_adapter::supports_seek). Adapters that do not
+// expose the flag - e.g. file, stream, wide-string, and user-defined adapters -
+// are treated as non-seekable streaming input, for which the lexer keeps
+// copying every scanned character eagerly. The value is read via tag dispatch
+// on is_detected so the flag is only referenced for adapters that provide it.
+template<typename InputAdapterType>
+using detect_supports_seek = decltype(InputAdapterType::supports_seek);
+
+template<typename InputAdapterType>
+constexpr bool input_adapter_supports_seek(std::true_type /*detected*/)
+{
+    return InputAdapterType::supports_seek;
+}
+
+template<typename InputAdapterType>
+constexpr bool input_adapter_supports_seek(std::false_type /*detected*/)
+{
+    return false;
+}
+
 /*!
@brief lexical analysis

@@ -7525,6 +7574,12 @@ class lexer : public lexer_base<BasicJsonType>
    using char_type = typename InputAdapterType::char_type;
    using char_int_type = typename char_traits<char_type>::int_type;

+    /// whether the last read token can be reconstructed from the input adapter
+    /// on demand (in error paths) instead of being copied on every scanned
+    /// character; see input_adapter_supports_seek
+    static constexpr bool lazy_token_string =
+        input_adapter_supports_seek<InputAdapterType>(is_detected<detect_supports_seek, InputAdapterType> {});
+
  public:
    using token_type = typename lexer_base<BasicJsonType>::token_type;

@@ -8734,8 +8789,24 @@ scan_number_done:
    void reset() noexcept
    {
        token_buffer.clear();
-        token_string.clear();
        decimal_point_position = std::string::npos;
+
+        note_token_start(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+    /// seekable adapter: remember where the current token starts so it can be
+    /// reconstructed from the input on error; current has already been
+    /// consumed, hence the -1
+    void note_token_start(std::true_type /*lazy*/) noexcept
+    {
+        token_string_start = ia.get_consumed_count() - 1;
+    }
+
+    /// streaming adapter: start copying the token eagerly, beginning with the
+    /// already-read first character
+    void note_token_start(std::false_type /*lazy*/) noexcept
+    {
+        token_string.clear();
        token_string.push_back(char_traits<char_type>::to_char_type(current));
    }

@@ -8764,10 +8835,9 @@ scan_number_done:
            current = ia.get_character();
        }

-        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
-        {
-            token_string.push_back(char_traits<char_type>::to_char_type(current));
-        }
+        // seekable adapters reconstruct the token lazily on error (see
+        // get_token_string), so the eager per-character copy is skipped
+        capture_char(std::integral_constant<bool, lazy_token_string> {});

        if (current == '\n')
        {
@@ -8778,6 +8848,18 @@ scan_number_done:
        return current;
    }

+    /// seekable adapter: nothing to capture, the token is rebuilt on error
+    void capture_char(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: copy the scanned character into token_string
+    void capture_char(std::false_type /*lazy*/)
+    {
+        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
+        {
+            token_string.push_back(char_traits<char_type>::to_char_type(current));
+        }
+    }
+
    /*!
    @brief unget current character (read it again on next get)

@@ -8805,6 +8887,15 @@ scan_number_done:
            --position.chars_read_current_line;
        }

+        uncapture_char(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+    /// seekable adapter: nothing was captured, so nothing to undo
+    void uncapture_char(std::true_type /*lazy*/) const noexcept {}
+
+    /// streaming adapter: drop the character copied by the matching get()
+    void uncapture_char(std::false_type /*lazy*/)
+    {
        if (JSON_HEDLEY_LIKELY(current != char_traits<char_type>::eof()))
        {
            JSON_ASSERT(!token_string.empty());
@@ -8862,14 +8953,38 @@ scan_number_done:
        return position;
    }

+    /// seekable adapter: rebuild the last read token from the input on demand
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& out, std::true_type /*lazy*/) const
+    {
+        // a pending unget of a real (non-EOF) character means that character
+        // was consumed from the input but is not part of the token; EOF is
+        // never consumed, so it must not be subtracted (mirrors unget())
+        const bool pending_real_unget = next_unget && current != char_traits<char_type>::eof();
+        const std::size_t stop = ia.get_consumed_count() - (pending_real_unget ? 1u : 0u);
+        if (JSON_HEDLEY_LIKELY(stop >= token_string_start))
+        {
+            ia.copy_consumed_range(token_string_start, stop, out);
+        }
+        return out;
+    }
+
+    /// streaming adapter: the token was copied eagerly while scanning
+    const std::vector<char_type>& collect_token_chars(std::vector<char_type>& /*out*/, std::false_type /*lazy*/) const
+    {
+        return token_string;
+    }
+
    /// return the last read token (for errors only).  Will never contain EOF
    /// (an arbitrary value that is not a valid char value, often -1), because
    /// 255 may legitimately occur.  May contain NUL, which should be escaped.
    std::string get_token_string() const
    {
+        std::vector<char_type> reconstructed;
+        const std::vector<char_type>& chars = collect_token_chars(reconstructed, std::integral_constant<bool, lazy_token_string> {});
+
        // escape control characters
        std::string result;
-        for (const auto c : token_string)
+        for (const auto c : chars)
        {
            if (static_cast<unsigned char>(c) <= '\x1F')
            {
@@ -9030,9 +9145,14 @@ scan_number_done:
    /// the start position of the current token
    position_t position {};

-    /// raw input token string (for error messages)
+    /// raw input token string for error messages; only populated for streaming
+    /// adapters (seekable adapters reconstruct it lazily via token_string_start)
    std::vector<char_type> token_string {};

+    /// start offset of the current token within the input, used to reconstruct
+    /// the last read token on error for seekable adapters (see collect_token_chars)
+    std::size_t token_string_start = 0;
+
    /// buffer for variable-length tokens (numbers, strings)
    string_t token_buffer {};

@@ -16,6 +16,12 @@ using nlohmann::json;
 #endif

 #include <valarray>
+#include <algorithm>
+#include <list>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>

 namespace
 {
@@ -1725,3 +1731,110 @@ TEST_CASE("parser class")
        CHECK_THROWS_WITH_AS(_ = json::parse("/*", nullptr, true, true), "[json.exception.parse_error.101] parse error at line 1, column 3: syntax error while parsing value - invalid comment; missing closing '*/'; last read: '/*<U+0000>'", json::parse_error);
    }
 }
+
+// this test relies on parse errors being thrown, so it is skipped when
+// exceptions are disabled (json::parse aborts instead of throwing there)
+#if !defined(JSON_NOEXCEPTION)
+namespace
+{
+// Return the exception message from parsing @a input, or a "<no error ...>"
+// sentinel if the parse unexpectedly succeeds. json::parse is nodiscard, so the
+// result is consumed (via size()) to keep -Wunused-result / -Werror happy.
+template<typename InputType>
+std::string parse_error_message(InputType&& input)
+{
+    try
+    {
+        const json j = json::parse(std::forward<InputType>(input));
+        return "<no error, size " + std::to_string(j.size()) + ">";
+    }
+    catch (const json::exception& e)
+    {
+        return e.what();
+    }
+}
+
+template<typename IteratorType>
+std::string parse_error_message_range(IteratorType first, IteratorType last)
+{
+    try
+    {
+        const json j = json::parse(first, last);
+        return "<no error, size " + std::to_string(j.size()) + ">";
+    }
+    catch (const json::exception& e)
+    {
+        return e.what();
+    }
+}
+} // namespace
+
+TEST_CASE("last-read diagnostics are identical across input adapters")
+{
+    // The lexer reconstructs the "last read" token lazily for seekable adapters
+    // (contiguous byte input) and copies it eagerly for streaming adapters.
+    // Both strategies must yield byte-for-byte identical error messages.
+
+    // a selection of malformed inputs that exercise different token kinds,
+    // whitespace/structural accumulation, number overflow, and control-char
+    // escaping in the reconstructed "last read" token
+    const std::vector<std::string> inputs =
+    {
+        "[1,2,x]",
+        "  \n  @",
+        "{\"a\": }",
+        "1.18973e+4932",
+        "\"\t\"",
+        "tru",
+        "[1 2]",
+        "\xEF\xBB\xBF   nul",
+    };
+
+    for (const auto& s : inputs)
+    {
+        CAPTURE(s);
+
+        // reference: contiguous std::string -> seekable (lazy) path
+        const std::string reference = parse_error_message(s);
+        // every input is malformed, so parsing must fail (error messages start
+        // with '['; the success sentinel returned above starts with '<')
+        CHECK(reference.front() == '[');
+
+        // const char* -> also seekable
+        CHECK(parse_error_message(s.c_str()) == reference);
+
+        // std::vector<char> iterators -> seekable (random-access)
+        {
+            const std::vector<char> v(s.begin(), s.end());
+            CHECK(parse_error_message_range(v.begin(), v.end()) == reference);
+        }
+
+        // std::list iterators -> non-seekable (bidirectional) eager path
+        {
+            const std::list<char> l(s.begin(), s.end());
+            CHECK(parse_error_message_range(l.begin(), l.end()) == reference);
+        }
+
+        // std::istringstream -> non-seekable streaming eager path
+        {
+            std::istringstream ss(s);
+            CHECK(parse_error_message(ss) == reference);
+        }
+
+        // wide strings -> wide_string_input_adapter eager path; only comparable
+        // for ASCII input, as non-ASCII bytes are transcoded to different UTF-8
+        const bool is_ascii = std::all_of(s.begin(), s.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x80;
+        });
+        if (is_ascii)
+        {
+            const std::u16string w16(s.begin(), s.end());
+            CHECK(parse_error_message(w16) == reference);
+
+            const std::u32string w32(s.begin(), s.end());
+            CHECK(parse_error_message(w32) == reference);
+        }
+    }
+}
+#endif // !defined(JSON_NOEXCEPTION)