Count lines incrementally when a recovering parse reports many errors

With seekable input, the lexer derives line and column on demand (#5809)
by counting the newlines in all input consumed so far. A SAX parser that
recovers from errors asks for the position once per error, so input with
an error at every character took quadratic time: "{" followed by 100000
commas took 85 s at -O0, and the parser tests timed out under Valgrind
(1500 s, was 69 s on develop).

get_position() now remembers what it counted and only scans the input
consumed since the previous call; count_consumed_lines() takes the start
of the range. The 100000 commas now take 0.17 s, unit-class_parser 0.9 s
instead of 86 s. Over 69284 error messages from random multi-line inputs,
the messages are identical to those before the change.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/input_adapters.hpp

@@ -314,17 +314,22 @@ class iterator_input_adapter
         return get_consumed_count() + eof_reads;
     }
 
-    // number of newlines in the consumed range [0, last_index); @a line_start
+    // number of newlines in the consumed range [first_index, last_index); if
-    // is set to the index after the last of them (0 if there is none); only
+    // there is one, @a line_start is set to the index after the last of them,
-    // valid when supports_seek
+    // otherwise it is left unchanged; only valid when supports_seek
-    std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const
+    std::size_t count_consumed_lines(std::size_t first_index, std::size_t last_index, std::size_t& line_start) const
     {
-        const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
+        using difference_type = typename std::iterator_traits<IteratorType>::difference_type;
+        const auto from = std::next(begin, static_cast<difference_type>(first_index));
+        const auto to = std::next(from, static_cast<difference_type>(last_index - first_index));
         const auto newline = char_traits<char_type>::to_char_type('\n');
         using reverse_it = std::reverse_iterator<IteratorType>;
-        const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline);
+        const auto last_newline = std::find(reverse_it(to), reverse_it(from), newline);
-        line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
+        if (last_newline.base() != from)
-        return static_cast<std::size_t>(std::count(begin, last_newline.base(), newline));
+        {
+            line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
+        }
+        return static_cast<std::size_t>(std::count(from, last_newline.base(), newline));
     }
 
     // append the already-consumed characters in the half-open range

include/nlohmann/detail/input/lexer.hpp

@@ -2012,17 +2012,25 @@ scan_number_done:
         return position.chars_read_total;
     }
 
-    /// seekable adapter: count the lines in the consumed input; O(input read
+    /// seekable adapter: count the lines in the consumed input. Only the
-    /// so far), but only called on errors
+    /// input consumed since the previous call is scanned, so a parse that
+    /// reports many errors (a SAX parser recovering from them) stays linear
+    /// in the input instead of rescanning it for every error.
     position_t get_position_impl(std::true_type /*lazy*/) const
     {
         position_t result;
         result.chars_read_total = get_chars_read_total();
         // reads at EOF count as characters but are not part of the input
         const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count());
-        std::size_t line_start = 0;
+        if (JSON_HEDLEY_UNLIKELY(consumed < position_cache.consumed))
-        result.lines_read = ia.count_consumed_lines(consumed, line_start);
+        {
-        result.chars_read_current_line = result.chars_read_total - line_start;
+            // the consumed input never shrinks; recount from the start if it did
+            position_cache = {};
+        }
+        position_cache.lines += ia.count_consumed_lines(position_cache.consumed, consumed, position_cache.line_start);
+        position_cache.consumed = consumed;
+        result.lines_read = position_cache.lines;
+        result.chars_read_current_line = result.chars_read_total - position_cache.line_start;
         return result;
     }
 
@@ -2898,6 +2906,19 @@ scan_number_done:
     /// that unget() can restore the column instead of leaving it at 0
     std::size_t chars_read_before_newline = 0;
 
+    /// seekable adapter: what get_position() counted last, so that the next
+    /// call only scans the input consumed since
+    struct position_cache_t
+    {
+        /// consumed characters scanned so far
+        std::size_t consumed = 0;
+        /// newlines among them
+        std::size_t lines = 0;
+        /// index after the last of these newlines
+        std::size_t line_start = 0;
+    };
+    mutable position_cache_t position_cache {};
+
     /// raw input token string for error messages; only populated for streaming
     /// adapters (seekable adapters reconstruct it lazily via token_string_start)
     std::vector<char_type> token_string {};

single_include/nlohmann/json.hpp

@@ -8624,17 +8624,22 @@ class iterator_input_adapter
         return get_consumed_count() + eof_reads;
     }
 
-    // number of newlines in the consumed range [0, last_index); @a line_start
+    // number of newlines in the consumed range [first_index, last_index); if
-    // is set to the index after the last of them (0 if there is none); only
+    // there is one, @a line_start is set to the index after the last of them,
-    // valid when supports_seek
+    // otherwise it is left unchanged; only valid when supports_seek
-    std::size_t count_consumed_lines(std::size_t last_index, std::size_t& line_start) const
+    std::size_t count_consumed_lines(std::size_t first_index, std::size_t last_index, std::size_t& line_start) const
     {
-        const auto to = std::next(begin, static_cast<typename std::iterator_traits<IteratorType>::difference_type>(last_index));
+        using difference_type = typename std::iterator_traits<IteratorType>::difference_type;
+        const auto from = std::next(begin, static_cast<difference_type>(first_index));
+        const auto to = std::next(from, static_cast<difference_type>(last_index - first_index));
         const auto newline = char_traits<char_type>::to_char_type('\n');
         using reverse_it = std::reverse_iterator<IteratorType>;
-        const auto last_newline = std::find(reverse_it(to), reverse_it(begin), newline);
+        const auto last_newline = std::find(reverse_it(to), reverse_it(from), newline);
-        line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
+        if (last_newline.base() != from)
-        return static_cast<std::size_t>(std::count(begin, last_newline.base(), newline));
+        {
+            line_start = static_cast<std::size_t>(std::distance(begin, last_newline.base()));
+        }
+        return static_cast<std::size_t>(std::count(from, last_newline.base(), newline));
     }
 
     // append the already-consumed characters in the half-open range
@@ -13209,17 +13214,25 @@ scan_number_done:
         return position.chars_read_total;
     }
 
-    /// seekable adapter: count the lines in the consumed input; O(input read
+    /// seekable adapter: count the lines in the consumed input. Only the
-    /// so far), but only called on errors
+    /// input consumed since the previous call is scanned, so a parse that
+    /// reports many errors (a SAX parser recovering from them) stays linear
+    /// in the input instead of rescanning it for every error.
     position_t get_position_impl(std::true_type /*lazy*/) const
     {
         position_t result;
         result.chars_read_total = get_chars_read_total();
         // reads at EOF count as characters but are not part of the input
         const std::size_t consumed = (std::min)(result.chars_read_total, ia.get_consumed_count());
-        std::size_t line_start = 0;
+        if (JSON_HEDLEY_UNLIKELY(consumed < position_cache.consumed))
-        result.lines_read = ia.count_consumed_lines(consumed, line_start);
+        {
-        result.chars_read_current_line = result.chars_read_total - line_start;
+            // the consumed input never shrinks; recount from the start if it did
+            position_cache = {};
+        }
+        position_cache.lines += ia.count_consumed_lines(position_cache.consumed, consumed, position_cache.line_start);
+        position_cache.consumed = consumed;
+        result.lines_read = position_cache.lines;
+        result.chars_read_current_line = result.chars_read_total - position_cache.line_start;
         return result;
     }
 
@@ -14095,6 +14108,19 @@ scan_number_done:
     /// that unget() can restore the column instead of leaving it at 0
     std::size_t chars_read_before_newline = 0;
 
+    /// seekable adapter: what get_position() counted last, so that the next
+    /// call only scans the input consumed since
+    struct position_cache_t
+    {
+        /// consumed characters scanned so far
+        std::size_t consumed = 0;
+        /// newlines among them
+        std::size_t lines = 0;
+        /// index after the last of these newlines
+        std::size_t line_start = 0;
+    };
+    mutable position_cache_t position_cache {};
+
     /// raw input token string for error messages; only populated for streaming
     /// adapters (seekable adapters reconstruct it lazily via token_string_start)
     std::vector<char_type> token_string {};