Keep the string scan from slowing down short strings and dump() on Arm

Three benchmarks were 7-9% slower than develop on Apple M1 (Apple Clang,
-O3 -flto): Accept/jeopardy, Dump/citm_catalog, and Dump/jeopardy.

- dump(): the serializer passes a string without its closing quote, so
  plain_run_end() ran out of input and called the out-of-line NEON loop
  for every string, with fewer than 16 bytes left. The loop is now only
  called for 16 bytes or more.
- accept(): no string of jeopardy.json reaches the vector UTF-8 check,
  yet compiling it into the lexer's scan function made it 5% slower: the
  function runs once per token and now saved more registers and kept a
  stack slot. lexer_string_bulk_run() checks the first four words inline
  and continues out of line, and the serializer keeps string_bulk_run()
  inline, which is faster there.
- The byte that ends a run is read from the scanned buffer instead of
  through get(), which Apple Clang merged with the other get() calls that
  load the read position again right after it was stored (3%).

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/lexer.hpp

@@ -417,30 +417,43 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// contiguous input: bulk-append the run of ordinary characters and complete
-    /// well-formed UTF-8 sequences starting at the current read position, leaving
-    /// the first byte that needs individual handling (the closing quote, an
-    /// escape, a control character, or an ill-formed UTF-8 byte) for get()
-    void scan_string_bulk(std::true_type /*bulk*/)
+    /// well-formed UTF-8 sequences starting at the current read position, then
+    /// read the first byte that needs individual handling (the closing quote, an
+    /// escape, a control character, or an ill-formed UTF-8 byte) like get().
+    /// That byte is taken from the buffer that was just scanned. Through get(),
+    /// Apple Clang merged this read with the other get() calls of the scan
+    /// function, which load the read position again right after it was stored
+    /// (3% of json::accept() on jeopardy.json on Apple M1).
+    char_int_type get_after_string_run(std::true_type /*bulk*/)
     {
         // a pending unget must be consumed through the normal path first
         if (next_unget)
         {
-            return;
+            return get();
         }
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
         {
-            return;
+            return get();
         }
-        const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
+        const auto* const raw = ia.bulk_data();
+        const auto* const data = reinterpret_cast<const unsigned char*>(raw);
 
-        const std::size_t pos = string_bulk_run(data, remaining);
-        if (pos == 0)
+        const std::size_t pos = lexer_string_bulk_run(data, remaining);
+        if (pos != 0)
         {
-            return;
+            token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
+            bulk_consume(reinterpret_cast<const char*>(data), pos);
         }
-        token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
-        bulk_consume(reinterpret_cast<const char*>(data), pos);
+        if (pos == remaining)
+        {
+            return get(); // end of the contiguous input
+        }
+        // what get() does, with the byte at hand
+        advance_position();
+        ia.bulk_skip(1);
+        current = char_traits<char_type>::to_int_type(raw[pos]);
+        return track_after_read();
     }
 
     /// consume @a n characters at @a data (the adapter's bulk_data()) and do
@@ -473,7 +486,10 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// streaming input: no bulk fast path
-    void scan_string_bulk(std::false_type /*bulk*/) const noexcept {}
+    char_int_type get_after_string_run(std::false_type /*bulk*/)
+    {
+        return get();
+    }
 
     /*!
     @brief scan a string literal
@@ -501,11 +517,8 @@ class lexer : public lexer_base<BasicJsonType>
         while (true)
         {
             // bulk-consume ordinary characters from contiguous input, then
-            // handle the next special byte through the switch below
-            scan_string_bulk(std::integral_constant<bool, bulk_scan> {});
-
-            // get the next character
-            switch (get())
+            // get the next character, the one that ends the run
+            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
             {
                 // end of file while parsing the string
                 case char_traits<char_type>::eof():

include/nlohmann/detail/input/string_scan.hpp

@@ -35,7 +35,7 @@
 // This file contains the byte-level string-scanning helpers used by the lexer's
 // contiguous fast path. They operate purely on raw bytes (no dependency on the
 // lexer's template parameters) so they are free functions, keeping the lexer
-// itself focused on the state machine; see lexer::scan_string_bulk().
+// itself focused on the state machine; see lexer::get_after_string_run().
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
@@ -311,12 +311,19 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char
         }
         p += 8;
     }
-    p = vector_plain_run_long(p, e);
-    if (e - p < 16) // else the vector loop stopped at a special byte
+    // the vector loop only for 16 bytes or more: the serializer passes a
+    // string without its closing quote, so most of its runs end at e, and a
+    // call of the out-of-line NEON loop for a shorter rest made dump() up to
+    // 10% slower
+    if (e - p >= 16)
     {
-        p += find_string_special(p, static_cast<std::size_t>(e - p));
+        p = vector_plain_run_long(p, e);
+        if (e - p >= 16)
+        {
+            return p; // the vector loop stopped at a special byte
+        }
     }
-    return p;
+    return p + find_string_special(p, static_cast<std::size_t>(e - p));
 }
 #endif
 
@@ -418,6 +425,42 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data,
     return scalar_string_bulk_run(data, n);
 }
 
+#if JSON_SCAN_NEON
+/// scalar_string_bulk_run() from @a start on, out of line (see
+/// lexer_string_bulk_run())
+JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned char* data, std::size_t start, std::size_t n) noexcept
+{
+    return start + scalar_string_bulk_run(data + start, n - start);
+}
+#endif
+
+// string_bulk_run() for the lexer. With NEON, only the first four words are
+// checked inline (most keys and short values end there), and longer runs and
+// non-ASCII text continue out of line. The lexer's scan function runs once per
+// token, and the vector code inlined into it needs more registers, which every
+// token pays for: 5% of json::accept() on jeopardy.json with Apple Clang on
+// Apple M1, even though no string of that file reaches the vector code. The
+// serializer keeps string_bulk_run() inline, which is faster there.
+JSON_HEDLEY_ALWAYS_INLINE std::size_t lexer_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
+{
+#if JSON_SCAN_NEON && !(defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17))
+    std::size_t i = 0;
+    for (int words = 4; words > 0 && n - i >= 8; --words)
+    {
+        const std::uint64_t special = swar_string_special(read_eight_bytes(data + i));
+        if (special != 0)
+        {
+            const std::size_t pos = i + (static_cast<std::size_t>(count_trailing_zeros(special)) / 8);
+            return data[pos] < 0x80u ? pos : string_bulk_run_from(data, pos, n);
+        }
+        i += 8;
+    }
+    return string_bulk_run_from(data, i, n);
+#else
+    return string_bulk_run(data, n);
+#endif
+}
+
 // Decode the 4 hex digits at [data, data+4) - the digits following a `\u`
 // escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte
 // (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a

single_include/nlohmann/json.hpp

@@ -11493,7 +11493,7 @@ NLOHMANN_JSON_NAMESPACE_END
 // This file contains the byte-level string-scanning helpers used by the lexer's
 // contiguous fast path. They operate purely on raw bytes (no dependency on the
 // lexer's template parameters) so they are free functions, keeping the lexer
-// itself focused on the state machine; see lexer::scan_string_bulk().
+// itself focused on the state machine; see lexer::get_after_string_run().
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
@@ -11769,12 +11769,19 @@ JSON_HEDLEY_ALWAYS_INLINE const unsigned char* plain_run_end(const unsigned char
         }
         p += 8;
     }
-    p = vector_plain_run_long(p, e);
-    if (e - p < 16) // else the vector loop stopped at a special byte
+    // the vector loop only for 16 bytes or more: the serializer passes a
+    // string without its closing quote, so most of its runs end at e, and a
+    // call of the out-of-line NEON loop for a shorter rest made dump() up to
+    // 10% slower
+    if (e - p >= 16)
     {
-        p += find_string_special(p, static_cast<std::size_t>(e - p));
+        p = vector_plain_run_long(p, e);
+        if (e - p >= 16)
+        {
+            return p; // the vector loop stopped at a special byte
+        }
     }
-    return p;
+    return p + find_string_special(p, static_cast<std::size_t>(e - p));
 }
 #endif
 
@@ -11876,6 +11883,42 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t string_bulk_run(const unsigned char* data,
     return scalar_string_bulk_run(data, n);
 }
 
+#if JSON_SCAN_NEON
+/// scalar_string_bulk_run() from @a start on, out of line (see
+/// lexer_string_bulk_run())
+JSON_HEDLEY_NEVER_INLINE inline std::size_t string_bulk_run_from(const unsigned char* data, std::size_t start, std::size_t n) noexcept
+{
+    return start + scalar_string_bulk_run(data + start, n - start);
+}
+#endif
+
+// string_bulk_run() for the lexer. With NEON, only the first four words are
+// checked inline (most keys and short values end there), and longer runs and
+// non-ASCII text continue out of line. The lexer's scan function runs once per
+// token, and the vector code inlined into it needs more registers, which every
+// token pays for: 5% of json::accept() on jeopardy.json with Apple Clang on
+// Apple M1, even though no string of that file reaches the vector code. The
+// serializer keeps string_bulk_run() inline, which is faster there.
+JSON_HEDLEY_ALWAYS_INLINE std::size_t lexer_string_bulk_run(const unsigned char* data, std::size_t n) noexcept
+{
+#if JSON_SCAN_NEON && !(defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17))
+    std::size_t i = 0;
+    for (int words = 4; words > 0 && n - i >= 8; --words)
+    {
+        const std::uint64_t special = swar_string_special(read_eight_bytes(data + i));
+        if (special != 0)
+        {
+            const std::size_t pos = i + (static_cast<std::size_t>(count_trailing_zeros(special)) / 8);
+            return data[pos] < 0x80u ? pos : string_bulk_run_from(data, pos, n);
+        }
+        i += 8;
+    }
+    return string_bulk_run_from(data, i, n);
+#else
+    return string_bulk_run(data, n);
+#endif
+}
+
 // Decode the 4 hex digits at [data, data+4) - the digits following a `\u`
 // escape - into a codepoint 0x0000..0xFFFF via one table lookup per byte
 // (after yyjson's read_hex_u16), or return -1 if any of the 4 bytes is not a
@@ -12322,30 +12365,43 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// contiguous input: bulk-append the run of ordinary characters and complete
-    /// well-formed UTF-8 sequences starting at the current read position, leaving
-    /// the first byte that needs individual handling (the closing quote, an
-    /// escape, a control character, or an ill-formed UTF-8 byte) for get()
-    void scan_string_bulk(std::true_type /*bulk*/)
+    /// well-formed UTF-8 sequences starting at the current read position, then
+    /// read the first byte that needs individual handling (the closing quote, an
+    /// escape, a control character, or an ill-formed UTF-8 byte) like get().
+    /// That byte is taken from the buffer that was just scanned. Through get(),
+    /// Apple Clang merged this read with the other get() calls of the scan
+    /// function, which load the read position again right after it was stored
+    /// (3% of json::accept() on jeopardy.json on Apple M1).
+    char_int_type get_after_string_run(std::true_type /*bulk*/)
     {
         // a pending unget must be consumed through the normal path first
         if (next_unget)
         {
-            return;
+            return get();
         }
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
         {
-            return;
+            return get();
         }
-        const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
+        const auto* const raw = ia.bulk_data();
+        const auto* const data = reinterpret_cast<const unsigned char*>(raw);
 
-        const std::size_t pos = string_bulk_run(data, remaining);
-        if (pos == 0)
+        const std::size_t pos = lexer_string_bulk_run(data, remaining);
+        if (pos != 0)
         {
-            return;
+            token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
+            bulk_consume(reinterpret_cast<const char*>(data), pos);
         }
-        token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
-        bulk_consume(reinterpret_cast<const char*>(data), pos);
+        if (pos == remaining)
+        {
+            return get(); // end of the contiguous input
+        }
+        // what get() does, with the byte at hand
+        advance_position();
+        ia.bulk_skip(1);
+        current = char_traits<char_type>::to_int_type(raw[pos]);
+        return track_after_read();
     }
 
     /// consume @a n characters at @a data (the adapter's bulk_data()) and do
@@ -12378,7 +12434,10 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// streaming input: no bulk fast path
-    void scan_string_bulk(std::false_type /*bulk*/) const noexcept {}
+    char_int_type get_after_string_run(std::false_type /*bulk*/)
+    {
+        return get();
+    }
 
     /*!
     @brief scan a string literal
@@ -12406,11 +12465,8 @@ class lexer : public lexer_base<BasicJsonType>
         while (true)
         {
             // bulk-consume ordinary characters from contiguous input, then
-            // handle the next special byte through the switch below
-            scan_string_bulk(std::integral_constant<bool, bulk_scan> {});
-
-            // get the next character
-            switch (get())
+            // get the next character, the one that ends the run
+            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
             {
                 // end of file while parsing the string
                 case char_traits<char_type>::eof():