Fold ensure_ascii into the escaper and write bytes without dump_integer

Two hot spots that the write buffer and the bulk scanner left behind.

dump_escaped took ensure_ascii as a runtime flag and tested it inside the
loop, once per character run, although it cannot change while a string is
written. It is now a template parameter, dispatched once per string, which
folds the choice of scanner and lets each of the two be inlined into a loop
of its own. This is the hottest loop in the serializer: it runs over every
string and every object key.

A binary value's bytes went through dump_integer, which counts digits and
does 64-bit arithmetic for a number that is always in [0, 255]. dump_byte
writes the three digits it takes at most straight into the write buffer
instead. Any byte type that is not a plain unsigned byte is still left to
dump_integer, whose representation of it may differ.

Measured against the previous commit (medians of 9 interleaved runs, clang
-O3): binary values -33.8%, dense CJK with ensure_ascii -20.6%, key-heavy
objects -17.8%, deeply nested pretty output -17.9%, dense CJK without
ensure_ascii -11.8%, object-heavy documents -9.3% compact and -9.5% pretty,
a small value dumped in a loop -21.4%, wide objects -2.3%. Arrays of plain
ASCII strings measured 3.5% to 4.2% slower, the one shape that loses; number
and integer arrays are unchanged.

Also tried and dropped: leaving the write and string buffers uninitialized
rather than zeroing 1.5 KB per dump() call. It is worth -30% on small values,
but two nearly identical string workloads moved 18% apart in opposite
directions, so the measurements did not support it.

The output is unchanged for every value: the differential now also covers
every one of the 256 byte values, alone and together, in both binary layouts.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/output/serializer.hpp

@@ -305,10 +305,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_literal(", ");
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\n");
@@ -336,10 +336,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_char(',');
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\"subtype\":");
@@ -656,10 +656,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_literal(", ");
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\n");
@@ -687,10 +687,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_char(',');
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\"subtype\":");
@@ -780,6 +780,20 @@ class serializer
     characters by a sequence of "\u" followed by a four-digit hex
     representation. The escaped string is written to output stream @a o.
 
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+
+    @complexity Linear in the length of string @a s.
+    */
+    /*!
+    @brief dump escaped string
+
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
+
     @param[in] s  the string to escape
     @param[in] ensure_ascii  whether to escape non-ASCII characters with
                              \uXXXX sequences
@@ -787,6 +801,22 @@ class serializer
     @complexity Linear in the length of string @a s.
     */
     void dump_escaped(const string_t& s, const bool ensure_ascii)
+    {
+        // dispatch once here rather than test the flag inside the loop: it does
+        // not change while a string is written, and folding it lets each of the
+        // two scanners be inlined into a loop of its own
+        if (ensure_ascii)
+        {
+            dump_escaped_impl<true>(s);
+        }
+        else
+        {
+            dump_escaped_impl<false>(s);
+        }
+    }
+
+    template<bool EnsureAscii>
+    void dump_escaped_impl(const string_t& s)
     {
         std::uint32_t codepoint{};
         std::uint8_t state = UTF8_ACCEPT;
@@ -805,16 +835,16 @@ class serializer
             // individually, so that byte is left to the byte-at-a-time path
             // below, keeping escaping output and error diagnostics unchanged.
             //
-            // - ensure_ascii == false: string_bulk_run() copies ordinary bytes
+            // - EnsureAscii == false: string_bulk_run() copies ordinary bytes
             //   and complete well-formed UTF-8, stopping at a quote, backslash,
             //   control character (< 0x20), or ill-formed/truncated sequence.
-            // - ensure_ascii == true: only printable ASCII may be copied
+            // - EnsureAscii == true: only printable ASCII may be copied
             //   verbatim; find_ascii_copyable_run() additionally stops at 0x7F
             //   and every non-ASCII byte (>= 0x80), which must be \u-escaped.
             if (state == UTF8_ACCEPT)
             {
                 const auto* const data = reinterpret_cast<const unsigned char*>(s.data());
-                const std::size_t run = ensure_ascii
+                const std::size_t run = EnsureAscii
                                         ? find_ascii_copyable_run(data + i, s.size() - i)
                                         : string_bulk_run(data + i, s.size() - i);
                 if (run != 0)
@@ -897,8 +927,8 @@ class serializer
                         default:
                         {
                             // escape control characters (0x00..0x1F) or, if
-                            // ensure_ascii parameter is used, non-ASCII characters
+                            // EnsureAscii parameter is used, non-ASCII characters
-                            if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))
+                            if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
                             {
                                 if (codepoint <= 0xFFFF)
                                 {
@@ -963,7 +993,7 @@ class serializer
                             if (error_handler == error_handler_t::replace)
                             {
                                 // add a replacement character
-                                if (ensure_ascii)
+                                if (EnsureAscii)
                                 {
                                     string_buffer[bytes++] = '\\';
                                     string_buffer[bytes++] = 'u';
@@ -1006,7 +1036,7 @@ class serializer
 
                 default:  // decode found yet incomplete multibyte code point
                 {
-                    if (!ensure_ascii)
+                    if (!EnsureAscii)
                     {
                         // code point will not be escaped - copy byte to buffer
                         string_buffer[bytes++] = s[i];
@@ -1048,7 +1078,7 @@ class serializer
                     // write all accepted bytes
                     put_buffer(string_buffer, bytes_after_last_accept);
                     // add a replacement character
-                    if (ensure_ascii)
+                    if (EnsureAscii)
                     {
                         put_literal("\\ufffd");
                     }
@@ -1320,6 +1350,57 @@ class serializer
         return false;
     }
 
+    /*!
+    @brief write the decimal representation of the byte @a value
+
+    A binary value's bytes are always in [0, 255], so writing one needs neither
+    the digit counting nor the 64-bit arithmetic that @ref dump_integer does for
+    an arbitrary number, and the three digits it takes at most are written
+    straight into the write buffer.
+
+    Any byte type that is not a plain unsigned byte is left to @ref dump_integer,
+    whose representation of it may differ.
+    */
+    template<typename ByteType>
+    void dump_byte(const ByteType value)
+    {
+        dump_byte(value, std::integral_constant < bool,
+                  std::is_unsigned<ByteType>::value && sizeof(ByteType) == 1
+                  && !std::is_same<ByteType, bool>::value > {});
+    }
+
+    template<typename ByteType>
+    void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/)
+    {
+        dump_integer(value);
+    }
+
+    template<typename ByteType>
+    void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/)
+    {
+        if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size()))
+        {
+            flush();
+        }
+
+        const auto byte = static_cast<unsigned>(value);
+        char* out = write_buffer.data() + write_buffer_pos;
+
+        if (byte >= 100)
+        {
+            *out++ = static_cast<char>('0' + (byte / 100));
+            *out++ = static_cast<char>('0' + ((byte / 10) % 10));
+        }
+        else if (byte >= 10)
+        {
+            *out++ = static_cast<char>('0' + (byte / 10));
+        }
+
+        *out++ = static_cast<char>('0' + (byte % 10));
+
+        write_buffer_pos = static_cast<std::size_t>(out - write_buffer.data());
+    }
+
     /*!
     @brief dump an integer
 

single_include/nlohmann/json.hpp

@@ -21285,10 +21285,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_literal(", ");
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\n");
@@ -21316,10 +21316,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_char(',');
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\"subtype\":");
@@ -21636,10 +21636,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_literal(", ");
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\n");
@@ -21667,10 +21667,10 @@ class serializer
                         for (auto i = val.m_data.m_value.binary->cbegin();
                                 i != val.m_data.m_value.binary->cend() - 1; ++i)
                         {
-                            dump_integer(*i);
+                            dump_byte(*i);
                             put_char(',');
                         }
-                        dump_integer(val.m_data.m_value.binary->back());
+                        dump_byte(val.m_data.m_value.binary->back());
                     }
 
                     put_literal("],\"subtype\":");
@@ -21760,6 +21760,20 @@ class serializer
     characters by a sequence of "\u" followed by a four-digit hex
     representation. The escaped string is written to output stream @a o.
 
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+
+    @complexity Linear in the length of string @a s.
+    */
+    /*!
+    @brief dump escaped string
+
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
+
     @param[in] s  the string to escape
     @param[in] ensure_ascii  whether to escape non-ASCII characters with
                              \uXXXX sequences
@@ -21767,6 +21781,22 @@ class serializer
     @complexity Linear in the length of string @a s.
     */
     void dump_escaped(const string_t& s, const bool ensure_ascii)
+    {
+        // dispatch once here rather than test the flag inside the loop: it does
+        // not change while a string is written, and folding it lets each of the
+        // two scanners be inlined into a loop of its own
+        if (ensure_ascii)
+        {
+            dump_escaped_impl<true>(s);
+        }
+        else
+        {
+            dump_escaped_impl<false>(s);
+        }
+    }
+
+    template<bool EnsureAscii>
+    void dump_escaped_impl(const string_t& s)
     {
         std::uint32_t codepoint{};
         std::uint8_t state = UTF8_ACCEPT;
@@ -21785,16 +21815,16 @@ class serializer
             // individually, so that byte is left to the byte-at-a-time path
             // below, keeping escaping output and error diagnostics unchanged.
             //
-            // - ensure_ascii == false: string_bulk_run() copies ordinary bytes
+            // - EnsureAscii == false: string_bulk_run() copies ordinary bytes
             //   and complete well-formed UTF-8, stopping at a quote, backslash,
             //   control character (< 0x20), or ill-formed/truncated sequence.
-            // - ensure_ascii == true: only printable ASCII may be copied
+            // - EnsureAscii == true: only printable ASCII may be copied
             //   verbatim; find_ascii_copyable_run() additionally stops at 0x7F
             //   and every non-ASCII byte (>= 0x80), which must be \u-escaped.
             if (state == UTF8_ACCEPT)
             {
                 const auto* const data = reinterpret_cast<const unsigned char*>(s.data());
-                const std::size_t run = ensure_ascii
+                const std::size_t run = EnsureAscii
                                         ? find_ascii_copyable_run(data + i, s.size() - i)
                                         : string_bulk_run(data + i, s.size() - i);
                 if (run != 0)
@@ -21877,8 +21907,8 @@ class serializer
                         default:
                         {
                             // escape control characters (0x00..0x1F) or, if
-                            // ensure_ascii parameter is used, non-ASCII characters
+                            // EnsureAscii parameter is used, non-ASCII characters
-                            if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))
+                            if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
                             {
                                 if (codepoint <= 0xFFFF)
                                 {
@@ -21943,7 +21973,7 @@ class serializer
                             if (error_handler == error_handler_t::replace)
                             {
                                 // add a replacement character
-                                if (ensure_ascii)
+                                if (EnsureAscii)
                                 {
                                     string_buffer[bytes++] = '\\';
                                     string_buffer[bytes++] = 'u';
@@ -21986,7 +22016,7 @@ class serializer
 
                 default:  // decode found yet incomplete multibyte code point
                 {
-                    if (!ensure_ascii)
+                    if (!EnsureAscii)
                     {
                         // code point will not be escaped - copy byte to buffer
                         string_buffer[bytes++] = s[i];
@@ -22028,7 +22058,7 @@ class serializer
                     // write all accepted bytes
                     put_buffer(string_buffer, bytes_after_last_accept);
                     // add a replacement character
-                    if (ensure_ascii)
+                    if (EnsureAscii)
                     {
                         put_literal("\\ufffd");
                     }
@@ -22300,6 +22330,57 @@ class serializer
         return false;
     }
 
+    /*!
+    @brief write the decimal representation of the byte @a value
+
+    A binary value's bytes are always in [0, 255], so writing one needs neither
+    the digit counting nor the 64-bit arithmetic that @ref dump_integer does for
+    an arbitrary number, and the three digits it takes at most are written
+    straight into the write buffer.
+
+    Any byte type that is not a plain unsigned byte is left to @ref dump_integer,
+    whose representation of it may differ.
+    */
+    template<typename ByteType>
+    void dump_byte(const ByteType value)
+    {
+        dump_byte(value, std::integral_constant < bool,
+                  std::is_unsigned<ByteType>::value && sizeof(ByteType) == 1
+                  && !std::is_same<ByteType, bool>::value > {});
+    }
+
+    template<typename ByteType>
+    void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/)
+    {
+        dump_integer(value);
+    }
+
+    template<typename ByteType>
+    void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/)
+    {
+        if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size()))
+        {
+            flush();
+        }
+
+        const auto byte = static_cast<unsigned>(value);
+        char* out = write_buffer.data() + write_buffer_pos;
+
+        if (byte >= 100)
+        {
+            *out++ = static_cast<char>('0' + (byte / 100));
+            *out++ = static_cast<char>('0' + ((byte / 10) % 10));
+        }
+        else if (byte >= 10)
+        {
+            *out++ = static_cast<char>('0' + (byte / 10));
+        }
+
+        *out++ = static_cast<char>('0' + (byte % 10));
+
+        write_buffer_pos = static_cast<std::size_t>(out - write_buffer.data());
+    }
+
     /*!
     @brief dump an integer