amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-08-21 16:43:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fold ensure_ascii into the escaper and write bytes without dump_integer

Browse Source 
Two hot spots that the write buffer and the bulk scanner left behind.

dump_escaped took ensure_ascii as a runtime flag and tested it inside the
loop, once per character run, although it cannot change while a string is
written. It is now a template parameter, dispatched once per string, which
folds the choice of scanner and lets each of the two be inlined into a loop
of its own. This is the hottest loop in the serializer: it runs over every
string and every object key.

A binary value's bytes went through dump_integer, which counts digits and
does 64-bit arithmetic for a number that is always in [0, 255]. dump_byte
writes the three digits it takes at most straight into the write buffer
instead. Any byte type that is not a plain unsigned byte is still left to
dump_integer, whose representation of it may differ.

Measured against the previous commit (medians of 9 interleaved runs, clang
-O3): binary values -33.8%, dense CJK with ensure_ascii -20.6%, key-heavy
objects -17.8%, deeply nested pretty output -17.9%, dense CJK without
ensure_ascii -11.8%, object-heavy documents -9.3% compact and -9.5% pretty,
a small value dumped in a loop -21.4%, wide objects -2.3%. Arrays of plain
ASCII strings measured 3.5% to 4.2% slower, the one shape that loses; number
and integer arrays are unchanged.

Also tried and dropped: leaving the write and string buffers uninitialized
rather than zeroing 1.5 KB per dump() call. It is worth -30% on small values,
but two nearly identical string workloads moved 18% apart in opposite
directions, so the measurements did not support it.

The output is unchanged for every value: the differential now also covers
every one of the 256 byte values, alone and together, in both binary layouts.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-08-21 04:51:22 +02:00
parent 5bf19dc28d
commit aa80ade75b
2 changed files with 194 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/output/serializer.hpp
+97 -16
View File
@@ -305,10 +305,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_literal(", ");
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\n");
@@ -336,10 +336,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_char(',');
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\"subtype\":");
@@ -656,10 +656,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_literal(", ");
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\n");
@@ -687,10 +687,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_char(',');
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\"subtype\":");
@@ -780,6 +780,20 @@ class serializer
characters by a sequence of "\u" followed by a four-digit hex
representation. The escaped string is written to output stream @a o.
@param[in] s the string to escape
@param[in] ensure_ascii whether to escape non-ASCII characters with
\uXXXX sequences
@complexity Linear in the length of string @a s.
*/
/*!
@brief dump escaped string
Escape a string by replacing certain special characters by a sequence of an
escape character (backslash) and another character and other control
characters by a sequence of "\u" followed by a four-digit hex
representation. The escaped string is written to output stream @a o.
@param[in] s the string to escape
@param[in] ensure_ascii whether to escape non-ASCII characters with
\uXXXX sequences
@@ -787,6 +801,22 @@ class serializer
@complexity Linear in the length of string @a s.
*/
void dump_escaped(const string_t& s, const bool ensure_ascii)
{
// dispatch once here rather than test the flag inside the loop: it does
// not change while a string is written, and folding it lets each of the
// two scanners be inlined into a loop of its own
if (ensure_ascii)
{
dump_escaped_impl<true>(s);
}
else
{
dump_escaped_impl<false>(s);
}
}
template<bool EnsureAscii>
void dump_escaped_impl(const string_t& s)
{
std::uint32_t codepoint{};
std::uint8_t state = UTF8_ACCEPT;
@@ -805,16 +835,16 @@ class serializer
// individually, so that byte is left to the byte-at-a-time path
// below, keeping escaping output and error diagnostics unchanged.
//
// - ensure_ascii == false: string_bulk_run() copies ordinary bytes
// - EnsureAscii == false: string_bulk_run() copies ordinary bytes
// and complete well-formed UTF-8, stopping at a quote, backslash,
// control character (< 0x20), or ill-formed/truncated sequence.
// - ensure_ascii == true: only printable ASCII may be copied
// - EnsureAscii == true: only printable ASCII may be copied
// verbatim; find_ascii_copyable_run() additionally stops at 0x7F
// and every non-ASCII byte (>= 0x80), which must be \u-escaped.
if (state == UTF8_ACCEPT)
{
const auto* const data = reinterpret_cast<const unsigned char*>(s.data());
const std::size_t run = ensure_ascii
const std::size_t run = EnsureAscii
? find_ascii_copyable_run(data + i, s.size() - i)
: string_bulk_run(data + i, s.size() - i);
if (run != 0)
@@ -897,8 +927,8 @@ class serializer
default:
{
// escape control characters (0x00..0x1F) or, if
// ensure_ascii parameter is used, non-ASCII characters
if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))
// EnsureAscii parameter is used, non-ASCII characters
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
{
if (codepoint <= 0xFFFF)
{
@@ -963,7 +993,7 @@ class serializer
if (error_handler == error_handler_t::replace)
{
// add a replacement character
if (ensure_ascii)
if (EnsureAscii)
{
string_buffer[bytes++] = '\\';
string_buffer[bytes++] = 'u';
@@ -1006,7 +1036,7 @@ class serializer
default: // decode found yet incomplete multibyte code point
{
if (!ensure_ascii)
if (!EnsureAscii)
{
// code point will not be escaped - copy byte to buffer
string_buffer[bytes++] = s[i];
@@ -1048,7 +1078,7 @@ class serializer
// write all accepted bytes
put_buffer(string_buffer, bytes_after_last_accept);
// add a replacement character
if (ensure_ascii)
if (EnsureAscii)
{
put_literal("\\ufffd");
}
@@ -1320,6 +1350,57 @@ class serializer
return false;
}
/*!
@brief write the decimal representation of the byte @a value
A binary value's bytes are always in [0, 255], so writing one needs neither
the digit counting nor the 64-bit arithmetic that @ref dump_integer does for
an arbitrary number, and the three digits it takes at most are written
straight into the write buffer.
Any byte type that is not a plain unsigned byte is left to @ref dump_integer,
whose representation of it may differ.
*/
template<typename ByteType>
void dump_byte(const ByteType value)
{
dump_byte(value, std::integral_constant < bool,
std::is_unsigned<ByteType>::value && sizeof(ByteType) == 1
&& !std::is_same<ByteType, bool>::value > {});
}
template<typename ByteType>
void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/)
{
dump_integer(value);
}
template<typename ByteType>
void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/)
{
if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size()))
{
flush();
}
const auto byte = static_cast<unsigned>(value);
char* out = write_buffer.data() + write_buffer_pos;
if (byte >= 100)
{
*out++ = static_cast<char>('0' + (byte / 100));
*out++ = static_cast<char>('0' + ((byte / 10) % 10));
}
else if (byte >= 10)
{
*out++ = static_cast<char>('0' + (byte / 10));
}
*out++ = static_cast<char>('0' + (byte % 10));
write_buffer_pos = static_cast<std::size_t>(out - write_buffer.data());
}
/*!
@brief dump an integer
single_include/nlohmann/json.hpp
+97 -16
View File
@@ -21238,10 +21238,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_literal(", ");
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\n");
@@ -21269,10 +21269,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_char(',');
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\"subtype\":");
@@ -21589,10 +21589,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_literal(", ");
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\n");
@@ -21620,10 +21620,10 @@ class serializer
for (auto i = val.m_data.m_value.binary->cbegin();
i != val.m_data.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
dump_byte(*i);
put_char(',');
}
dump_integer(val.m_data.m_value.binary->back());
dump_byte(val.m_data.m_value.binary->back());
}
put_literal("],\"subtype\":");
@@ -21713,6 +21713,20 @@ class serializer
characters by a sequence of "\u" followed by a four-digit hex
representation. The escaped string is written to output stream @a o.
@param[in] s the string to escape
@param[in] ensure_ascii whether to escape non-ASCII characters with
\uXXXX sequences
@complexity Linear in the length of string @a s.
*/
/*!
@brief dump escaped string
Escape a string by replacing certain special characters by a sequence of an
escape character (backslash) and another character and other control
characters by a sequence of "\u" followed by a four-digit hex
representation. The escaped string is written to output stream @a o.
@param[in] s the string to escape
@param[in] ensure_ascii whether to escape non-ASCII characters with
\uXXXX sequences
@@ -21720,6 +21734,22 @@ class serializer
@complexity Linear in the length of string @a s.
*/
void dump_escaped(const string_t& s, const bool ensure_ascii)
{
// dispatch once here rather than test the flag inside the loop: it does
// not change while a string is written, and folding it lets each of the
// two scanners be inlined into a loop of its own
if (ensure_ascii)
{
dump_escaped_impl<true>(s);
}
else
{
dump_escaped_impl<false>(s);
}
}
template<bool EnsureAscii>
void dump_escaped_impl(const string_t& s)
{
std::uint32_t codepoint{};
std::uint8_t state = UTF8_ACCEPT;
@@ -21738,16 +21768,16 @@ class serializer
// individually, so that byte is left to the byte-at-a-time path
// below, keeping escaping output and error diagnostics unchanged.
//
// - ensure_ascii == false: string_bulk_run() copies ordinary bytes
// - EnsureAscii == false: string_bulk_run() copies ordinary bytes
// and complete well-formed UTF-8, stopping at a quote, backslash,
// control character (< 0x20), or ill-formed/truncated sequence.
// - ensure_ascii == true: only printable ASCII may be copied
// - EnsureAscii == true: only printable ASCII may be copied
// verbatim; find_ascii_copyable_run() additionally stops at 0x7F
// and every non-ASCII byte (>= 0x80), which must be \u-escaped.
if (state == UTF8_ACCEPT)
{
const auto* const data = reinterpret_cast<const unsigned char*>(s.data());
const std::size_t run = ensure_ascii
const std::size_t run = EnsureAscii
? find_ascii_copyable_run(data + i, s.size() - i)
: string_bulk_run(data + i, s.size() - i);
if (run != 0)
@@ -21830,8 +21860,8 @@ class serializer
default:
{
// escape control characters (0x00..0x1F) or, if
// ensure_ascii parameter is used, non-ASCII characters
if ((codepoint <= 0x1F) || (ensure_ascii && (codepoint >= 0x7F)))
// EnsureAscii parameter is used, non-ASCII characters
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
{
if (codepoint <= 0xFFFF)
{
@@ -21896,7 +21926,7 @@ class serializer
if (error_handler == error_handler_t::replace)
{
// add a replacement character
if (ensure_ascii)
if (EnsureAscii)
{
string_buffer[bytes++] = '\\';
string_buffer[bytes++] = 'u';
@@ -21939,7 +21969,7 @@ class serializer
default: // decode found yet incomplete multibyte code point
{
if (!ensure_ascii)
if (!EnsureAscii)
{
// code point will not be escaped - copy byte to buffer
string_buffer[bytes++] = s[i];
@@ -21981,7 +22011,7 @@ class serializer
// write all accepted bytes
put_buffer(string_buffer, bytes_after_last_accept);
// add a replacement character
if (ensure_ascii)
if (EnsureAscii)
{
put_literal("\\ufffd");
}
@@ -22253,6 +22283,57 @@ class serializer
return false;
}
/*!
@brief write the decimal representation of the byte @a value
A binary value's bytes are always in [0, 255], so writing one needs neither
the digit counting nor the 64-bit arithmetic that @ref dump_integer does for
an arbitrary number, and the three digits it takes at most are written
straight into the write buffer.
Any byte type that is not a plain unsigned byte is left to @ref dump_integer,
whose representation of it may differ.
*/
template<typename ByteType>
void dump_byte(const ByteType value)
{
dump_byte(value, std::integral_constant < bool,
std::is_unsigned<ByteType>::value && sizeof(ByteType) == 1
&& !std::is_same<ByteType, bool>::value > {});
}
template<typename ByteType>
void dump_byte(const ByteType value, std::false_type /*is_plain_byte*/)
{
dump_integer(value);
}
template<typename ByteType>
void dump_byte(const ByteType value, std::true_type /*is_plain_byte*/)
{
if (JSON_HEDLEY_UNLIKELY(write_buffer_pos + 3 > write_buffer.size()))
{
flush();
}
const auto byte = static_cast<unsigned>(value);
char* out = write_buffer.data() + write_buffer_pos;
if (byte >= 100)
{
*out++ = static_cast<char>('0' + (byte / 100));
*out++ = static_cast<char>('0' + ((byte / 10) % 10));
}
else if (byte >= 10)
{
*out++ = static_cast<char>('0' + (byte / 10));
}
*out++ = static_cast<char>('0' + (byte % 10));
write_buffer_pos = static_cast<std::size_t>(out - write_buffer.data());
}
/*!
@brief dump an integer