mirror of
https://github.com/nlohmann/json.git
synced 2026-10-14 18:27:13 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Niels Lohmann
|
9dc60905ed | ||
|
Niels Lohmann
|
467670aa7a |
No files matched your search
@@ -772,33 +772,6 @@ class serializer
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A byte below 0x60 is a complete code point and needs no
|
||||
// UTF-8 decoding. The scanner stops only at a control
|
||||
// character, a quote, or a backslash in that range, so look up
|
||||
// its escape sequence and write all 8 bytes of the table entry
|
||||
// straight into the write buffer; only the length stored in the
|
||||
// last byte of the entry is kept.
|
||||
if (data[i] < 0x60)
|
||||
{
|
||||
// emit any bytes still pending in string_buffer first to
|
||||
// preserve output order
|
||||
if (bytes != 0)
|
||||
{
|
||||
put_buffer(string_buffer, bytes);
|
||||
bytes = 0;
|
||||
}
|
||||
const auto& escape = ascii_escapes()[data[i]];
|
||||
if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size()))
|
||||
{
|
||||
flush();
|
||||
}
|
||||
std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size());
|
||||
write_buffer_pos += static_cast<std::size_t>(escape[7]);
|
||||
bytes_after_last_accept = 0;
|
||||
undumped_chars = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const auto byte = static_cast<std::uint8_t>(s[i]);
|
||||
@@ -807,38 +780,91 @@ class serializer
|
||||
{
|
||||
case UTF8_ACCEPT: // decode found a new code point
|
||||
{
|
||||
// the code point is at least 0x60 (smaller ones are handled
|
||||
// above) and is not printable ASCII (which the scanner
|
||||
// copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII
|
||||
// character
|
||||
if (EnsureAscii)
|
||||
switch (codepoint)
|
||||
{
|
||||
JSON_ASSERT(codepoint >= 0x7F);
|
||||
if (error_handler == error_handler_t::keep)
|
||||
case 0x08: // backspace
|
||||
{
|
||||
// this character was buffered as raw bytes below
|
||||
// in case it turned out to be part of an
|
||||
// ill-formed sequence (which is kept as is); now
|
||||
// that it decoded to a well-formed code point,
|
||||
// undo that and \u-escape it like any other
|
||||
// character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'b';
|
||||
break;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
|
||||
case 0x09: // horizontal tab
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 't';
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
case 0x0A: // newline
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'n';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0C: // formfeed
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'f';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0D: // carriage return
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'r';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x22: // quotation mark
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '"';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x5C: // reverse solidus
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '\\';
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
// escape control characters (0x00..0x1F) or, if
|
||||
// EnsureAscii parameter is used, non-ASCII characters
|
||||
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
|
||||
{
|
||||
if (EnsureAscii && error_handler == error_handler_t::keep)
|
||||
{
|
||||
// this character was buffered as raw bytes
|
||||
// below in case it turned out to be part of
|
||||
// an ill-formed sequence (which is kept as
|
||||
// is); now that it decoded to a well-formed
|
||||
// code point, undo that and \u-escape it
|
||||
// like any other character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes
|
||||
// been copied have in default case above)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes of the
|
||||
// sequence have been copied in the default case below)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
|
||||
// write buffer and reset index; there must be 13 bytes
|
||||
@@ -1227,121 +1253,6 @@ class serializer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the JSON form of each character below 0x60
|
||||
|
||||
Entry c holds the escape sequence of character c - the short form for
|
||||
\\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters -
|
||||
or c itself if it needs no escaping, padded with zeros to 7 bytes. The last
|
||||
byte is the length of the sequence. The 8 bytes of an entry are copied at
|
||||
once, so writing an escape needs neither a branch nor a byte-wise loop.
|
||||
*/
|
||||
static const std::array<std::array<char, 8>, 0x60>& ascii_escapes() noexcept
|
||||
{
|
||||
static const std::array<std::array<char, 8>, 0x60> escapes =
|
||||
{
|
||||
{
|
||||
{{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00
|
||||
{{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01
|
||||
{{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02
|
||||
{{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03
|
||||
{{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04
|
||||
{{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05
|
||||
{{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06
|
||||
{{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07
|
||||
{{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08
|
||||
{{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09
|
||||
{{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A
|
||||
{{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B
|
||||
{{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C
|
||||
{{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D
|
||||
{{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E
|
||||
{{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F
|
||||
{{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10
|
||||
{{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11
|
||||
{{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12
|
||||
{{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13
|
||||
{{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14
|
||||
{{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15
|
||||
{{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16
|
||||
{{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17
|
||||
{{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18
|
||||
{{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19
|
||||
{{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A
|
||||
{{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B
|
||||
{{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C
|
||||
{{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D
|
||||
{{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E
|
||||
{{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F
|
||||
{{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20
|
||||
{{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21
|
||||
{{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22
|
||||
{{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23
|
||||
{{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24
|
||||
{{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25
|
||||
{{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26
|
||||
{{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27
|
||||
{{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28
|
||||
{{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29
|
||||
{{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A
|
||||
{{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B
|
||||
{{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C
|
||||
{{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D
|
||||
{{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E
|
||||
{{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F
|
||||
{{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30
|
||||
{{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31
|
||||
{{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32
|
||||
{{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33
|
||||
{{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34
|
||||
{{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35
|
||||
{{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36
|
||||
{{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37
|
||||
{{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38
|
||||
{{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39
|
||||
{{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A
|
||||
{{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B
|
||||
{{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C
|
||||
{{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D
|
||||
{{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E
|
||||
{{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F
|
||||
{{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40
|
||||
{{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41
|
||||
{{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42
|
||||
{{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43
|
||||
{{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44
|
||||
{{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45
|
||||
{{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46
|
||||
{{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47
|
||||
{{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48
|
||||
{{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49
|
||||
{{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A
|
||||
{{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B
|
||||
{{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C
|
||||
{{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D
|
||||
{{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E
|
||||
{{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F
|
||||
{{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50
|
||||
{{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51
|
||||
{{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52
|
||||
{{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53
|
||||
{{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54
|
||||
{{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55
|
||||
{{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56
|
||||
{{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57
|
||||
{{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58
|
||||
{{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59
|
||||
{{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A
|
||||
{{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B
|
||||
{{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C
|
||||
{{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D
|
||||
{{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E
|
||||
{{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F
|
||||
}
|
||||
};
|
||||
return escapes;
|
||||
}
|
||||
|
||||
/*!
|
||||
* @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer
|
||||
*
|
||||
|
||||
Generated
+78
-167
@@ -27087,33 +27087,6 @@ class serializer
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// A byte below 0x60 is a complete code point and needs no
|
||||
// UTF-8 decoding. The scanner stops only at a control
|
||||
// character, a quote, or a backslash in that range, so look up
|
||||
// its escape sequence and write all 8 bytes of the table entry
|
||||
// straight into the write buffer; only the length stored in the
|
||||
// last byte of the entry is kept.
|
||||
if (data[i] < 0x60)
|
||||
{
|
||||
// emit any bytes still pending in string_buffer first to
|
||||
// preserve output order
|
||||
if (bytes != 0)
|
||||
{
|
||||
put_buffer(string_buffer, bytes);
|
||||
bytes = 0;
|
||||
}
|
||||
const auto& escape = ascii_escapes()[data[i]];
|
||||
if (JSON_HEDLEY_UNLIKELY(write_buffer.size() - write_buffer_pos < escape.size()))
|
||||
{
|
||||
flush();
|
||||
}
|
||||
std::memcpy(write_buffer.data() + write_buffer_pos, escape.data(), escape.size());
|
||||
write_buffer_pos += static_cast<std::size_t>(escape[7]);
|
||||
bytes_after_last_accept = 0;
|
||||
undumped_chars = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const auto byte = static_cast<std::uint8_t>(s[i]);
|
||||
@@ -27122,38 +27095,91 @@ class serializer
|
||||
{
|
||||
case UTF8_ACCEPT: // decode found a new code point
|
||||
{
|
||||
// the code point is at least 0x60 (smaller ones are handled
|
||||
// above) and is not printable ASCII (which the scanner
|
||||
// copies): it is DEL (0x7F) under EnsureAscii or a non-ASCII
|
||||
// character
|
||||
if (EnsureAscii)
|
||||
switch (codepoint)
|
||||
{
|
||||
JSON_ASSERT(codepoint >= 0x7F);
|
||||
if (error_handler == error_handler_t::keep)
|
||||
case 0x08: // backspace
|
||||
{
|
||||
// this character was buffered as raw bytes below
|
||||
// in case it turned out to be part of an
|
||||
// ill-formed sequence (which is kept as is); now
|
||||
// that it decoded to a well-formed code point,
|
||||
// undo that and \u-escape it like any other
|
||||
// character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'b';
|
||||
break;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
|
||||
case 0x09: // horizontal tab
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 't';
|
||||
break;
|
||||
}
|
||||
else
|
||||
|
||||
case 0x0A: // newline
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'n';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0C: // formfeed
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'f';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x0D: // carriage return
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = 'r';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x22: // quotation mark
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '"';
|
||||
break;
|
||||
}
|
||||
|
||||
case 0x5C: // reverse solidus
|
||||
{
|
||||
string_buffer[bytes++] = '\\';
|
||||
string_buffer[bytes++] = '\\';
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
// escape control characters (0x00..0x1F) or, if
|
||||
// EnsureAscii parameter is used, non-ASCII characters
|
||||
if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
|
||||
{
|
||||
if (EnsureAscii && error_handler == error_handler_t::keep)
|
||||
{
|
||||
// this character was buffered as raw bytes
|
||||
// below in case it turned out to be part of
|
||||
// an ill-formed sequence (which is kept as
|
||||
// is); now that it decoded to a well-formed
|
||||
// code point, undo that and \u-escape it
|
||||
// like any other character instead
|
||||
bytes = bytes_after_last_accept;
|
||||
}
|
||||
if (codepoint <= 0xFFFF)
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
|
||||
}
|
||||
else
|
||||
{
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xD7C0u + (codepoint >> 10u)));
|
||||
write_u_escape(bytes, static_cast<std::uint16_t>(0xDC00u + (codepoint & 0x3FFu)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes
|
||||
// been copied have in default case above)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// copy byte to buffer (all previous bytes of the
|
||||
// sequence have been copied in the default case below)
|
||||
string_buffer[bytes++] = s[i];
|
||||
}
|
||||
|
||||
// write buffer and reset index; there must be 13 bytes
|
||||
@@ -27542,121 +27568,6 @@ class serializer
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
@brief the JSON form of each character below 0x60
|
||||
|
||||
Entry c holds the escape sequence of character c - the short form for
|
||||
\\b, \\t, \\n, \\f, \\r, \\" and \\\\, \\u00xx for the other control characters -
|
||||
or c itself if it needs no escaping, padded with zeros to 7 bytes. The last
|
||||
byte is the length of the sequence. The 8 bytes of an entry are copied at
|
||||
once, so writing an escape needs neither a branch nor a byte-wise loop.
|
||||
*/
|
||||
static const std::array<std::array<char, 8>, 0x60>& ascii_escapes() noexcept
|
||||
{
|
||||
static const std::array<std::array<char, 8>, 0x60> escapes =
|
||||
{
|
||||
{
|
||||
{{'\\', 'u', '0', '0', '0', '0', 0, 6}}, // 0x00
|
||||
{{'\\', 'u', '0', '0', '0', '1', 0, 6}}, // 0x01
|
||||
{{'\\', 'u', '0', '0', '0', '2', 0, 6}}, // 0x02
|
||||
{{'\\', 'u', '0', '0', '0', '3', 0, 6}}, // 0x03
|
||||
{{'\\', 'u', '0', '0', '0', '4', 0, 6}}, // 0x04
|
||||
{{'\\', 'u', '0', '0', '0', '5', 0, 6}}, // 0x05
|
||||
{{'\\', 'u', '0', '0', '0', '6', 0, 6}}, // 0x06
|
||||
{{'\\', 'u', '0', '0', '0', '7', 0, 6}}, // 0x07
|
||||
{{'\\', 'b', 0, 0, 0, 0, 0, 2}}, // 0x08
|
||||
{{'\\', 't', 0, 0, 0, 0, 0, 2}}, // 0x09
|
||||
{{'\\', 'n', 0, 0, 0, 0, 0, 2}}, // 0x0A
|
||||
{{'\\', 'u', '0', '0', '0', 'b', 0, 6}}, // 0x0B
|
||||
{{'\\', 'f', 0, 0, 0, 0, 0, 2}}, // 0x0C
|
||||
{{'\\', 'r', 0, 0, 0, 0, 0, 2}}, // 0x0D
|
||||
{{'\\', 'u', '0', '0', '0', 'e', 0, 6}}, // 0x0E
|
||||
{{'\\', 'u', '0', '0', '0', 'f', 0, 6}}, // 0x0F
|
||||
{{'\\', 'u', '0', '0', '1', '0', 0, 6}}, // 0x10
|
||||
{{'\\', 'u', '0', '0', '1', '1', 0, 6}}, // 0x11
|
||||
{{'\\', 'u', '0', '0', '1', '2', 0, 6}}, // 0x12
|
||||
{{'\\', 'u', '0', '0', '1', '3', 0, 6}}, // 0x13
|
||||
{{'\\', 'u', '0', '0', '1', '4', 0, 6}}, // 0x14
|
||||
{{'\\', 'u', '0', '0', '1', '5', 0, 6}}, // 0x15
|
||||
{{'\\', 'u', '0', '0', '1', '6', 0, 6}}, // 0x16
|
||||
{{'\\', 'u', '0', '0', '1', '7', 0, 6}}, // 0x17
|
||||
{{'\\', 'u', '0', '0', '1', '8', 0, 6}}, // 0x18
|
||||
{{'\\', 'u', '0', '0', '1', '9', 0, 6}}, // 0x19
|
||||
{{'\\', 'u', '0', '0', '1', 'a', 0, 6}}, // 0x1A
|
||||
{{'\\', 'u', '0', '0', '1', 'b', 0, 6}}, // 0x1B
|
||||
{{'\\', 'u', '0', '0', '1', 'c', 0, 6}}, // 0x1C
|
||||
{{'\\', 'u', '0', '0', '1', 'd', 0, 6}}, // 0x1D
|
||||
{{'\\', 'u', '0', '0', '1', 'e', 0, 6}}, // 0x1E
|
||||
{{'\\', 'u', '0', '0', '1', 'f', 0, 6}}, // 0x1F
|
||||
{{' ', 0, 0, 0, 0, 0, 0, 1}}, // 0x20
|
||||
{{'!', 0, 0, 0, 0, 0, 0, 1}}, // 0x21
|
||||
{{'\\', '"', 0, 0, 0, 0, 0, 2}}, // 0x22
|
||||
{{'#', 0, 0, 0, 0, 0, 0, 1}}, // 0x23
|
||||
{{'$', 0, 0, 0, 0, 0, 0, 1}}, // 0x24
|
||||
{{'%', 0, 0, 0, 0, 0, 0, 1}}, // 0x25
|
||||
{{'&', 0, 0, 0, 0, 0, 0, 1}}, // 0x26
|
||||
{{'\'', 0, 0, 0, 0, 0, 0, 1}}, // 0x27
|
||||
{{'(', 0, 0, 0, 0, 0, 0, 1}}, // 0x28
|
||||
{{')', 0, 0, 0, 0, 0, 0, 1}}, // 0x29
|
||||
{{'*', 0, 0, 0, 0, 0, 0, 1}}, // 0x2A
|
||||
{{'+', 0, 0, 0, 0, 0, 0, 1}}, // 0x2B
|
||||
{{',', 0, 0, 0, 0, 0, 0, 1}}, // 0x2C
|
||||
{{'-', 0, 0, 0, 0, 0, 0, 1}}, // 0x2D
|
||||
{{'.', 0, 0, 0, 0, 0, 0, 1}}, // 0x2E
|
||||
{{'/', 0, 0, 0, 0, 0, 0, 1}}, // 0x2F
|
||||
{{'0', 0, 0, 0, 0, 0, 0, 1}}, // 0x30
|
||||
{{'1', 0, 0, 0, 0, 0, 0, 1}}, // 0x31
|
||||
{{'2', 0, 0, 0, 0, 0, 0, 1}}, // 0x32
|
||||
{{'3', 0, 0, 0, 0, 0, 0, 1}}, // 0x33
|
||||
{{'4', 0, 0, 0, 0, 0, 0, 1}}, // 0x34
|
||||
{{'5', 0, 0, 0, 0, 0, 0, 1}}, // 0x35
|
||||
{{'6', 0, 0, 0, 0, 0, 0, 1}}, // 0x36
|
||||
{{'7', 0, 0, 0, 0, 0, 0, 1}}, // 0x37
|
||||
{{'8', 0, 0, 0, 0, 0, 0, 1}}, // 0x38
|
||||
{{'9', 0, 0, 0, 0, 0, 0, 1}}, // 0x39
|
||||
{{':', 0, 0, 0, 0, 0, 0, 1}}, // 0x3A
|
||||
{{';', 0, 0, 0, 0, 0, 0, 1}}, // 0x3B
|
||||
{{'<', 0, 0, 0, 0, 0, 0, 1}}, // 0x3C
|
||||
{{'=', 0, 0, 0, 0, 0, 0, 1}}, // 0x3D
|
||||
{{'>', 0, 0, 0, 0, 0, 0, 1}}, // 0x3E
|
||||
{{'?', 0, 0, 0, 0, 0, 0, 1}}, // 0x3F
|
||||
{{'@', 0, 0, 0, 0, 0, 0, 1}}, // 0x40
|
||||
{{'A', 0, 0, 0, 0, 0, 0, 1}}, // 0x41
|
||||
{{'B', 0, 0, 0, 0, 0, 0, 1}}, // 0x42
|
||||
{{'C', 0, 0, 0, 0, 0, 0, 1}}, // 0x43
|
||||
{{'D', 0, 0, 0, 0, 0, 0, 1}}, // 0x44
|
||||
{{'E', 0, 0, 0, 0, 0, 0, 1}}, // 0x45
|
||||
{{'F', 0, 0, 0, 0, 0, 0, 1}}, // 0x46
|
||||
{{'G', 0, 0, 0, 0, 0, 0, 1}}, // 0x47
|
||||
{{'H', 0, 0, 0, 0, 0, 0, 1}}, // 0x48
|
||||
{{'I', 0, 0, 0, 0, 0, 0, 1}}, // 0x49
|
||||
{{'J', 0, 0, 0, 0, 0, 0, 1}}, // 0x4A
|
||||
{{'K', 0, 0, 0, 0, 0, 0, 1}}, // 0x4B
|
||||
{{'L', 0, 0, 0, 0, 0, 0, 1}}, // 0x4C
|
||||
{{'M', 0, 0, 0, 0, 0, 0, 1}}, // 0x4D
|
||||
{{'N', 0, 0, 0, 0, 0, 0, 1}}, // 0x4E
|
||||
{{'O', 0, 0, 0, 0, 0, 0, 1}}, // 0x4F
|
||||
{{'P', 0, 0, 0, 0, 0, 0, 1}}, // 0x50
|
||||
{{'Q', 0, 0, 0, 0, 0, 0, 1}}, // 0x51
|
||||
{{'R', 0, 0, 0, 0, 0, 0, 1}}, // 0x52
|
||||
{{'S', 0, 0, 0, 0, 0, 0, 1}}, // 0x53
|
||||
{{'T', 0, 0, 0, 0, 0, 0, 1}}, // 0x54
|
||||
{{'U', 0, 0, 0, 0, 0, 0, 1}}, // 0x55
|
||||
{{'V', 0, 0, 0, 0, 0, 0, 1}}, // 0x56
|
||||
{{'W', 0, 0, 0, 0, 0, 0, 1}}, // 0x57
|
||||
{{'X', 0, 0, 0, 0, 0, 0, 1}}, // 0x58
|
||||
{{'Y', 0, 0, 0, 0, 0, 0, 1}}, // 0x59
|
||||
{{'Z', 0, 0, 0, 0, 0, 0, 1}}, // 0x5A
|
||||
{{'[', 0, 0, 0, 0, 0, 0, 1}}, // 0x5B
|
||||
{{'\\', '\\', 0, 0, 0, 0, 0, 2}}, // 0x5C
|
||||
{{']', 0, 0, 0, 0, 0, 0, 1}}, // 0x5D
|
||||
{{'^', 0, 0, 0, 0, 0, 0, 1}}, // 0x5E
|
||||
{{'_', 0, 0, 0, 0, 0, 0, 1}}, // 0x5F
|
||||
}
|
||||
};
|
||||
return escapes;
|
||||
}
|
||||
|
||||
/*!
|
||||
* @brief write a lowercase "\\uXXXX" escape sequence into @a string_buffer
|
||||
*
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
// __ _____ _____ _____
|
||||
// __| | __| | | | JSON for Modern C++ (supporting code)
|
||||
// | | |__ | | | | | | version 3.12.0
|
||||
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
|
||||
//
|
||||
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// The fast paths for contiguous input (bulk string scanning, SWAR searches,
|
||||
// UTF-8 prefix validation, the 8-digit number loop, the 8-byte indentation
|
||||
// skip, the BSON/BON8 bulk string readers) load several bytes at a time. This
|
||||
// test places every input directly in front of (or directly behind) an
|
||||
// inaccessible page, so reading even one byte outside the input crashes, with
|
||||
// or without sanitizers. ASan alone misses over-reads into the slack of the
|
||||
// same heap allocation.
|
||||
|
||||
#include "doctest_compatibility.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
using nlohmann::json;
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define JSON_TEST_GUARD_PAGE_WINDOWS
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif defined(__unix__) || defined(__APPLE__)
|
||||
#define JSON_TEST_GUARD_PAGE_POSIX
|
||||
#include <sys/mman.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS) || defined(JSON_TEST_GUARD_PAGE_POSIX)
|
||||
|
||||
namespace
|
||||
{
|
||||
// Three pages: an inaccessible one, a readable one, an inaccessible one. An
|
||||
// input is copied either to the end of the middle page (any read past its last
|
||||
// byte faults) or to its start (any read before its first byte faults).
|
||||
class guarded_buffer
|
||||
{
|
||||
public:
|
||||
guarded_buffer()
|
||||
: page(page_size())
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
void* const p = VirtualAlloc(nullptr, 3 * page, MEM_RESERVE | MEM_COMMIT, PAGE_NOACCESS);
|
||||
REQUIRE(p != nullptr);
|
||||
base = static_cast<unsigned char*>(p);
|
||||
DWORD old_protect = 0;
|
||||
REQUIRE(VirtualProtect(base + page, page, PAGE_READWRITE, &old_protect) != 0);
|
||||
#else
|
||||
void* const p = mmap(nullptr, 3 * page, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
|
||||
REQUIRE(p != MAP_FAILED);
|
||||
base = static_cast<unsigned char*>(p);
|
||||
REQUIRE(mprotect(base + page, page, PROT_READ | PROT_WRITE) == 0);
|
||||
#endif
|
||||
}
|
||||
|
||||
~guarded_buffer()
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
VirtualFree(base, 0, MEM_RELEASE);
|
||||
#else
|
||||
munmap(base, 3 * page);
|
||||
#endif
|
||||
}
|
||||
|
||||
guarded_buffer(const guarded_buffer&) = delete;
|
||||
guarded_buffer& operator=(const guarded_buffer&) = delete;
|
||||
guarded_buffer(guarded_buffer&&) = delete;
|
||||
guarded_buffer& operator=(guarded_buffer&&) = delete;
|
||||
|
||||
// copies @a input so its last byte is the last readable byte
|
||||
const std::uint8_t* at_end(const std::string& input)
|
||||
{
|
||||
REQUIRE(input.size() <= page);
|
||||
unsigned char* const first = base + (2 * page) - input.size();
|
||||
std::copy(input.begin(), input.end(), first);
|
||||
return first;
|
||||
}
|
||||
|
||||
// copies @a input so its first byte is the first readable byte
|
||||
const std::uint8_t* at_start(const std::string& input)
|
||||
{
|
||||
REQUIRE(input.size() <= page);
|
||||
unsigned char* const first = base + page;
|
||||
std::copy(input.begin(), input.end(), first);
|
||||
return first;
|
||||
}
|
||||
|
||||
private:
|
||||
static std::size_t page_size()
|
||||
{
|
||||
#if defined(JSON_TEST_GUARD_PAGE_WINDOWS)
|
||||
SYSTEM_INFO info;
|
||||
GetSystemInfo(&info);
|
||||
return static_cast<std::size_t>(info.dwPageSize);
|
||||
#else
|
||||
return static_cast<std::size_t>(sysconf(_SC_PAGESIZE));
|
||||
#endif
|
||||
}
|
||||
|
||||
std::size_t page = 0;
|
||||
unsigned char* base = nullptr;
|
||||
};
|
||||
|
||||
// the serialization of the result of @a f, or the exception message
|
||||
template<typename F>
|
||||
std::string outcome(F f)
|
||||
{
|
||||
try
|
||||
{
|
||||
return "value: " + f().dump(-1, ' ', false, json::error_handler_t::keep);
|
||||
}
|
||||
catch (const json::exception& e)
|
||||
{
|
||||
return std::string("error: ") + e.what();
|
||||
}
|
||||
}
|
||||
|
||||
// a SAX handler that only counts events, so sax_parse runs without a DOM
|
||||
struct counting_sax : nlohmann::json_sax<json>
|
||||
{
|
||||
std::size_t events = 0;
|
||||
std::string error{}; // NOLINT(readability-redundant-member-init)
|
||||
|
||||
bool null() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool boolean(bool /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_integer(number_integer_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_unsigned(number_unsigned_t /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool string(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool binary(binary_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_object(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool key(string_t& /*val*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_object() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool start_array(std::size_t /*elements*/) override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool end_array() override
|
||||
{
|
||||
return count();
|
||||
}
|
||||
bool parse_error(std::size_t position, const std::string& /*last_token*/, const nlohmann::detail::exception& ex) override
|
||||
{
|
||||
error = std::to_string(position) + " " + ex.what();
|
||||
return false;
|
||||
}
|
||||
|
||||
private:
|
||||
bool count()
|
||||
{
|
||||
++events;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
// all prefixes of each seed, so the input ends at every byte of it
|
||||
std::vector<std::string> all_prefixes(const std::vector<std::string>& seeds)
|
||||
{
|
||||
std::vector<std::string> result;
|
||||
for (const auto& seed : seeds)
|
||||
{
|
||||
for (std::size_t n = 0; n <= seed.size(); ++n)
|
||||
{
|
||||
result.push_back(seed.substr(0, n));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<std::string> json_seeds()
|
||||
{
|
||||
std::vector<std::string> seeds =
|
||||
{
|
||||
// string runs: ASCII, escapes, \u escapes, surrogate pairs
|
||||
"\"abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ\"",
|
||||
R"("ab\ncd\"ef\\gh\u00e4ij\uD83D\uDE00kl\u12")",
|
||||
// 2-, 3- and 4-byte UTF-8 sequences, then invalid UTF-8
|
||||
std::string("\"\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80") + "abcdefgh\"",
|
||||
"\"abcdefgh\xC3\x28 \xED\xA0\x80 \xF4\x90\x80\x80 \xF8 \xC0\xAF\"",
|
||||
std::string("\"abcdefg\x01") + "abcdefgh\"",
|
||||
// numbers, so the 8-digit loop runs up to the end
|
||||
"12345678",
|
||||
"1234567890123456",
|
||||
"12345678901234567890123",
|
||||
"-0.000000001234567890123456789",
|
||||
"3.14159265358979323846264338327950288e+100",
|
||||
"-1.2345678901234567e-12",
|
||||
"1E-5",
|
||||
"[1.5,2e10,-3E-5,0.000000001,123456789012345678]",
|
||||
// literals
|
||||
"[true,false,null]",
|
||||
// comments (only parsed with ignore_comments)
|
||||
"/* comment */ [1, // line comment\n 2] /**/",
|
||||
// a pretty-printed document
|
||||
json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(4),
|
||||
json::parse(R"({"a": [1, 2.5, "three", {"b": null, "c": [true, false]}], "d": {"e": {"f": {"g": "h"}}}})").dump(8, '\t'),
|
||||
};
|
||||
|
||||
// indentation runs of every length, with and without a closing bracket
|
||||
for (std::size_t n = 0; n <= 20; ++n)
|
||||
{
|
||||
seeds.push_back("[\n" + std::string(n, ' ') + "1,\n" + std::string(n, ' ') + "2\n]");
|
||||
}
|
||||
|
||||
return all_prefixes(seeds);
|
||||
}
|
||||
|
||||
// a value whose binary serializations contain long strings and keys, numbers
|
||||
// of every width, and nested containers
|
||||
json binary_document()
|
||||
{
|
||||
json j;
|
||||
j["a key that is longer than eight bytes"] = "a string value that is longer than eight bytes";
|
||||
j["\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80"] = "\xF0\x9F\x98\x80\xE2\x82\xAC\xC3\xA4 utf-8 0123456789";
|
||||
j["numbers"] = {0, 1, -1, 255, -256, 65536, -2147483649LL, 4294967296ULL, 1.5, -2.25e100};
|
||||
j["nested"] = {{"x", {{"y", {true, false, nullptr}}}}};
|
||||
j["s"] = "";
|
||||
return j;
|
||||
}
|
||||
|
||||
// readers for check_binary(), callable with a container or an iterator range
|
||||
#define JSON_TEST_GUARD_PAGE_READER(name, from) \
|
||||
struct name \
|
||||
{ \
|
||||
template<typename... Args> \
|
||||
json operator()(Args&& ... args) const \
|
||||
{ \
|
||||
return json::from(std::forward<Args>(args)...); \
|
||||
} \
|
||||
};
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bson, from_bson)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bon8, from_bon8)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_cbor, from_cbor)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_msgpack, from_msgpack)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_ubjson, from_ubjson)
|
||||
JSON_TEST_GUARD_PAGE_READER(read_bjdata, from_bjdata)
|
||||
#undef JSON_TEST_GUARD_PAGE_READER
|
||||
|
||||
// reads every prefix of @a bytes at both ends of the guarded page and from a
|
||||
// std::string, which must give the same outcome
|
||||
template<typename Reader>
|
||||
void check_binary(guarded_buffer& guard, const std::vector<std::uint8_t>& bytes, Reader parse)
|
||||
{
|
||||
for (const auto& input : all_prefixes({std::string(bytes.begin(), bytes.end())}))
|
||||
{
|
||||
CAPTURE(input)
|
||||
const std::string expected = outcome([&] { return parse(input); });
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
CHECK(outcome([&] { return parse(first, first + input.size()); }) == expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("guard page")
|
||||
{
|
||||
guarded_buffer guard;
|
||||
|
||||
SECTION("string scanning")
|
||||
{
|
||||
// fills that drive each function to a different stop
|
||||
const std::vector<std::string> fills =
|
||||
{
|
||||
std::string(64, 'a'),
|
||||
std::string(64, '"'),
|
||||
std::string(64, '\\'),
|
||||
std::string(64, '\x01'),
|
||||
std::string(64, '\x7F'),
|
||||
std::string(64, '\xFF'),
|
||||
};
|
||||
std::vector<std::string> patterns = fills;
|
||||
{
|
||||
std::string utf8;
|
||||
while (utf8.size() < 64)
|
||||
{
|
||||
utf8 += "\xC3\xA4\xE2\x82\xAC\xF0\x9F\x98\x80" "ab";
|
||||
}
|
||||
patterns.push_back(utf8);
|
||||
}
|
||||
{
|
||||
std::string bytes;
|
||||
for (unsigned int i = 0; i < 64; ++i)
|
||||
{
|
||||
bytes.push_back(static_cast<char>((i * 37u) & 0xFFu));
|
||||
}
|
||||
patterns.push_back(bytes);
|
||||
}
|
||||
|
||||
for (const auto& pattern : patterns)
|
||||
{
|
||||
for (std::size_t n = 0; n <= 64; ++n)
|
||||
{
|
||||
const std::string input = pattern.substr(0, n);
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const unsigned char* const p = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
|
||||
CHECK(nlohmann::detail::find_string_special(p, n) <= n);
|
||||
CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
|
||||
CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
|
||||
#if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
|
||||
CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
|
||||
#endif
|
||||
CHECK(nlohmann::detail::string_bulk_run(p, n) <= n);
|
||||
CHECK(nlohmann::detail::scalar_string_bulk_run(p, n) <= n);
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
{
|
||||
CHECK(nlohmann::detail::validate_one_utf8(p + i, n - i) <= n - i);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("Eisel-Lemire")
|
||||
{
|
||||
const std::vector<std::string> numbers = all_prefixes(
|
||||
{
|
||||
"1234567812345678123456781234567812345678",
|
||||
"-1.2345678901234567890123456789e-123",
|
||||
"0.00000000000000000000000000000000000001",
|
||||
"98765432.10987654",
|
||||
});
|
||||
for (const auto& number : numbers)
|
||||
{
|
||||
double out = 0;
|
||||
const char* const end = reinterpret_cast<const char*>(guard.at_end(number));
|
||||
static_cast<void>(nlohmann::detail::parse_float_eisel_lemire(end, end + number.size(), out));
|
||||
const char* const start = reinterpret_cast<const char*>(guard.at_start(number));
|
||||
static_cast<void>(nlohmann::detail::parse_float_eisel_lemire(start, start + number.size(), out));
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("JSON")
|
||||
{
|
||||
for (const auto& input : json_seeds())
|
||||
{
|
||||
CAPTURE(input)
|
||||
const std::string expected = outcome([&] { return json::parse(input); });
|
||||
const std::string expected_comments = outcome([&] { return json::parse(input, nullptr, true, true); });
|
||||
const bool expected_accept = json::accept(input);
|
||||
counting_sax expected_sax;
|
||||
const bool expected_sax_result = json::sax_parse(input, &expected_sax);
|
||||
|
||||
for (int placement = 0; placement < 2; ++placement)
|
||||
{
|
||||
const std::uint8_t* const first = placement == 0 ? guard.at_end(input) : guard.at_start(input);
|
||||
const std::uint8_t* const last = first + input.size();
|
||||
|
||||
CHECK(outcome([&] { return json::parse(first, last); }) == expected);
|
||||
CHECK(outcome([&] { return json::parse(first, last, nullptr, true, true); }) == expected_comments);
|
||||
CHECK(json::accept(first, last) == expected_accept);
|
||||
|
||||
counting_sax sax;
|
||||
CHECK(json::sax_parse(first, last, &sax) == expected_sax_result);
|
||||
CHECK(sax.events == expected_sax.events);
|
||||
CHECK(sax.error == expected_sax.error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("binary formats")
|
||||
{
|
||||
// every prefix ends inside a value, e.g., inside a BSON key or BON8
|
||||
// string without its terminator
|
||||
const json j = binary_document();
|
||||
check_binary(guard, json::to_bson(j), read_bson());
|
||||
check_binary(guard, json::to_bon8(j), read_bon8());
|
||||
check_binary(guard, json::to_cbor(j), read_cbor());
|
||||
check_binary(guard, json::to_msgpack(j), read_msgpack());
|
||||
check_binary(guard, json::to_ubjson(j), read_ubjson());
|
||||
check_binary(guard, json::to_bjdata(j), read_bjdata());
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -923,68 +923,66 @@ TEST_CASE("serialization boundary values for the string buffer")
|
||||
// mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes
|
||||
// < 13`), 13 being one more than the most a single code point can ever
|
||||
// write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12
|
||||
// bytes). Characters below 0x60 (control characters, '"' and '\\') are
|
||||
// escaped straight into the write buffer and never reach string_buffer, so
|
||||
// under ensure_ascii it fills in steps of 6 (one "\uXXXX" unit for a
|
||||
// character below U+10000) or 12 (a surrogate pair). With 2-byte characters
|
||||
// such as U+00E4, `bytes` takes multiples of 6 at the point the check runs;
|
||||
// the tightest values reachable are therefore 498 (512 - 498 == 14, after
|
||||
// 83 characters) and 504 (512 - 504 == 8, after 84 characters, where the
|
||||
// flush fires immediately and resets bytes to 0).
|
||||
// bytes). Every write into string_buffer that this check protects happens
|
||||
// in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so
|
||||
// `bytes` only ever takes even values at the point the check runs - the
|
||||
// tightest values actually reachable are therefore 498 (512 - 498 == 14,
|
||||
// one simple escape away from the threshold) and 500 (512 - 500 == 12,
|
||||
// where the flush fires immediately and resets bytes to 0).
|
||||
|
||||
SECTION("a run of 6-byte escapes lands bytes on, and one step past, the flush threshold")
|
||||
SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold")
|
||||
{
|
||||
for (const int count :
|
||||
{
|
||||
83, 84, 85
|
||||
249, 250, 251
|
||||
})
|
||||
{
|
||||
CAPTURE(count)
|
||||
std::string umlauts;
|
||||
const json j = std::string(static_cast<std::size_t>(count), '\n');
|
||||
std::string expected = "\"";
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
umlauts += "\xC3\xA4";
|
||||
expected += "\\u00e4";
|
||||
expected += "\\n";
|
||||
}
|
||||
expected += '"';
|
||||
CHECK(json(umlauts).dump(-1, ' ', true) == expected);
|
||||
CHECK(j.dump() == expected);
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("a prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
|
||||
SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
|
||||
{
|
||||
// U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under
|
||||
// ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that
|
||||
// write happens in a single step with no intermediate flush check, so
|
||||
// it is the write most exposed by an off-by-one in the "< 13" guard.
|
||||
// A prefix of 83 U+00E4 characters leaves exactly 14 bytes of headroom
|
||||
// A prefix of 249 newlines leaves exactly 14 bytes of headroom
|
||||
// (512 - 498), the smallest margin the guard ever actually allows
|
||||
// into a new code point; 84 characters instead trigger the guard's own
|
||||
// into a new code point; 250 newlines instead trigger the guard's own
|
||||
// flush first, so the emoji starts from a freshly emptied (512-byte)
|
||||
// buffer, and 85 repeats that with one more escape already past the
|
||||
// buffer, and 251 repeats that with one more escape already past the
|
||||
// reset. Together they cover the margin the guard allows landing on,
|
||||
// one step before, and one step after - all must still produce the
|
||||
// identical, correct escapes.
|
||||
for (const int prefix_count :
|
||||
{
|
||||
83, 84, 85
|
||||
249, 250, 251
|
||||
})
|
||||
{
|
||||
CAPTURE(prefix_count)
|
||||
std::string prefix;
|
||||
std::string expected_prefix;
|
||||
for (int i = 0; i < prefix_count; ++i)
|
||||
{
|
||||
prefix += "\xC3\xA4";
|
||||
expected_prefix += "\\u00e4";
|
||||
}
|
||||
const std::string prefix(static_cast<std::size_t>(prefix_count), '\n');
|
||||
const std::string emoji = "\xF0\x9F\x98\x80";
|
||||
const json j = prefix + emoji;
|
||||
|
||||
// without ensure_ascii, the whole string is copied unchanged
|
||||
std::string expected_prefix;
|
||||
for (int i = 0; i < prefix_count; ++i)
|
||||
{
|
||||
expected_prefix += "\\n";
|
||||
}
|
||||
|
||||
// newline escaping does not depend on ensure_ascii: only the
|
||||
// emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair)
|
||||
std::string expected_raw = "\"";
|
||||
expected_raw += prefix;
|
||||
expected_raw += expected_prefix;
|
||||
expected_raw += emoji;
|
||||
expected_raw += '"';
|
||||
std::string expected_ascii = "\"";
|
||||
@@ -1029,132 +1027,3 @@ TEST_CASE("serialization boundary values for the string buffer")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
// reference escaper for ASCII strings, written independently of the table in
|
||||
// serializer::ascii_escapes()
|
||||
std::string reference_escape_ascii(const std::string& s, const bool ensure_ascii)
|
||||
{
|
||||
std::string result = "\"";
|
||||
for (const char c : s)
|
||||
{
|
||||
const auto byte = static_cast<unsigned char>(c);
|
||||
switch (byte)
|
||||
{
|
||||
case '\b':
|
||||
result += "\\b";
|
||||
break;
|
||||
case '\t':
|
||||
result += "\\t";
|
||||
break;
|
||||
case '\n':
|
||||
result += "\\n";
|
||||
break;
|
||||
case '\f':
|
||||
result += "\\f";
|
||||
break;
|
||||
case '\r':
|
||||
result += "\\r";
|
||||
break;
|
||||
case '"':
|
||||
result += "\\\"";
|
||||
break;
|
||||
case '\\':
|
||||
result += "\\\\";
|
||||
break;
|
||||
default:
|
||||
if (byte < 0x20 || (ensure_ascii && byte == 0x7F))
|
||||
{
|
||||
// byte < 0x80, so the first two hex digits are 0
|
||||
const char* const hex = "0123456789abcdef";
|
||||
result += "\\u00";
|
||||
result += hex[byte >> 4U];
|
||||
result += hex[byte & 0x0FU];
|
||||
}
|
||||
else
|
||||
{
|
||||
result += c;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
result += '"';
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("characters below 0x60 are escaped from a table straight into the write buffer")
|
||||
{
|
||||
SECTION("every ASCII character at every position")
|
||||
{
|
||||
// lengths up to 17 cover the 8-byte strides of the bulk scanner on
|
||||
// both sides of the character, so the scanner stops at it from every
|
||||
// offset
|
||||
for (std::size_t length = 1; length <= 17; ++length)
|
||||
{
|
||||
for (std::size_t pos = 0; pos < length; ++pos)
|
||||
{
|
||||
for (unsigned int c = 0; c < 0x80; ++c)
|
||||
{
|
||||
std::string s(length, 'a');
|
||||
s[pos] = static_cast<char>(c);
|
||||
CAPTURE(length)
|
||||
CAPTURE(pos)
|
||||
CAPTURE(c)
|
||||
const json j = s;
|
||||
CHECK(j.dump(-1, ' ', false) == reference_escape_ascii(s, false));
|
||||
CHECK(j.dump(-1, ' ', true) == reference_escape_ascii(s, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("runs of escaped characters")
|
||||
{
|
||||
std::string s;
|
||||
for (unsigned int c = 0; c < 0x20; ++c)
|
||||
{
|
||||
s += static_cast<char>(c);
|
||||
}
|
||||
s += R"("\"\)";
|
||||
CHECK(json(s).dump() == reference_escape_ascii(s, false));
|
||||
CHECK(json(s).dump(-1, ' ', true) == reference_escape_ascii(s, true));
|
||||
}
|
||||
|
||||
SECTION("escapes straddle the end of the write buffer")
|
||||
{
|
||||
// the 8-byte store of a table entry needs room for all 8 bytes, even
|
||||
// when the escape itself is shorter; a prefix of k plain characters
|
||||
// shifts the 6-byte escapes across every offset of the 1024-byte
|
||||
// write buffer
|
||||
for (std::size_t k = 0; k < 8; ++k)
|
||||
{
|
||||
CAPTURE(k)
|
||||
const std::string s = std::string(k, 'a') + std::string(400, '\x01');
|
||||
CHECK(json(s).dump() == reference_escape_ascii(s, false));
|
||||
CHECK(json::array({s, s}).dump() == "[" + reference_escape_ascii(s, false) + "," + reference_escape_ascii(s, false) + "]");
|
||||
}
|
||||
}
|
||||
|
||||
SECTION("pending \\u escapes of non-ASCII characters are written first")
|
||||
{
|
||||
// under ensure_ascii, U+00E4 is \u-escaped into the string buffer;
|
||||
// a following escape from the table must not overtake it
|
||||
const json j = "\xC3\xA4\n\xC3\xA4\"\xF0\x9F\x98\x80\\";
|
||||
CHECK(j.dump(-1, ' ', true) == R"("\u00e4\n\u00e4\"\ud83d\ude00\\")");
|
||||
CHECK(j.dump(-1, ' ', false) == "\"\xC3\xA4\\n\xC3\xA4\\\"\xF0\x9F\x98\x80\\\\\"");
|
||||
}
|
||||
|
||||
SECTION("escapes after invalid UTF-8 for every error handler")
|
||||
{
|
||||
const json j = "a\xFF\n\xC3\"";
|
||||
CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xFF", json::type_error&);
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == R"("a\n\"")");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::ignore) == R"("a\n\"")");
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"a\xEF\xBF\xBD\\n\xEF\xBF\xBD\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == R"("a\ufffd\n\ufffd\"")");
|
||||
CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
|
||||
CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"a\xFF\\n\xC3\\\"\"");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user