amorfo77/json
1
0
Fork 0
mirror of https://github.com/nlohmann/json.git synced 2026-10-05 22:20:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add BON8 support

Browse Source 
Add to_bon8/from_bon8 and input_format_t::bon8 for BON8, a binary format
that uses the byte values that cannot begin a UTF-8 character as type
markers, so strings need no length prefix. It is the most compact of the
supported binary formats on the benchmark files.

The reader is non-recursive like the other binary readers. A string ends
at the first byte that cannot continue it, so the reader hands the one or
two bytes it reads past a string back to the value that follows. The
writer produces the canonical representation of the specification, except
for NFC normalization; its output is identical to that of the reference
implementation (HikoGUI) on all files of the test data.

The round-trip tests need the .bon8 files of json_test_data 3.2.0.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
This commit is contained in:
Niels Lohmann
2026-09-24 22:53:08 +02:00
parent 4daca40d7b
commit e94e164b07
45 changed files with 3574 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/nlohmann/detail/output/binary_writer.hpp
+397 -1
View File
@@ -76,7 +76,7 @@ std::size_t binary_reserve_hint(const BasicJsonType& j)
}
/*!
@brief serialization to CBOR and MessagePack values
@brief serialization to BJData, BON8, BSON, CBOR, MessagePack, and UBJSON values
*/
template<typename BasicJsonType, typename CharType, typename OutputSinkType = output_adapter_sink<CharType>>
class binary_writer
@@ -1032,6 +1032,21 @@ class binary_writer
}
}
/*!
@param[in] j JSON value to serialize
*/
void write_bon8(const BasicJsonType& j)
{
bool string_open = false;
write_bon8_value(j, string_open);
// the last string of a message must be terminated
if (string_open)
{
oa.write_character(to_char_type(0xFF));
}
}
private:
//////////
// BSON //
@@ -1431,6 +1446,16 @@ class binary_writer
return to_char_type(0xCB); // float 64
}
static constexpr CharType get_bon8_float_prefix(float /*unused*/)
{
return to_char_type(0x8E); // binary32
}
static constexpr CharType get_bon8_float_prefix(double /*unused*/)
{
return to_char_type(0x8F); // binary64
}
////////////
// UBJSON //
////////////
@@ -2023,6 +2048,373 @@ class binary_writer
return false;
}
//////////
// BON8 //
//////////
/*!
@brief write a BON8 value
A string is written without length or terminator: it ends at the first
byte that cannot continue it, which is the first byte of any non-string
value and of the end-of-container marker 0xFE. It only needs an explicit
end-of-string marker (0xFF) when it is empty, when another string follows,
or when it is the last thing in the message.
@param[in] j JSON value to serialize
@param[in,out] string_open whether the output ends with a non-empty
string that has not been terminated with 0xFF
*/
void write_bon8_value(const BasicJsonType& j, bool& string_open)
{
switch (j.type())
{
case value_t::null:
{
write_bon8_marker(0xFA, string_open);
break;
}
case value_t::boolean:
{
write_bon8_marker(j.m_data.m_value.boolean ? 0xF9 : 0xF8, string_open);
break;
}
case value_t::number_unsigned:
{
if (j.m_data.m_value.number_unsigned > static_cast<typename BasicJsonType::number_unsigned_t>((std::numeric_limits<std::int64_t>::max)()))
{
JSON_THROW(out_of_range::create(407, concat("integer number ", std::to_string(j.m_data.m_value.number_unsigned), " cannot be represented by BON8 as it does not fit int64"), &j));
}
write_bon8_integer(static_cast<std::int64_t>(j.m_data.m_value.number_unsigned));
string_open = false;
break;
}
case value_t::number_integer:
{
write_bon8_integer(static_cast<std::int64_t>(j.m_data.m_value.number_integer));
string_open = false;
break;
}
case value_t::number_float:
{
write_bon8_float(j.m_data.m_value.number_float);
string_open = false;
break;
}
case value_t::string:
{
write_bon8_string(*j.m_data.m_value.string, string_open, j);
break;
}
case value_t::array:
{
const auto N = j.m_data.m_value.array->size();
// 0x80..0x84: array with 0..4 elements; 0x85: array ended by 0xFE
write_bon8_marker(static_cast<std::uint8_t>(N <= 4 ? 0x80 + N : 0x85), string_open);
for (const auto& el : *j.m_data.m_value.array)
{
write_bon8_value(el, string_open);
}
if (N > 4)
{
write_bon8_marker(0xFE, string_open);
}
break;
}
case value_t::object:
{
const auto N = j.m_data.m_value.object->size();
// 0x86..0x8A: object with 0..4 members; 0x8B: object ended by 0xFE
write_bon8_marker(static_cast<std::uint8_t>(N <= 4 ? 0x86 + N : 0x8B), string_open);
for (const auto& el : *j.m_data.m_value.object)
{
write_bon8_string(el.first, string_open, j);
write_bon8_value(el.second, string_open);
}
if (N > 4)
{
write_bon8_marker(0xFE, string_open);
}
break;
}
case value_t::binary:
{
// BON8 has no binary type: write the bytes as an array of
// integers, like UBJSON and BJData do
const auto N = j.m_data.m_value.binary->size();
write_bon8_marker(static_cast<std::uint8_t>(N <= 4 ? 0x80 + N : 0x85), string_open);
for (std::size_t i = 0; i < N; ++i)
{
// the cast is needed for binary types whose value type
// is not an integer (e.g., std::byte)
write_bon8_integer(static_cast<std::uint8_t>(j.m_data.m_value.binary->data()[i]));
}
if (N > 4)
{
oa.write_character(to_char_type(0xFE));
}
break;
}
case value_t::discarded:
default:
break;
}
}
/*!
@brief write a single byte that is not part of a string
@param[in] marker the byte to write
@param[in,out] string_open see @ref write_bon8_value
*/
void write_bon8_marker(const std::uint8_t marker, bool& string_open)
{
oa.write_character(to_char_type(marker));
string_open = false;
}
/*!
@brief write a string
@param[in] s the string to write
@param[in,out] string_open see @ref write_bon8_value
@param[in] context the value the string belongs to (for diagnostics)
@throw type_error.316 if @a s is not valid UTF-8, because the end of a
string is determined from its encoding
*/
void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context)
{
check_bon8_utf8(s, context);
// a string that follows another string terminates it
if (string_open)
{
oa.write_character(to_char_type(0xFF));
}
if (s.empty())
{
// the empty string is just the end-of-string marker
oa.write_character(to_char_type(0xFF));
string_open = false;
}
else
{
oa.write_characters(reinterpret_cast<const CharType*>(s.data()), s.size());
string_open = true;
}
}
/*!
@brief check that a string is valid UTF-8 (RFC 3629)
@param[in] s the string to check
@param[in] context the value the string belongs to (for diagnostics)
@throw type_error.316 if @a s is not valid UTF-8
*/
static void check_bon8_utf8(const string_t& s, const BasicJsonType& context)
{
const auto byte_at = [&s](const std::size_t i)
{
return static_cast<std::uint8_t>(s[i]);
};
for (std::size_t i = 0; i < s.size();)
{
const std::uint8_t lead = byte_at(i);
std::size_t continuation_bytes = 0;
// valid range of the byte after the lead byte, which excludes
// overlong forms, surrogates, and code points above U+10FFFF
std::uint8_t lower = 0x80;
std::uint8_t upper = 0xBF;
if (lead <= 0x7F)
{
++i;
continue;
}
if (0xC2 <= lead && lead <= 0xDF)
{
continuation_bytes = 1;
}
else if (0xE0 <= lead && lead <= 0xEF)
{
continuation_bytes = 2;
lower = lead == 0xE0 ? 0xA0 : 0x80;
upper = lead == 0xED ? 0x9F : 0xBF;
}
else if (0xF0 <= lead && lead <= 0xF4)
{
continuation_bytes = 3;
lower = lead == 0xF0 ? 0x90 : 0x80;
upper = lead == 0xF4 ? 0x8F : 0xBF;
}
else
{
throw_bon8_utf8_error(i, lead, context);
}
for (std::size_t k = 1; k <= continuation_bytes; ++k)
{
if (i + k >= s.size())
{
JSON_THROW(type_error::create(316, concat("incomplete UTF-8 string; last byte: 0x", hex_byte(byte_at(s.size() - 1))), &context));
}
const std::uint8_t b = byte_at(i + k);
if (b < (k == 1 ? lower : 0x80) || b > (k == 1 ? upper : 0xBF))
{
throw_bon8_utf8_error(i + k, b, context);
}
}
i += continuation_bytes + 1;
}
}
[[noreturn]] static void throw_bon8_utf8_error(const std::size_t index, const std::uint8_t byte, const BasicJsonType& context)
{
JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(index), ": 0x", hex_byte(byte)), &context));
}
/// @return a byte as two uppercase hexadecimal digits
static std::string hex_byte(const std::uint8_t byte)
{
std::string result = "00";
constexpr const char* nibble_to_hex = "0123456789ABCDEF";
result[0] = nibble_to_hex[byte / 16];
result[1] = nibble_to_hex[byte % 16];
return result;
}
/*!
@brief write an integer in the shortest encoding
Integers from -10 to 39 take one byte. Up to -33818506 and 67637031, an
integer takes 2 to 4 bytes that begin with a UTF-8 lead byte (0xC2..0xF7)
followed by a byte that is not a continuation byte: 0x00..0x7F for
positive and 0xC0..0xFF for negative integers. Each range starts where the
shorter one ends. Larger integers are written as int32 (0x8C) or int64
(0x8D) in big-endian byte order.
@param[in] value the integer to write
*/
void write_bon8_integer(std::int64_t value)
{
if (value < (std::numeric_limits<std::int32_t>::min)() || value > (std::numeric_limits<std::int32_t>::max)())
{
oa.write_character(to_char_type(0x8D));
write_number(value);
}
else if (value < -33818506 || value > 67637031)
{
oa.write_character(to_char_type(0x8C));
write_number(static_cast<std::int32_t>(value));
}
else if (value <= -264075)
{
value = -(value + 264075);
write_bon8_bytes(0xF0 + ((value >> 22) & 0x07), 0xC0 + ((value >> 16) & 0x3F), value >> 8, value);
}
else if (value <= -1931)
{
value = -(value + 1931);
write_bon8_bytes(0xE0 + ((value >> 14) & 0x0F), 0xC0 + ((value >> 8) & 0x3F), value);
}
else if (value <= -11)
{
value = -(value + 11);
write_bon8_bytes(0xC2 + ((value >> 6) & 0x1F), 0xC0 + (value & 0x3F));
}
else if (value <= -1)
{
write_bon8_bytes(0xB8 - (value + 1));
}
else if (value <= 39)
{
write_bon8_bytes(0x90 + value);
}
else if (value <= 3879)
{
value -= 40;
write_bon8_bytes(0xC2 + ((value >> 7) & 0x1F), value & 0x7F);
}
else if (value <= 528167)
{
value -= 3880;
write_bon8_bytes(0xE0 + ((value >> 15) & 0x0F), (value >> 8) & 0x7F, value);
}
else
{
value -= 528168;
write_bon8_bytes(0xF0 + ((value >> 23) & 0x07), (value >> 16) & 0x7F, value >> 8, value);
}
}
/// write the low byte of each argument
template<typename... Bytes>
void write_bon8_bytes(const Bytes... bytes)
{
const std::array<CharType, sizeof...(Bytes)> buffer{{to_char_type(static_cast<std::uint8_t>(bytes & 0xFF))...}};
oa.write_characters(buffer.data(), buffer.size());
}
/*!
@brief write a floating-point number
-1.0, +0.0, and 1.0 take one byte. Other numbers are written as binary32
(0x8E) if that loses no precision, and as binary64 (0x8F) otherwise; -0.0,
infinities, and NaN are always written as binary32, NaN as 0x7F800001.
@param[in] n the number to write
*/
void write_bon8_float(const number_float_t n)
{
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_PUSH
JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal")
#endif
if (n == -1.0)
{
oa.write_character(to_char_type(0xFB));
}
else if (n == 0.0 && !std::signbit(n))
{
oa.write_character(to_char_type(0xFC));
}
else if (n == 1.0)
{
oa.write_character(to_char_type(0xFD));
}
else if (std::isnan(n))
{
write_bon8_bytes(0x8E, 0x7F, 0x80, 0x00, 0x01);
}
else
{
write_compact_float(n, detail::input_format_t::bon8);
}
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_POP
#endif
}
///////////////////////
// Utility functions //
///////////////////////
@@ -2159,6 +2551,8 @@ class binary_writer
{
oa.write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(static_cast<float>(n))
: format == detail::input_format_t::bon8
? get_bon8_float_prefix(static_cast<float>(n))
: get_msgpack_float_prefix(static_cast<float>(n)));
write_number(static_cast<float>(n));
}
@@ -2166,6 +2560,8 @@ class binary_writer
{
oa.write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(n)
: format == detail::input_format_t::bon8
? get_bon8_float_prefix(n)
: get_msgpack_float_prefix(n));
write_number(n);
}