diff --git a/docs/mkdocs/docs/features/binary_formats/bon8.md b/docs/mkdocs/docs/features/binary_formats/bon8.md index 6317fb527..9b5fa1dda 100644 --- a/docs/mkdocs/docs/features/binary_formats/bon8.md +++ b/docs/mkdocs/docs/features/binary_formats/bon8.md @@ -137,8 +137,8 @@ Non-negative integers are read as number_unsigned, negative integers as number_i !!! info Values that do not use the canonical representation, such as integers with a longer encoding than necessary, - arrays and objects with up to four elements that are terminated by 0xFE, unsorted object keys, or unneeded 0xFF - bytes after a string, are accepted. + arrays and objects with up to four elements that are terminated by 0xFE, unsorted object keys, or a 0xFF after a + string that would also end without it, are accepted. A second 0xFF is not a terminator but an empty string. Strings must be valid UTF-8, and the last string of a message must be terminated by 0xFF. diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 8ac969c1b..f0d34909c 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,11 @@ class binary_reader using char_type = typename InputAdapterType::char_type; using char_int_type = typename char_traits::int_type; + /// whether the input is a contiguous block of bytes that BON8 strings can + /// be copied from in bulk; see @ref get_bon8_string_bulk + static constexpr bool bon8_bulk_scan = + input_adapter_supports_bulk_scan(is_detected {}); + public: /*! @brief create a binary reader @@ -3589,6 +3595,42 @@ class binary_reader return bon8_error("expected a string; last byte", "key"); } + /*! + @brief append the run of valid UTF-8 at the read position to a string + + For contiguous input, the ASCII characters and complete well-formed UTF-8 + sequences at the read position are appended to @a result in one step. The + byte that stops the run (an end-of-string marker, the first byte of the + next value, or an ill-formed byte) is left for @ref get_bon8_string, so + that strings end and errors are reported exactly as without this step. + + @param[in,out] result the string to append to + */ + void get_bon8_string_bulk(string_t& result, std::true_type /*bulk*/) + { + // bytes handed back must be read through get_bon8() first + if (bon8_pushback_size != 0) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + const std::size_t length = valid_utf8_prefix(data, remaining); + if (length != 0) + { + result.append(reinterpret_cast(data), length); + ia.bulk_skip(length); + chars_read += length; + } + } + + /// input that is not contiguous: strings are read byte by byte + void get_bon8_string_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept {} + /*! @brief read a string @@ -3605,6 +3647,8 @@ class binary_reader { while (true) { + get_bon8_string_bulk(result, std::integral_constant {}); + const auto byte = get_bon8(); if (byte == char_traits::eof()) diff --git a/include/nlohmann/detail/input/string_scan.hpp b/include/nlohmann/detail/input/string_scan.hpp index 6af0e6c5d..8a6f03122 100644 --- a/include/nlohmann/detail/input/string_scan.hpp +++ b/include/nlohmann/detail/input/string_scan.hpp @@ -201,6 +201,43 @@ inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avai return 0; // invalid, incomplete, or must be diagnosed by the byte path } +// Return the length of the longest prefix of [data, data+n) that consists of +// ASCII characters and complete well-formed UTF-8 sequences; n if all of it is +// valid UTF-8. Unlike scalar_string_bulk_run(), quotes, escapes, and control +// characters are ordinary characters here. ASCII is skipped 8 bytes at a time. +inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t pos = 0; + while (pos < n) + { + if (pos + 8 <= n) + { + std::uint64_t word = 0; + std::memcpy(&word, data + pos, sizeof(word)); + if ((word & high) == 0) + { + pos += 8; + continue; + } + } + + if (data[pos] < 0x80u) + { + ++pos; + continue; + } + + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated + } + pos += seq; + } + return pos; +} + // Scalar (C++11) computation of the bulk run length: the number of leading // bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 // sequences, stopping before the first byte that needs individual handling (the diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index d06e0480d..af13574a8 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2178,7 +2178,7 @@ class binary_writer if (N > 4) { - oa.write_character(to_char_type(0xFE)); + write_bon8_marker(0xFE, string_open); } break; } @@ -2248,20 +2248,10 @@ class binary_writer { static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); - for (std::size_t i = 0; i < s.size();) + const std::size_t valid = valid_utf8_prefix(data, s.size()); + if (JSON_HEDLEY_UNLIKELY(valid != s.size())) { - if (data[i] < 0x80) - { - ++i; - continue; - } - - const std::size_t length = validate_one_utf8(data + i, s.size() - i); - if (JSON_HEDLEY_UNLIKELY(length == 0)) - { - JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(i), ": 0x", hex_byte(data[i])), &context)); - } - i += length; + JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(valid), ": 0x", hex_byte(data[valid])), &context)); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index b11cf1bb0..6092172cc 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8780,6 +8780,43 @@ inline std::size_t validate_one_utf8(const unsigned char* data, std::size_t avai return 0; // invalid, incomplete, or must be diagnosed by the byte path } +// Return the length of the longest prefix of [data, data+n) that consists of +// ASCII characters and complete well-formed UTF-8 sequences; n if all of it is +// valid UTF-8. Unlike scalar_string_bulk_run(), quotes, escapes, and control +// characters are ordinary characters here. ASCII is skipped 8 bytes at a time. +inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) noexcept +{ + constexpr std::uint64_t high = 0x8080808080808080ull; + std::size_t pos = 0; + while (pos < n) + { + if (pos + 8 <= n) + { + std::uint64_t word = 0; + std::memcpy(&word, data + pos, sizeof(word)); + if ((word & high) == 0) + { + pos += 8; + continue; + } + } + + if (data[pos] < 0x80u) + { + ++pos; + continue; + } + + const std::size_t seq = validate_one_utf8(data + pos, n - pos); + if (seq == 0) + { + break; // ill-formed or truncated + } + pos += seq; + } + return pos; +} + // Scalar (C++11) computation of the bulk run length: the number of leading // bytes in [data, data+n) that are ordinary ASCII or complete well-formed UTF-8 // sequences, stopping before the first byte that needs individual handling (the @@ -12342,6 +12379,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -12578,6 +12617,11 @@ class binary_reader using char_type = typename InputAdapterType::char_type; using char_int_type = typename char_traits::int_type; + /// whether the input is a contiguous block of bytes that BON8 strings can + /// be copied from in bulk; see @ref get_bon8_string_bulk + static constexpr bool bon8_bulk_scan = + input_adapter_supports_bulk_scan(is_detected {}); + public: /*! @brief create a binary reader @@ -16070,6 +16114,42 @@ class binary_reader return bon8_error("expected a string; last byte", "key"); } + /*! + @brief append the run of valid UTF-8 at the read position to a string + + For contiguous input, the ASCII characters and complete well-formed UTF-8 + sequences at the read position are appended to @a result in one step. The + byte that stops the run (an end-of-string marker, the first byte of the + next value, or an ill-formed byte) is left for @ref get_bon8_string, so + that strings end and errors are reported exactly as without this step. + + @param[in,out] result the string to append to + */ + void get_bon8_string_bulk(string_t& result, std::true_type /*bulk*/) + { + // bytes handed back must be read through get_bon8() first + if (bon8_pushback_size != 0) + { + return; + } + const std::size_t remaining = ia.bulk_remaining(); + if (remaining == 0) + { + return; + } + const auto* const data = reinterpret_cast(ia.bulk_data()); + const std::size_t length = valid_utf8_prefix(data, remaining); + if (length != 0) + { + result.append(reinterpret_cast(data), length); + ia.bulk_skip(length); + chars_read += length; + } + } + + /// input that is not contiguous: strings are read byte by byte + void get_bon8_string_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept {} + /*! @brief read a string @@ -16086,6 +16166,8 @@ class binary_reader { while (true) { + get_bon8_string_bulk(result, std::integral_constant {}); + const auto byte = get_bon8(); if (byte == char_traits::eof()) @@ -21918,7 +22000,7 @@ class binary_writer if (N > 4) { - oa.write_character(to_char_type(0xFE)); + write_bon8_marker(0xFE, string_open); } break; } @@ -21988,20 +22070,10 @@ class binary_writer { static_cast(context); // only used when exceptions are enabled const auto* data = reinterpret_cast(s.data()); - for (std::size_t i = 0; i < s.size();) + const std::size_t valid = valid_utf8_prefix(data, s.size()); + if (JSON_HEDLEY_UNLIKELY(valid != s.size())) { - if (data[i] < 0x80) - { - ++i; - continue; - } - - const std::size_t length = validate_one_utf8(data + i, s.size() - i); - if (JSON_HEDLEY_UNLIKELY(length == 0)) - { - JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(i), ": 0x", hex_byte(data[i])), &context)); - } - i += length; + JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(valid), ": 0x", hex_byte(data[valid])), &context)); } } diff --git a/tests/src/fuzzer-parse_bon8.cpp b/tests/src/fuzzer-parse_bon8.cpp index ac1abdc3f..e97d4f17b 100644 --- a/tests/src/fuzzer-parse_bon8.cpp +++ b/tests/src/fuzzer-parse_bon8.cpp @@ -15,6 +15,10 @@ array data, it performs the following steps: - j2 = from_bon8(vec) - assert(j1 == j2) +It also checks that reading the data from a stream, which reads strings byte by +byte, gives the same value or error as reading it from contiguous memory, which +copies strings in bulk. + The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer drivers. */ @@ -31,9 +35,33 @@ drivers. using json = nlohmann::json; +namespace +{ +// the serialization of the value read from @a input, or the error message +template +std::string read_bon8(InputType&& input) +{ + try + { + const auto vec = json::to_bon8(json::from_bon8(std::forward(input))); + return {vec.begin(), vec.end()}; + } + catch (const json::exception& e) + { + return e.what(); + } +} +} // namespace + // see http://llvm.org/docs/LibFuzzer.html extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { + // contiguous and stream input must be read alike + { + std::istringstream stream(std::string(reinterpret_cast(data), size)); + assert(read_bon8(std::vector(data, data + size)) == read_bon8(stream)); + } + try { // step 1: parse input diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index f54a7ff6b..40a59a973 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -102,6 +102,12 @@ class SaxCountdown using bytes = std::vector; +/// @return the string with the given bytes +std::string str(const bytes& b) +{ + return {b.begin(), b.end()}; +} + /// check that @a j is serialized to @a expected and that @a expected is read back as @a j void check_bon8(const json& j, const bytes& expected) { @@ -112,12 +118,10 @@ void check_bon8(const json& j, const bytes& expected) CHECK(decoded == j); // integers are not read back as floats and vice versa CHECK(decoded.type() == j.type()); -} -/// @return the string with the given bytes -std::string str(const bytes& b) -{ - return {b.begin(), b.end()}; + // a stream is read byte by byte rather than in bulk + std::istringstream stream(str(expected)); + CHECK(json::from_bon8(stream) == decoded); } /// @return @a b followed by @a tail @@ -320,6 +324,9 @@ TEST_CASE("BON8") CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xC2, 'a'})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC2", json::type_error&); CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({'a', 0xE2, 0x82})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xE2", json::type_error&); CHECK_THROWS_WITH_AS(_ = json::to_bon8(json::object({{str({0xFF}), 1}})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); + // after a run of ASCII characters that is checked 8 bytes at a time + CHECK_THROWS_WITH_AS(_ = json::to_bon8(std::string(17, 'a') + str({0xC0})), "[json.exception.type_error.316] invalid UTF-8 byte at index 17: 0xC0", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(std::string(8, 'a') + "\xC3\xA4" + std::string(8, 'a') + str({0xE2, 0x82})), "[json.exception.type_error.316] invalid UTF-8 byte at index 18: 0xE2", json::type_error&); } } @@ -638,6 +645,62 @@ TEST_CASE("BON8") } } +TEST_CASE("BON8 strings from contiguous and stream input") +{ + // contiguous input copies the valid UTF-8 of a string in bulk, a stream + // is read byte by byte; both must end strings and report errors alike + const std::string ascii(20, 'a'); + const std::vector inputs = + { + // the string ends at 0xFF, at a marker, and at an integer + concat(concat(bytes(ascii.begin(), ascii.end()), {0xC3, 0xA4}), {0xFF}), + concat(concat({0x82}, bytes(ascii.begin(), ascii.end())), {0x91}), + concat(concat({0x85}, bytes(ascii.begin(), ascii.end())), {0xFE}), + concat(concat({0x82}, bytes(ascii.begin(), ascii.end())), {0xC2, 0x05}), + concat(concat({0x87}, bytes(ascii.begin(), ascii.end())), {0xE2, 0x82, 0xAC, 0xF0, 0x05, 0x00, 0x00}), + // invalid UTF-8 and a premature end after a run of valid characters + concat(bytes(ascii.begin(), ascii.end()), {0xE0, 0x80, 0x80}), + concat(bytes(ascii.begin(), ascii.end()), {0xE2, 0x82, 0x2F}), + concat(bytes(ascii.begin(), ascii.end()), {0xE2, 0x82}), + bytes(ascii.begin(), ascii.end()), + // trailing bytes after a string that ends a container + concat(concat({0x81}, bytes(ascii.begin(), ascii.end())), {0x91}), + }; + + for (const auto& input : inputs) + { + CAPTURE(input) + std::string from_vector; + std::string from_stream; + try + { + from_vector = json::from_bon8(input).dump(); + } + catch (const json::parse_error& e) + { + from_vector = e.what(); + } + try + { + std::istringstream stream(str(input)); + from_stream = json::from_bon8(stream).dump(); + } + catch (const json::parse_error& e) + { + from_stream = e.what(); + } + CHECK(from_vector == from_stream); + } + + json _; + CHECK(json::from_bon8(inputs[0]) == ascii + "\xC3\xA4"); + CHECK(json::from_bon8(inputs[3]) == json({ascii, 45})); + CHECK_THROWS_WITH_AS(_ = json::from_bon8(inputs[5]), "[json.exception.parse_error.112] parse error at byte 22: syntax error while parsing BON8 string: invalid UTF-8 byte: 0x80", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bon8(inputs[6]), "[json.exception.parse_error.112] parse error at byte 23: syntax error while parsing BON8 string: invalid UTF-8 byte: 0x2F", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bon8(inputs[7]), "[json.exception.parse_error.110] parse error at byte 23: syntax error while parsing BON8 string: unexpected end of input", json::parse_error&); + CHECK_THROWS_WITH_AS(_ = json::from_bon8(inputs[9]), "[json.exception.parse_error.110] parse error at byte 22: syntax error while parsing BON8 value: expected end of input; last byte: 0x91", json::parse_error&); +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("BON8 nesting does not consume the call stack") {