From 5f673b0eaef202135892432ed086a94c9ba7935d Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 25 Sep 2026 08:16:55 +0200 Subject: [PATCH] Address review comments - Reuse detail::validate_one_utf8 to check strings in to_bon8; the error now names the first byte of the invalid sequence. - Document that to_bon8 leaves bytes in the output adapter on an exception, and that string_open is only an output of write_bon8_marker. - Explain why the pushback buffer of the BON8 reader cannot overflow. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/to_bon8.md | 3 +- .../nlohmann/detail/input/binary_reader.hpp | 6 ++ .../nlohmann/detail/output/binary_writer.hpp | 65 ++++------------- single_include/nlohmann/json.hpp | 72 +++++-------------- tests/src/unit-bon8.cpp | 10 +-- 5 files changed, 44 insertions(+), 112 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/to_bon8.md b/docs/mkdocs/docs/api/basic_json/to_bon8.md index bd2d70d43..26d552e8a 100644 --- a/docs/mkdocs/docs/api/basic_json/to_bon8.md +++ b/docs/mkdocs/docs/api/basic_json/to_bon8.md @@ -32,7 +32,8 @@ The exact mapping and its limitations are described on a [dedicated page](../../ ## Exception safety -Strong guarantee: if an exception is thrown, there are no changes in the JSON value. +Strong guarantee: if an exception is thrown, there are no changes in the JSON value `j`, which is never modified. +With (2), the bytes written before the exception remain in the output adapter. ## Exceptions diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index e9958ad87..cbe515855 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3215,6 +3215,12 @@ class binary_reader */ void unget_bon8(const char_int_type c) { + // At most two bytes are ever handed back: a byte is only handed back + // right after it was read with get_bon8(), and the only place that + // hands back two bytes (a lead byte and the byte after it) read both + // of them in a row, which emptied the buffer first. This is an + // invariant of the reader rather than a property of the input, so + // an assertion suffices (the fuzzers are built with assertions). JSON_ASSERT(bon8_pushback_size < bon8_pushback.size()); bon8_pushback[bon8_pushback_size++] = c; --chars_read; diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index dc108e9ae..db6c90394 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -25,6 +25,7 @@ #endif #include +#include #include #include #include @@ -2179,8 +2180,9 @@ class binary_writer /*! @brief write a single byte that is not part of a string - @param[in] marker the byte to write - @param[in,out] string_open see @ref write_bon8_value + @param[in] marker the byte to write + @param[out] string_open set to false, because the output no longer ends + with a string; see @ref write_bon8_value */ void write_bon8_marker(const std::uint8_t marker, bool& string_open) { @@ -2227,72 +2229,29 @@ class binary_writer @param[in] s the string to check @param[in] context the value the string belongs to (for diagnostics) - @throw type_error.316 if @a s is not valid UTF-8 + @throw type_error.316 if @a s is not valid UTF-8; the message names the + first byte of the first invalid or incomplete sequence */ static void check_bon8_utf8(const string_t& s, const BasicJsonType& context) { - const auto byte_at = [&s](const std::size_t i) - { - return static_cast(s[i]); - }; - + const auto* data = reinterpret_cast(s.data()); for (std::size_t i = 0; i < s.size();) { - const std::uint8_t lead = byte_at(i); - std::size_t continuation_bytes = 0; - // valid range of the byte after the lead byte, which excludes - // overlong forms, surrogates, and code points above U+10FFFF - std::uint8_t lower = 0x80; - std::uint8_t upper = 0xBF; - - if (lead <= 0x7F) + if (data[i] < 0x80) { ++i; continue; } - if (0xC2 <= lead && lead <= 0xDF) - { - continuation_bytes = 1; - } - else if (0xE0 <= lead && lead <= 0xEF) - { - continuation_bytes = 2; - lower = lead == 0xE0 ? 0xA0 : 0x80; - upper = lead == 0xED ? 0x9F : 0xBF; - } - else if (0xF0 <= lead && lead <= 0xF4) - { - continuation_bytes = 3; - lower = lead == 0xF0 ? 0x90 : 0x80; - upper = lead == 0xF4 ? 0x8F : 0xBF; - } - else - { - throw_bon8_utf8_error(i, lead, context); - } - for (std::size_t k = 1; k <= continuation_bytes; ++k) + const std::size_t length = validate_one_utf8(data + i, s.size() - i); + if (JSON_HEDLEY_UNLIKELY(length == 0)) { - if (i + k >= s.size()) - { - JSON_THROW(type_error::create(316, concat("incomplete UTF-8 string; last byte: 0x", hex_byte(byte_at(s.size() - 1))), &context)); - } - const std::uint8_t b = byte_at(i + k); - if (b < (k == 1 ? lower : 0x80) || b > (k == 1 ? upper : 0xBF)) - { - throw_bon8_utf8_error(i + k, b, context); - } + JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(i), ": 0x", hex_byte(data[i])), &context)); } - - i += continuation_bytes + 1; + i += length; } } - [[noreturn]] static void throw_bon8_utf8_error(const std::size_t index, const std::uint8_t byte, const BasicJsonType& context) - { - JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(index), ": 0x", hex_byte(byte)), &context)); - } - /// @return a byte as two uppercase hexadecimal digits static std::string hex_byte(const std::uint8_t byte) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 1a13a935e..dadda405c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -15538,6 +15538,12 @@ class binary_reader */ void unget_bon8(const char_int_type c) { + // At most two bytes are ever handed back: a byte is only handed back + // right after it was read with get_bon8(), and the only place that + // hands back two bytes (a lead byte and the byte after it) read both + // of them in a row, which emptied the buffer first. This is an + // invariant of the reader rather than a property of the input, so + // an assertion suffices (the fuzzers are built with assertions). JSON_ASSERT(bon8_pushback_size < bon8_pushback.size()); bon8_pushback[bon8_pushback_size++] = c; --chars_read; @@ -19348,6 +19354,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include // #include @@ -21725,8 +21733,9 @@ class binary_writer /*! @brief write a single byte that is not part of a string - @param[in] marker the byte to write - @param[in,out] string_open see @ref write_bon8_value + @param[in] marker the byte to write + @param[out] string_open set to false, because the output no longer ends + with a string; see @ref write_bon8_value */ void write_bon8_marker(const std::uint8_t marker, bool& string_open) { @@ -21773,72 +21782,29 @@ class binary_writer @param[in] s the string to check @param[in] context the value the string belongs to (for diagnostics) - @throw type_error.316 if @a s is not valid UTF-8 + @throw type_error.316 if @a s is not valid UTF-8; the message names the + first byte of the first invalid or incomplete sequence */ static void check_bon8_utf8(const string_t& s, const BasicJsonType& context) { - const auto byte_at = [&s](const std::size_t i) - { - return static_cast(s[i]); - }; - + const auto* data = reinterpret_cast(s.data()); for (std::size_t i = 0; i < s.size();) { - const std::uint8_t lead = byte_at(i); - std::size_t continuation_bytes = 0; - // valid range of the byte after the lead byte, which excludes - // overlong forms, surrogates, and code points above U+10FFFF - std::uint8_t lower = 0x80; - std::uint8_t upper = 0xBF; - - if (lead <= 0x7F) + if (data[i] < 0x80) { ++i; continue; } - if (0xC2 <= lead && lead <= 0xDF) - { - continuation_bytes = 1; - } - else if (0xE0 <= lead && lead <= 0xEF) - { - continuation_bytes = 2; - lower = lead == 0xE0 ? 0xA0 : 0x80; - upper = lead == 0xED ? 0x9F : 0xBF; - } - else if (0xF0 <= lead && lead <= 0xF4) - { - continuation_bytes = 3; - lower = lead == 0xF0 ? 0x90 : 0x80; - upper = lead == 0xF4 ? 0x8F : 0xBF; - } - else - { - throw_bon8_utf8_error(i, lead, context); - } - for (std::size_t k = 1; k <= continuation_bytes; ++k) + const std::size_t length = validate_one_utf8(data + i, s.size() - i); + if (JSON_HEDLEY_UNLIKELY(length == 0)) { - if (i + k >= s.size()) - { - JSON_THROW(type_error::create(316, concat("incomplete UTF-8 string; last byte: 0x", hex_byte(byte_at(s.size() - 1))), &context)); - } - const std::uint8_t b = byte_at(i + k); - if (b < (k == 1 ? lower : 0x80) || b > (k == 1 ? upper : 0xBF)) - { - throw_bon8_utf8_error(i + k, b, context); - } + JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(i), ": 0x", hex_byte(data[i])), &context)); } - - i += continuation_bytes + 1; + i += length; } } - [[noreturn]] static void throw_bon8_utf8_error(const std::size_t index, const std::uint8_t byte, const BasicJsonType& context) - { - JSON_THROW(type_error::create(316, concat("invalid UTF-8 byte at index ", std::to_string(index), ": 0x", hex_byte(byte)), &context)); - } - /// @return a byte as two uppercase hexadecimal digits static std::string hex_byte(const std::uint8_t byte) { diff --git a/tests/src/unit-bon8.cpp b/tests/src/unit-bon8.cpp index f18f15ee4..f3c0ae42d 100644 --- a/tests/src/unit-bon8.cpp +++ b/tests/src/unit-bon8.cpp @@ -313,12 +313,12 @@ TEST_CASE("BON8") json _; CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0x80", json::type_error&); CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({'a', 0xC0, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xC0", json::type_error&); - CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xE0, 0x80, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x80", json::type_error&); - CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xED, 0xA0, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xA0", json::type_error&); - CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xF4, 0x90, 0x80, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x90", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xE0, 0x80, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xE0", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xED, 0xA0, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xF4, 0x90, 0x80, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xF4", json::type_error&); CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xF5, 0x80, 0x80, 0x80})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xF5", json::type_error&); - CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xC2, 'a'})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0x61", json::type_error&); - CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({'a', 0xE2, 0x82})), "[json.exception.type_error.316] incomplete UTF-8 string; last byte: 0x82", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({0xC2, 'a'})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC2", json::type_error&); + CHECK_THROWS_WITH_AS(_ = json::to_bon8(str({'a', 0xE2, 0x82})), "[json.exception.type_error.316] invalid UTF-8 byte at index 1: 0xE2", json::type_error&); CHECK_THROWS_WITH_AS(_ = json::to_bon8(json::object({{str({0xFF}), 1}})), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&); } }