From 643acf23e2c0cd9df59244415703e958747bc02f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 23 Oct 2026 10:16:09 +0200 Subject: [PATCH] 1.5: use utf8_shape in get_bon8_string, add exhaustive UTF-8 validator test (#5891) get_bon8_string takes the valid range of the second byte from utf8_shape() (0xF5..0xF7 stay errors, unget behavior unchanged). validate_one_utf8 keeps its if-chain (hot bulk path); the new test checks decode, validate_one_utf8, utf8_shape, json::accept and the BON8 reader against each other for every lead and second byte. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 22 ++--- tests/src/unit-class_lexer.cpp | 83 +++++++++++++++++++ 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index fef2c0f93..d644ccddd 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -3770,23 +3770,11 @@ class binary_reader // the valid range of the second byte excludes overlong forms, // surrogates, and code points above U+10FFFF - // (RFC 3629, section 4) - int continuation_bytes = 0; - bool valid_second = true; - if (byte <= 0xDF) - { - continuation_bytes = 1; - } - else if (byte <= 0xEF) - { - continuation_bytes = 2; - valid_second = (byte != 0xE0 || second >= 0xA0) && (byte != 0xED || second <= 0x9F); - } - else - { - continuation_bytes = 3; - valid_second = byte <= 0xF4 && (byte != 0xF0 || second >= 0x90) && (byte != 0xF4 || second <= 0x8F); - } + // (RFC 3629, section 4); length 0 flags the lead bytes + // 0xF5..0xF7, which BON8 can read but UTF-8 cannot encode + const auto shape = utf8_shape(static_cast(byte)); + const int continuation_bytes = (byte <= 0xDF) ? 1 : ((byte <= 0xEF) ? 2 : 3); + const bool valid_second = shape.length != 0 && second >= shape.second_lo && second <= shape.second_hi; if (JSON_HEDLEY_UNLIKELY(!valid_second)) { diff --git a/tests/src/unit-class_lexer.cpp b/tests/src/unit-class_lexer.cpp index c35d8e1a1..0f9d4cc6d 100644 --- a/tests/src/unit-class_lexer.cpp +++ b/tests/src/unit-class_lexer.cpp @@ -2071,3 +2071,86 @@ TEST_CASE("find_ascii_copyable_run agrees with is_ascii_copyable for every byte" } } } + +TEST_CASE("UTF-8 validators accept the same sequences") +{ + // The library checks UTF-8 in several places: detail::decode (serializer), + // the per-lead-byte switch of the lexer (json::accept), validate_one_utf8 + // (bulk string scan), utf8_shape, and the byte path of the BON8 reader. + // They must accept exactly the same sequences (RFC 3629, section 4). + using nlohmann::json; + + // bytes for the third and fourth position: the first two pairs are checked + // against all validators, the others only against the cheap ones + const std::array, 6> tails = + { + { + {{0x80, 0x80}}, {{0xBF, 0xBF}}, {{0x7F, 0x80}}, {{0xC0, 0x80}}, {{0x80, 0x7F}}, {{0x80, 0xC0}} + } + }; + + for (unsigned lead = 0x80; lead <= 0xFF; ++lead) + { + const auto shape = nlohmann::detail::utf8_shape(static_cast(lead)); + for (unsigned second = 0; second <= 0xFF; ++second) + { + for (std::size_t t = 0; t < tails.size(); ++t) + { + const std::array bytes = {{static_cast(lead), static_cast(second), tails[t][0], tails[t][1]}}; + + // expectation from the shape: the sequence is bytes[0..length) + const std::size_t length = shape.length; + bool expected = length >= 2 && second >= shape.second_lo && second <= shape.second_hi; + for (std::size_t i = 2; expected && i < length; ++i) + { + expected = bytes[i] >= 0x80 && bytes[i] <= 0xBF; + } + + // the DFA of the serializer + bool dfa = length >= 2; + if (dfa) + { + std::uint8_t state = nlohmann::detail::UTF8_ACCEPT; + std::uint32_t codep = 0; + for (std::size_t i = 0; i < length; ++i) + { + nlohmann::detail::decode(state, codep, bytes[i]); + } + dfa = state == nlohmann::detail::UTF8_ACCEPT; + } + + // the bulk path: exactly the sequence, and one byte short + const std::size_t bulk = nlohmann::detail::validate_one_utf8(bytes.data(), length); + const std::size_t bulk_short = length > 0 ? nlohmann::detail::validate_one_utf8(bytes.data(), length - 1) : 0; + + CAPTURE(lead) + CAPTURE(second) + CAPTURE(t) + CHECK((dfa == expected && bulk == (expected ? length : 0) && bulk_short == 0)); + + if (t < 2 && length >= 2) + { + // the lexer's byte path + const std::string sequence(reinterpret_cast(bytes.data()), length); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + CHECK(json::accept("\"" + sequence + "\"") == expected); + + // the byte path of the BON8 reader (a string ends at 0xFF) + if (lead >= 0xC2 && lead <= 0xF7) + { + std::vector packed(bytes.begin(), bytes.begin() + static_cast(length)); + packed.push_back(0xFF); + const json j = json::from_bon8(packed, true, false); + if (expected) + { + CHECK(j == json(sequence)); + } + else + { + CHECK_FALSE((j.is_string() && j.get() == sequence)); + } + } + } + } + } + } +}