1.5: use utf8_shape in get_bon8_string, add exhaustive UTF-8 validator test (#5891)

get_bon8_string takes the valid range of the second byte from utf8_shape()
(0xF5..0xF7 stay errors, unget behavior unchanged). validate_one_utf8 keeps its
if-chain (hot bulk path); the new test checks decode, validate_one_utf8,
utf8_shape, json::accept and the BON8 reader against each other for every lead
and second byte.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -3770,23 +3770,11 @@ class binary_reader
 
             // the valid range of the second byte excludes overlong forms,
             // surrogates, and code points above U+10FFFF
-            // (RFC 3629, section 4)
-            int continuation_bytes = 0;
-            bool valid_second = true;
-            if (byte <= 0xDF)
-            {
-                continuation_bytes = 1;
-            }
-            else if (byte <= 0xEF)
-            {
-                continuation_bytes = 2;
-                valid_second = (byte != 0xE0 || second >= 0xA0) && (byte != 0xED || second <= 0x9F);
-            }
-            else
-            {
-                continuation_bytes = 3;
-                valid_second = byte <= 0xF4 && (byte != 0xF0 || second >= 0x90) && (byte != 0xF4 || second <= 0x8F);
-            }
+            // (RFC 3629, section 4); length 0 flags the lead bytes
+            // 0xF5..0xF7, which BON8 can read but UTF-8 cannot encode
+            const auto shape = utf8_shape(static_cast<std::uint8_t>(byte));
+            const int continuation_bytes = (byte <= 0xDF) ? 1 : ((byte <= 0xEF) ? 2 : 3);
+            const bool valid_second = shape.length != 0 && second >= shape.second_lo && second <= shape.second_hi;
 
             if (JSON_HEDLEY_UNLIKELY(!valid_second))
             {

tests/src/unit-class_lexer.cpp

@@ -2071,3 +2071,86 @@ TEST_CASE("find_ascii_copyable_run agrees with is_ascii_copyable for every byte"
         }
     }
 }
+
+TEST_CASE("UTF-8 validators accept the same sequences")
+{
+    // The library checks UTF-8 in several places: detail::decode (serializer),
+    // the per-lead-byte switch of the lexer (json::accept), validate_one_utf8
+    // (bulk string scan), utf8_shape, and the byte path of the BON8 reader.
+    // They must accept exactly the same sequences (RFC 3629, section 4).
+    using nlohmann::json;
+
+    // bytes for the third and fourth position: the first two pairs are checked
+    // against all validators, the others only against the cheap ones
+    const std::array<std::array<unsigned char, 2>, 6> tails =
+    {
+        {
+            {{0x80, 0x80}}, {{0xBF, 0xBF}}, {{0x7F, 0x80}}, {{0xC0, 0x80}}, {{0x80, 0x7F}}, {{0x80, 0xC0}}
+        }
+    };
+
+    for (unsigned lead = 0x80; lead <= 0xFF; ++lead)
+    {
+        const auto shape = nlohmann::detail::utf8_shape(static_cast<std::uint8_t>(lead));
+        for (unsigned second = 0; second <= 0xFF; ++second)
+        {
+            for (std::size_t t = 0; t < tails.size(); ++t)
+            {
+                const std::array<unsigned char, 4> bytes = {{static_cast<unsigned char>(lead), static_cast<unsigned char>(second), tails[t][0], tails[t][1]}};
+
+                // expectation from the shape: the sequence is bytes[0..length)
+                const std::size_t length = shape.length;
+                bool expected = length >= 2 && second >= shape.second_lo && second <= shape.second_hi;
+                for (std::size_t i = 2; expected && i < length; ++i)
+                {
+                    expected = bytes[i] >= 0x80 && bytes[i] <= 0xBF;
+                }
+
+                // the DFA of the serializer
+                bool dfa = length >= 2;
+                if (dfa)
+                {
+                    std::uint8_t state = nlohmann::detail::UTF8_ACCEPT;
+                    std::uint32_t codep = 0;
+                    for (std::size_t i = 0; i < length; ++i)
+                    {
+                        nlohmann::detail::decode(state, codep, bytes[i]);
+                    }
+                    dfa = state == nlohmann::detail::UTF8_ACCEPT;
+                }
+
+                // the bulk path: exactly the sequence, and one byte short
+                const std::size_t bulk = nlohmann::detail::validate_one_utf8(bytes.data(), length);
+                const std::size_t bulk_short = length > 0 ? nlohmann::detail::validate_one_utf8(bytes.data(), length - 1) : 0;
+
+                CAPTURE(lead)
+                CAPTURE(second)
+                CAPTURE(t)
+                CHECK((dfa == expected && bulk == (expected ? length : 0) && bulk_short == 0));
+
+                if (t < 2 && length >= 2)
+                {
+                    // the lexer's byte path
+                    const std::string sequence(reinterpret_cast<const char*>(bytes.data()), length); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+                    CHECK(json::accept("\"" + sequence + "\"") == expected);
+
+                    // the byte path of the BON8 reader (a string ends at 0xFF)
+                    if (lead >= 0xC2 && lead <= 0xF7)
+                    {
+                        std::vector<std::uint8_t> packed(bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(length));
+                        packed.push_back(0xFF);
+                        const json j = json::from_bon8(packed, true, false);
+                        if (expected)
+                        {
+                            CHECK(j == json(sequence));
+                        }
+                        else
+                        {
+                            CHECK_FALSE((j.is_string() && j.get<std::string>() == sequence));
+                        }
+                    }
+                }
+            }
+        }
+    }
+}