diff --git a/include/nlohmann/detail/view/builder.hpp b/include/nlohmann/detail/view/builder.hpp index 89b5af2ab..daf0d59bb 100644 --- a/include/nlohmann/detail/view/builder.hpp +++ b/include/nlohmann/detail/view/builder.hpp @@ -343,6 +343,21 @@ class builder } arena_ensure(w, w_end, 4); w = put_utf8(w, cp); + // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is + // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a + // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports + // the same error at the same position as it would without this loop. + while (end - p >= 6 && p[0] == '\\' && p[1] == 'u') + { + const int next = hex_codepoint(p + 2); + if (next < 0 || (next & 0xF800) == 0xD800) + { + break; + } + p += 6; + arena_ensure(w, w_end, 4); + w = put_utf8(w, next); + } } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 8ff21c77f..e490f61cc 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -1585,6 +1585,21 @@ class builder } arena_ensure(w, w_end, 4); w = put_utf8(w, cp); + // A chain of unicode escapes ("\u3042\u3044\u3046", as Python's json.dumps writes non-ASCII text) is + // decoded here, without going round the loop above for each of them. Anything else (a surrogate, a + // bad digit, fewer than six bytes) leaves p at the backslash: the loop above handles it, and reports + // the same error at the same position as it would without this loop. + while (end - p >= 6 && p[0] == '\\' && p[1] == 'u') + { + const int next = hex_codepoint(p + 2); + if (next < 0 || (next & 0xF800) == 0xD800) + { + break; + } + p += 6; + arena_ensure(w, w_end, 4); + w = put_utf8(w, next); + } } else if (NLOHMANN_VIEW_LIKELY(d < 128 && simple_escape[d] != 0)) { diff --git a/tests/src/unit-json_view_builder.cpp b/tests/src/unit-json_view_builder.cpp index 4023c7848..7ac7a7d40 100644 --- a/tests/src/unit-json_view_builder.cpp +++ b/tests/src/unit-json_view_builder.cpp @@ -575,6 +575,59 @@ TEST_CASE("json_view builder: escapes between runs of text") } } +TEST_CASE("json_view builder: chains of unicode escapes") +{ + // Consecutive \uXXXX escapes are decoded in a loop of their own. Chains of + // every length from 1 to 40 of escapes that give one, two, and three + // bytes of UTF-8, in upper and lower case, followed by what ends a chain: + // the end of the string, text, another escape, UTF-8, and everything that + // is not a valid escape (a bad digit at every position of the last + // escape, a lone surrogate, a surrogate pair, a cut in every place). + const std::array chain = {{"\\u0041", "\\u00e4", "\\u00E4", "\\u20ac", "\\u3042", "\\uFFFF", "\\u0000", "\\u07ff"}}; + const std::array ends = {{"", "abc", "\\n", "\\u00e9", "\\ud83d\\ude00", "\\ud83d", "\\ude00", "\\ud83d\\u0041", "\\u12G4", "\\u12", "\xc3\xa4", "\\"}}; + std::mt19937 rng(2027); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible + for (std::size_t length = 1; length <= 40; ++length) + { + for (int i = 0; i < 20; ++i) + { + std::string content = std::string(rng() % 20, 'a'); + for (std::size_t k = 0; k < length; ++k) + { + content += chain[rng() % chain.size()]; + } + const std::string whole = content; + for (const char* end : ends) + { + check_string(whole + end); + check_string(whole + end + std::string(rng() % 19, 'z')); + } + // a bad digit at every position of every escape near the end of the chain + for (std::size_t back = 1; back <= 6 && back <= length; ++back) + { + for (std::size_t digit = 2; digit < 6; ++digit) + { + std::string broken = whole; + broken[broken.size() - (6 * back) + digit] = "gG\"\x01 z"[rng() % 6]; + check_string(broken); + } + } + // the text cut in every place of the last three escapes + for (std::size_t cut = 0; cut <= 18 && cut <= whole.size(); ++cut) + { + check_string(whole.substr(0, whole.size() - cut)); + } + } + } + // a chain longer than the first growth of the arena, in a key and in a value + std::string long_chain; + for (int i = 0; i < 3000; ++i) + { + long_chain += chain[static_cast(i) % chain.size()]; + } + check_string(long_chain); + check_string(long_chain + "\\ud83d\\ude00" + long_chain); +} + TEST_CASE("json_view node integer bits") { using nlohmann::detail::view::integer_bits;