Test escaped object keys decoded through NLOHMANN_DEFINE_TYPE from a json_view

A json_view or json_document decodes a macro-defined struct directly and looks
the members up by their decoded keys. Only plain keys were covered so far.

The new section decodes structs defined with NLOHMANN_DEFINE_TYPE_INTRUSIVE,
..._WITH_DEFAULT and ..._WITH_NAMES from documents whose keys contain escape
sequences, and compares the result with the conversion of the materialized
value. It also pins the difference for a duplicate key written once plain and
once escaped (the view takes the first member, the materialized value keeps
the last one), and that an escaped backslash is part of the decoded key.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/builder.hpp

@@ -1082,7 +1082,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_fraction_digits(p, e);
+                p = skip_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

include/nlohmann/detail/view/scan.hpp

@@ -198,57 +198,6 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
-/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
-/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
-/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
-/// lengths vary. Where there is no SSE2, skip_digits() is used.
-#if NLOHMANN_VIEW_SSE2
-// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
-// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
-// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
-#if defined(__GNUC__) && !defined(__clang__)
-inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#endif
-
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (e - p >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
-        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
-        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
-        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
-        if (not_digit != 0)
-        {
-            return p + count_trailing_zeros(not_digit);
-        }
-        p += 16;
-    }
-    return skip_digits_tail(p, e);
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    return skip_digits(p, e);
-}
-#endif
-
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {

single_include/nlohmann/json_view.hpp

@@ -1218,57 +1218,6 @@ inline const unsigned char* find_block_comment_end(const unsigned char* p, const
     return p;
 }
 
-/// The digits of a fraction, whose number varies from one number to the next: skip_digits() with 16 bytes compared at
-/// once and the end of the run taken from a bit mask (SSE2), instead of a compare and a branch for each digit. That
-/// is about a third of the instructions for a fraction of 12 digits, and no branch that is mispredicted when the
-/// lengths vary. Where there is no SSE2, skip_digits() is used.
-#if NLOHMANN_VIEW_SSE2
-// The last bytes of the input, one at a time. GCC unrolls this loop in every inlined copy of the function below, which
-// makes the parse loop 4 KiB larger and slows down documents that have hardly any fractions; out of line, it does
-// not. Clang does better with the loop inline (a call in the parse loop costs it more than the unrolling).
-#if defined(__GNUC__) && !defined(__clang__)
-inline NLOHMANN_VIEW_NOINLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_digits_tail(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (p != e && is_digit(*p))
-    {
-        ++p;
-    }
-    return p;
-}
-#endif
-
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    while (e - p >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(p))); // NOLINT(bugprone-casting-through-void)
-        // a digit is above '/' and below ':' (signed compares: a byte with the top bit set is negative, and neither)
-        const __m128i digits = _mm_and_si128(_mm_cmpgt_epi8(in, _mm_set1_epi8('/')), _mm_cmplt_epi8(in, _mm_set1_epi8(':')));
-        const auto not_digit = static_cast<std::uint64_t>(~static_cast<unsigned>(_mm_movemask_epi8(digits)) & 0xFFFFu);
-        if (not_digit != 0)
-        {
-            return p + count_trailing_zeros(not_digit);
-        }
-        p += 16;
-    }
-    return skip_digits_tail(p, e);
-}
-#else
-NLOHMANN_VIEW_ALWAYS_INLINE const unsigned char* skip_fraction_digits(const unsigned char* p, const unsigned char* e) noexcept
-{
-    return skip_digits(p, e);
-}
-#endif
-
 /// powers of ten up to 10^19 as integers
 inline std::uint64_t int_pow10(unsigned k) noexcept
 {
@@ -2375,7 +2324,7 @@ indent_done:
             {
                 ++p;
                 const unsigned char* const f0 = p;
-                p = skip_fraction_digits(p, e);
+                p = skip_digits(p, e);
                 if (NLOHMANN_VIEW_UNLIKELY(p == f0))
                 {
                     return fail(error_code::number_after_dot);

tests/src/unit-json_view_builder.cpp

@@ -523,47 +523,6 @@ TEST_CASE("json_view builder: strings across vector blocks")
     }
 }
 
-TEST_CASE("json_view builder: fraction digits")
-{
-    // The digits of a fraction are skipped 16 bytes at a time where there are
-    // that many left (SSE2), and one at a time after that. Fractions of every
-    // length from 0 to 40 digits, followed by every kind of byte that ends a
-    // number or lies next to the digits in ASCII ('/' and ':'), as values, as
-    // keys' neighbours, and at the very end of the text (a vector without a
-    // terminating NUL). The length of the number is part of the value that
-    // the test compares with json::parse().
-    const std::string followers = ",]}/:.eE+-x \n\t\"09";
-    std::mt19937 rng(2028); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
-    for (std::size_t digits = 0; digits <= 40; ++digits)
-    {
-        for (int i = 0; i < 3; ++i)
-        {
-            std::string fraction;
-            for (std::size_t k = 0; k < digits; ++k)
-            {
-                fraction += static_cast<char>('0' + (rng() % 10));
-            }
-            const std::string number = std::string(rng() % 2 == 0 ? "-" : "") + std::to_string(rng() % 100000) + "." + fraction;
-            check_same(number);
-            check_same("[" + number + "]");
-            check_same("[" + number + "," + number + "]");
-            check_same("{\"a\":" + number + "}");
-            for (const char follower : followers)
-            {
-                check_same(number + follower);
-                check_same("[" + number + follower + "]");
-                check_same("[" + number + follower + std::string(rng() % 20, '1'));
-            }
-            // everything near the end of the text: the number cut in every place
-            for (std::size_t cut = 1; cut <= 18 && cut <= number.size(); ++cut)
-            {
-                check_same(number.substr(0, number.size() - cut));
-                check_same("[" + number.substr(0, number.size() - cut));
-            }
-        }
-    }
-}
-
 TEST_CASE("json_view builder: escapes between runs of text")
 {
     // The text between escapes is copied 16 bytes at a time where there are

tests/src/unit-json_view_from_json.cpp

@@ -443,6 +443,30 @@ TEST_CASE("json_view from_json")
         CHECK(d.root().materialize().get<point>().x == 3);
     }
 
+    SECTION("escaped keys: members are found by their decoded names")
+    {
+        // keys are decoded before they are compared, as by parse(); a lookup
+        // that compares the raw bytes of the source would miss these members
+        const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
+        const json j = d.root().materialize();
+        CHECK(d.root()[0].get<point>().x == 1);
+        CHECK(d.root()[0].get<point>().y == 2);
+        CHECK((d.root()[0].get<point>() == j[0].get<point>()));
+        CHECK(d.root()[1].get<with_names>().value == 3);
+        CHECK(d.root()[1].get<with_names>().text == "t");
+        CHECK(d.root()[2].get<with_defaults>().b == "escaped");
+        CHECK(d.root()[2].get<with_defaults>().a == 1);
+        // a duplicate key written once plain and once escaped: still the first member
+        CHECK(d.root()[3].get<point>().x == 1);
+        CHECK(j[3].get<point>().x == 3);
+#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
+        // an escaped backslash is part of the decoded key: "x\\" is not "x"
+        const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
+#endif
+    }
+
     SECTION("editable documents and ordered_json")
     {
         json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");