Test escaped object keys decoded through NLOHMANN_DEFINE_TYPE from a json_view

A json_view or json_document decodes a macro-defined struct directly and looks
the members up by their decoded keys. Only plain keys were covered so far.

The new section decodes structs defined with NLOHMANN_DEFINE_TYPE_INTRUSIVE,
..._WITH_DEFAULT and ..._WITH_NAMES from documents whose keys contain escape
sequences, and compares the result with the conversion of the materialized
value. It also pins the difference for a duplicate key written once plain and
once escaped (the view takes the first member, the materialized value keeps
the last one), and that an escaped backslash is part of the decoded key.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -465,17 +465,15 @@ class binary_reader
             }
 
             const std::size_t element_type_parse_position = chars_read;
-            // an array's elements are named "0", "1", ... in the wire format,
-            // and those names are not passed on: they are skipped unless
-            // error_handler_t::strict must reject ill-formed UTF-8 in them
-            const bool is_object = container_stack.back().is_object;
             key.clear();
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
             {
                 return false;
             }
 
-            if (is_object && !sax->key(key))
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on
+            if (container_stack.back().is_object && !sax->key(key))
             {
                 return false;
             }
@@ -491,18 +489,14 @@ class binary_reader
     @brief Parses a C-style string from the BSON input.
     @param[in,out] result  A reference to the string variable where the read
                             string is to be stored.
-    @param[in] store  whether the characters are needed; if not (the names of
-                      an array's elements), they are skipped and @a result may
-                      be left unchanged, and they are not checked against
-                      @ref error_handler
     @return `true` if the \\x00-byte indicating the end of the string was
              encountered before the EOF; false` indicates an unexpected EOF.
     */
-    bool get_bson_cstr(string_t& result, const bool store = true)
+    bool get_bson_cstr(string_t& result)
     {
-        if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
+        if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
         {
-            return !store || check_string_utf8(result, "key");
+            return check_string_utf8(result, "key");
         }
 
         auto out = std::back_inserter(result);
@@ -515,7 +509,7 @@ class binary_reader
             }
             if (current == 0x00)
             {
-                return !store || check_string_utf8(result, "key");
+                return check_string_utf8(result, "key");
             }
             *out++ = static_cast<typename string_t::value_type>(current);
         }
@@ -525,11 +519,10 @@ class binary_reader
     @brief read a C-style string from contiguous input in one step
 
     @param[in,out] result  the string to append to
-    @param[in] store  whether to append the characters to @a result
     @return whether the string was read; if the input has no \\x00-byte, nothing
             is read, and @ref get_bson_cstr reports the end of the input
     */
-    bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
+    bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
     {
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
@@ -537,15 +530,18 @@ class binary_reader
             return false;
         }
         const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
-        const std::size_t length = find_zero_byte(data, remaining);
+        // a plain loop rather than std::memchr: most keys are short (array
+        // indices are keys, too), and the call would cost more than it saves
+        std::size_t length = 0;
+        while (length < remaining && data[length] != 0x00)
+        {
+            ++length;
+        }
         if (length == remaining)
         {
             return false;
         }
-        if (store)
-        {
-            result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
-        }
+        result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
         // consume the string and its \x00-byte, as the byte-wise path does
         ia.bulk_skip(length + 1);
         chars_read += length + 1;
@@ -554,7 +550,7 @@ class binary_reader
     }
 
     /// input that is not contiguous: C-style strings are read byte by byte
-    bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
+    bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
     {
         return false;
     }

include/nlohmann/detail/input/string_scan.hpp

@@ -90,34 +90,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
     return n;
 }
 
-// return the index of the first 0x00 byte in [data, data+n), or n if there is
-// none; scans 8 bytes at a time (BSON's C-style strings)
-JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t ones = 0x0101010101010101ull;
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    std::size_t i = 0;
-    for (; i + 8 <= n; i += 8)
-    {
-        const std::uint64_t v = read_eight_bytes(data + i);
-        const std::uint64_t zero = (v - ones) & ~v & high;
-        if (zero != 0)
-        {
-            // the lowest flagged byte is the first zero byte (see
-            // find_string_special())
-            return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
-        }
-    }
-    for (; i < n; ++i)
-    {
-        if (data[i] == 0x00)
-        {
-            return i;
-        }
-    }
-    return n;
-}
-
 // classify a byte as one the serializer must NOT copy verbatim when
 // ensure_ascii is requested: the closing quote, an escape, a control character
 // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -

single_include/nlohmann/json.hpp

@@ -12458,34 +12458,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
     return n;
 }
 
-// return the index of the first 0x00 byte in [data, data+n), or n if there is
-// none; scans 8 bytes at a time (BSON's C-style strings)
-JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t ones = 0x0101010101010101ull;
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    std::size_t i = 0;
-    for (; i + 8 <= n; i += 8)
-    {
-        const std::uint64_t v = read_eight_bytes(data + i);
-        const std::uint64_t zero = (v - ones) & ~v & high;
-        if (zero != 0)
-        {
-            // the lowest flagged byte is the first zero byte (see
-            // find_string_special())
-            return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
-        }
-    }
-    for (; i < n; ++i)
-    {
-        if (data[i] == 0x00)
-        {
-            return i;
-        }
-    }
-    return n;
-}
-
 // classify a byte as one the serializer must NOT copy verbatim when
 // ensure_ascii is requested: the closing quote, an escape, a control character
 // (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -18269,17 +18241,15 @@ class binary_reader
             }
 
             const std::size_t element_type_parse_position = chars_read;
-            // an array's elements are named "0", "1", ... in the wire format,
-            // and those names are not passed on: they are skipped unless
-            // error_handler_t::strict must reject ill-formed UTF-8 in them
-            const bool is_object = container_stack.back().is_object;
             key.clear();
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
             {
                 return false;
             }
 
-            if (is_object && !sax->key(key))
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on
+            if (container_stack.back().is_object && !sax->key(key))
             {
                 return false;
             }
@@ -18295,18 +18265,14 @@ class binary_reader
     @brief Parses a C-style string from the BSON input.
     @param[in,out] result  A reference to the string variable where the read
                             string is to be stored.
-    @param[in] store  whether the characters are needed; if not (the names of
-                      an array's elements), they are skipped and @a result may
-                      be left unchanged, and they are not checked against
-                      @ref error_handler
     @return `true` if the \\x00-byte indicating the end of the string was
              encountered before the EOF; false` indicates an unexpected EOF.
     */
-    bool get_bson_cstr(string_t& result, const bool store = true)
+    bool get_bson_cstr(string_t& result)
     {
-        if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
+        if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
         {
-            return !store || check_string_utf8(result, "key");
+            return check_string_utf8(result, "key");
         }
 
         auto out = std::back_inserter(result);
@@ -18319,7 +18285,7 @@ class binary_reader
             }
             if (current == 0x00)
             {
-                return !store || check_string_utf8(result, "key");
+                return check_string_utf8(result, "key");
             }
             *out++ = static_cast<typename string_t::value_type>(current);
         }
@@ -18329,11 +18295,10 @@ class binary_reader
     @brief read a C-style string from contiguous input in one step
 
     @param[in,out] result  the string to append to
-    @param[in] store  whether to append the characters to @a result
     @return whether the string was read; if the input has no \\x00-byte, nothing
             is read, and @ref get_bson_cstr reports the end of the input
     */
-    bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
+    bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
     {
         const std::size_t remaining = ia.bulk_remaining();
         if (remaining == 0)
@@ -18341,15 +18306,18 @@ class binary_reader
             return false;
         }
         const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
-        const std::size_t length = find_zero_byte(data, remaining);
+        // a plain loop rather than std::memchr: most keys are short (array
+        // indices are keys, too), and the call would cost more than it saves
+        std::size_t length = 0;
+        while (length < remaining && data[length] != 0x00)
+        {
+            ++length;
+        }
         if (length == remaining)
         {
             return false;
         }
-        if (store)
-        {
-            result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
-        }
+        result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
         // consume the string and its \x00-byte, as the byte-wise path does
         ia.bulk_skip(length + 1);
         chars_read += length + 1;
@@ -18358,7 +18326,7 @@ class binary_reader
     }
 
     /// input that is not contiguous: C-style strings are read byte by byte
-    bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
+    bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
     {
         return false;
     }

tests/src/unit-bson.cpp

@@ -1164,13 +1164,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
     // contiguous input reads a key up to its \x00-byte in one step, a stream
     // reads it byte by byte; both must give the same value or error for the
     // complete document and for every truncation of it
-    json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
-    // keys and array element names ("0" to "11") that end at every position of
-    // an 8-byte word, which contiguous input scans at a time
-    for (std::size_t length = 1; length <= 17; ++length)
-    {
-        j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
-    }
+    const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
     const std::vector<std::uint8_t> bson = json::to_bson(j);
     CHECK(json::from_bson(bson) == j);
 
@@ -1190,7 +1184,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
         }
         try
         {
-            std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
+            std::istringstream stream(std::string(input.begin(), input.end()));
             from_stream = json::from_bson(stream).dump();
         }
         catch (const json::parse_error& e)
@@ -1200,33 +1194,6 @@ TEST_CASE("BSON keys from contiguous and stream input")
         CHECK(from_vector == from_stream);
     }
 }
-
-TEST_CASE("BSON array element names")
-{
-    // an array's element names are not part of the value, so they are not
-    // passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
-    const std::vector<std::uint8_t> bson =
-    {
-        0x1D, 0x00, 0x00, 0x00, // size (little endian)
-        0x04, 'a', 0x00,        // entry: array "a"
-        0x15, 0x00, 0x00, 0x00, // size (little endian)
-        0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
-        0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
-        0x00,                   // end of the array
-        0x00                    // end of the document
-    };
-    const json expected = {{"a", {true, false}}};
-
-    CHECK(json::from_bson(bson) == expected);
-    CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
-    CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
-
-    json _;
-    CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
-
-    std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
-    CHECK(json::from_bson(stream) == expected);
-}
 #endif
 
 TEST_CASE("Negative size of binary value")

tests/src/unit-json_view_from_json.cpp

@@ -443,6 +443,30 @@ TEST_CASE("json_view from_json")
         CHECK(d.root().materialize().get<point>().x == 3);
     }
 
+    SECTION("escaped keys: members are found by their decoded names")
+    {
+        // keys are decoded before they are compared, as by parse(); a lookup
+        // that compares the raw bytes of the source would miss these members
+        const json_document d = json_document::parse(R"([{"\u0078": 1, "y": 2}, {"V\u0061lue": 3, "Te\u0078t": "t"}, {"\u0062": "escaped"}, {"x": 1, "\u0078": 3, "y": 2}])");
+        const json j = d.root().materialize();
+        CHECK(d.root()[0].get<point>().x == 1);
+        CHECK(d.root()[0].get<point>().y == 2);
+        CHECK((d.root()[0].get<point>() == j[0].get<point>()));
+        CHECK(d.root()[1].get<with_names>().value == 3);
+        CHECK(d.root()[1].get<with_names>().text == "t");
+        CHECK(d.root()[2].get<with_defaults>().b == "escaped");
+        CHECK(d.root()[2].get<with_defaults>().a == 1);
+        // a duplicate key written once plain and once escaped: still the first member
+        CHECK(d.root()[3].get<point>().x == 1);
+        CHECK(j[3].get<point>().x == 3);
+#if !defined(JSON_NOEXCEPTION) && !JSON_DIAGNOSTICS && !JSON_DIAGNOSTIC_POSITIONS
+        // an escaped backslash is part of the decoded key: "x\\" is not "x"
+        const json_document e = json_document::parse(R"({"x\\": 1, "y": 2})");
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == exception_of_call([&] { static_cast<void>(e.root().materialize().get<point>()); }));
+        CHECK(exception_of_call([&] { static_cast<void>(e.root().get<point>()); }) == "[json.exception.out_of_range.403] key 'x' not found");
+#endif
+    }
+
     SECTION("editable documents and ordered_json")
     {
         json_editable_document e = json_editable_document::parse(R"({"x": 1, "y": 2})");