create objects for tokens that cannot be array indices (#5357)

When operator[](json_pointer) traverses a level that does not exist yet,
the null value is turned into an array or an object depending on the
reference token. The check only tested whether all characters are digits,
so tokens that can never be a valid array index selected an array and
then failed:

- "01" (and any other token with a leading '0') threw parse_error.106
- the empty token threw out_of_range.404

Both tokens are valid object keys, and both work when the level already
exists as an object, so creating the level changed the outcome. Test the
token against the RFC 6901, Sect. 4 grammar for array indices instead, so
that such tokens create an object. This only affects pointers that threw
before.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/operator[].md

@@ -128,10 +128,13 @@ Strong exception safety: if an exception occurs, the original value stays intact
 
         When the JSON pointer traverses intermediate levels that don't exist at all yet (not just a missing
         leaf), each missing level is created as an array or an object depending on whether the corresponding
-        pointer token parses as a non-negative integer: a numeric token creates an array, a non-numeric token
-        creates an object. For example, on an initially `#!json null` value, `/foo/0/0/0` creates nested arrays,
-        while `/foo/one/one/one` creates nested objects. This is not specified by the JSON Pointer RFC; it is
-        this library's own, intentional disambiguation rule. See also [JSON Pointer](../../features/json_pointer.md).
+        pointer token is a valid array index: a token that is a nonempty sequence of digits without a leading
+        `0` (or the token `-`) creates an array, and every other token creates an object. For example, on an
+        initially `#!json null` value, `/foo/0/0/0` creates nested arrays, while `/foo/one/one/one` creates
+        nested objects. Tokens such as `01` or the empty token cannot be array indices (cf. RFC 6901, Sect. 4)
+        and therefore create objects, just as they would if the level already existed as an object. This is not
+        specified by the JSON Pointer RFC; it is this library's own, intentional disambiguation rule. See also
+        [JSON Pointer](../../features/json_pointer.md).
 
 ## Examples
 

include/nlohmann/detail/json_pointer.hpp

@@ -396,15 +396,19 @@ class json_pointer
             // convert null values to arrays or objects before continuing
             if (ptr->is_null())
             {
-                // check if the reference token is a number
-                const bool nums =
-                    std::all_of(reference_token.begin(), reference_token.end(),
-                                [](const unsigned char x)
+                // check if the reference token is a valid array index, that is
+                // a nonempty sequence of digits without a leading '0'
+                // (cf. RFC 6901, Sect. 4); tokens that could never be a valid
+                // array index (such as "01" or "") are treated as object keys
+                const bool nums = !reference_token.empty()
+                                  && (reference_token.size() == 1 || reference_token[0] != '0')
+                                  && std::all_of(reference_token.begin(), reference_token.end(),
+                                                 [](const unsigned char x)
                 {
                     return std::isdigit(x);
                 });
 
-                // change value to an array for numbers or "-" or to object otherwise
+                // change value to an array for array indices or "-" or to object otherwise
                 *ptr = (nums || reference_token == "-")
                        ? detail::value_t::array
                        : detail::value_t::object;

include/nlohmann/detail/output/output_adapters.hpp

@@ -13,7 +13,6 @@
 #include <iterator> // back_inserter
 #include <memory> // shared_ptr, make_shared
 #include <string> // basic_string
-#include <utility> // move
 #include <vector> // vector
 
 #ifndef JSON_NO_IO
@@ -119,72 +118,6 @@ class output_string_adapter : public output_adapter_protocol<CharType>
     StringType& str;
 };
 
-/// @brief non-virtual output sink writing into a std::vector
-///
-/// Unlike output_vector_adapter, this sink is not part of the virtual
-/// output_adapter_protocol hierarchy: it is passed to binary_writer by value as
-/// a template parameter, so write_character()/write_characters() are ordinary
-/// (inlinable) calls with no vtable lookup and no shared_ptr. It is used for the
-/// common `to_cbor`/`to_msgpack`/... into a std::vector.
-template<typename CharType, typename AllocatorType = std::allocator<CharType>>
-class output_vector_sink
-{
-  public:
-    explicit output_vector_sink(std::vector<CharType, AllocatorType>& vec) noexcept
-        : v(vec)
-    {}
-
-    void write_character(CharType c)
-    {
-        v.push_back(c);
-    }
-
-    // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null
-    // pointer with length 0 for empty strings/binary values. Appending an empty
-    // range is a no-op; the type-erased path tolerates this via the (unattributed)
-    // virtual base, and the concrete sink must do the same.
-    void write_characters(const CharType* s, std::size_t length)
-    {
-        v.insert(v.end(), s, s + length);
-    }
-
-  private:
-    std::vector<CharType, AllocatorType>& v;
-};
-
-/// @brief output sink forwarding to a type-erased output adapter
-///
-/// Wraps the polymorphic output_adapter_t so the same binary_writer template can
-/// also target arbitrary adapters (output streams, strings, user-provided
-/// adapters) via the `output_adapter`-based overloads. Each write still goes
-/// through one virtual call, exactly as before; only the concrete sinks above
-/// avoid it.
-template<typename CharType>
-class output_adapter_sink
-{
-  public:
-    explicit output_adapter_sink(output_adapter_t<CharType> adapter)
-        : oa(std::move(adapter))
-    {
-        JSON_ASSERT(oa);
-    }
-
-    void write_character(CharType c)
-    {
-        oa->write_character(c);
-    }
-
-    // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as
-    // the type-erased path already did before this sink existed
-    void write_characters(const CharType* s, std::size_t length)
-    {
-        oa->write_characters(s, length);
-    }
-
-  private:
-    output_adapter_t<CharType> oa = nullptr;
-};
-
 template<typename CharType, typename StringType = std::basic_string<CharType>>
 class output_adapter
 {

include/nlohmann/json.hpp

@@ -140,7 +140,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     friend ::nlohmann::detail::serializer<basic_json>;
     template<typename BasicJsonType>
     friend class ::nlohmann::detail::iter_impl;
-    template<typename BasicJsonType, typename CharType, typename OutputSinkType>
+    template<typename BasicJsonType, typename CharType>
     friend class ::nlohmann::detail::binary_writer;
     template<typename BasicJsonType, typename InputType, typename SAX>
     friend class ::nlohmann::detail::binary_reader;
@@ -4327,9 +4327,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     static std::vector<std::uint8_t> to_cbor(const basic_json& j)
     {
         std::vector<std::uint8_t> result;
-        result.reserve(detail::binary_reserve_hint(j));
-        detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
-            detail::output_vector_sink<std::uint8_t>(result)).write_cbor(j);
+        to_cbor(j, result);
         return result;
     }
 
@@ -4352,9 +4350,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     static std::vector<std::uint8_t> to_msgpack(const basic_json& j)
     {
         std::vector<std::uint8_t> result;
-        result.reserve(detail::binary_reserve_hint(j));
-        detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
-            detail::output_vector_sink<std::uint8_t>(result)).write_msgpack(j);
+        to_msgpack(j, result);
         return result;
     }
 
@@ -4379,9 +4375,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             const bool use_type = false)
     {
         std::vector<std::uint8_t> result;
-        result.reserve(detail::binary_reserve_hint(j));
-        detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
-            detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type);
+        to_ubjson(j, result, use_size, use_type);
         return result;
     }
 
@@ -4409,9 +4403,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             const bjdata_version_t version = bjdata_version_t::draft2)
     {
         std::vector<std::uint8_t> result;
-        result.reserve(detail::binary_reserve_hint(j));
-        detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
-            detail::output_vector_sink<std::uint8_t>(result)).write_ubjson(j, use_size, use_type, true, true, version);
+        to_bjdata(j, result, use_size, use_type, version);
         return result;
     }
 
@@ -4438,9 +4430,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     static std::vector<std::uint8_t> to_bson(const basic_json& j)
     {
         std::vector<std::uint8_t> result;
-        result.reserve(detail::binary_reserve_hint(j));
-        detail::binary_writer<basic_json, std::uint8_t, detail::output_vector_sink<std::uint8_t>>(
-            detail::output_vector_sink<std::uint8_t>(result)).write_bson(j);
+        to_bson(j, result);
         return result;
     }
 

tests/src/unit-json_pointer.cpp

@@ -396,6 +396,72 @@ TEST_CASE("JSON pointers")
         }
     }
 
+    SECTION("creating intermediate levels")
+    {
+        SECTION("tokens that are valid array indices create arrays")
+        {
+            json j;
+            j["/0"_json_pointer] = 1;
+            CHECK(j == json({1}));
+
+            json j2;
+            j2["/2"_json_pointer] = 1;
+            CHECK(j2 == json({nullptr, nullptr, 1}));
+
+            json j3;
+            j3["/-"_json_pointer] = 1;
+            CHECK(j3 == json({1}));
+
+            json j4;
+            j4["/foo/0/0"_json_pointer] = 1;
+            CHECK(j4 == json({{"foo", {{1}}}}));
+        }
+
+        SECTION("tokens that are no valid array indices create objects")
+        {
+            json j;
+            j["/one"_json_pointer] = 1;
+            CHECK(j == json({{"one", 1}}));
+
+            // leading '0' can never be a valid array index (RFC 6901, Sect. 4)
+            json j2;
+            j2["/01"_json_pointer] = 1;
+            CHECK(j2 == json({{"01", 1}}));
+
+            // the empty token is a valid object key, but no valid array index
+            json j3;
+            j3["/"_json_pointer] = 1;
+            CHECK(j3 == json({{"", 1}}));
+        }
+
+        SECTION("creating a level yields the same result as reusing it (#5357)")
+        {
+            json j;
+            j["/a/b/01/d"_json_pointer] = "value";
+
+            json j_init = json::object();
+            j_init["/a/b"_json_pointer] = json::object();
+            j_init["/a/b/01/d"_json_pointer] = "value";
+
+            const json expected = json::parse(R"({"a":{"b":{"01":{"d":"value"}}}})");
+            CHECK(j == expected);
+            CHECK(j_init == expected);
+
+            // unflatten uses the same key
+            const json flat = {{"/a/b/01/d", "value"}};
+            CHECK(flat.unflatten() == expected);
+        }
+
+        SECTION("existing arrays still reject invalid indices")
+        {
+            json j = {1, 2, 3};
+            CHECK_THROWS_WITH_AS(j["/01"_json_pointer],
+                                 "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&);
+            CHECK_THROWS_WITH_AS(j.at("/01"_json_pointer),
+                                 "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", json::parse_error&);
+        }
+    }
+
     SECTION("flatten")
     {
         json j =