Add deprecated from_bon8/from_bjdata(ptr, len) overloads

from_bon8(ptr, len) and from_bjdata(ptr, len) had no overload for a
pointer and a length, unlike from_cbor/from_msgpack/from_ubjson/
from_bson. The call instead bound to from_*(InputType&&, bool strict),
which read ptr as a NUL-terminated C string via strlen and silently
converted len to the strict flag. Data containing a 0x00 byte was cut
off there; data without one was read past the end of the buffer.

Add a deprecated (ptr, len, strict, allow_exceptions) overload for
each function that forwards to (ptr, ptr + len, ...), matching the
existing deprecated overloads of the other four binary readers. Since
neither function ever had this overload, the deprecation is declared
as of version 3.13.0, the next unreleased version, rather than the
version each function was originally added in.

Fixes #5648.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/from_bjdata.md

@@ -111,3 +111,12 @@ Linear in the size of the input.
 - Added in version 3.11.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+
+!!! warning "Deprecation"
+
+    - Overload (2) replaces calls to `from_bjdata` with a pointer and a length as first two parameters, which has been
+      deprecated in version 3.13.0. This overload will be removed in version 4.0.0. Please replace all calls like
+      `#!cpp from_bjdata(ptr, len, ...);` with `#!cpp from_bjdata(ptr, ptr+len, ...);`.
+
+    You should be warned by your compiler with a `-Wdeprecated-declarations` warning if you are using a deprecated
+    function.

docs/mkdocs/docs/api/basic_json/from_bon8.md

@@ -106,3 +106,12 @@ Linear in the size of the input.
 ## Version history
 
 - Added in version 3.13.0.
+
+!!! warning "Deprecation"
+
+    - Overload (2) replaces calls to `from_bon8` with a pointer and a length as first two parameters, which has been
+      deprecated in version 3.13.0. This overload will be removed in version 4.0.0. Please replace all calls like
+      `#!cpp from_bon8(ptr, len, ...);` with `#!cpp from_bon8(ptr, ptr+len, ...);`.
+
+    You should be warned by your compiler with a `-Wdeprecated-declarations` warning if you are using a deprecated
+    function.

docs/mkdocs/docs/api/basic_json/to_msgpack.md

@@ -76,6 +76,3 @@ Linear in the size of the JSON value `j`.
 
 - Added in version 2.0.9.
 - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0.
-- Fixed in version 3.13.0 to serialize `number_integer_t`/`number_unsigned_t` pairs of different width correctly;
-  before, integers could be serialized with the wrong value if `number_integer_t` was narrower than
-  `number_unsigned_t`.

include/nlohmann/detail/output/binary_writer.hpp

@@ -337,24 +337,24 @@ class binary_writer
                     // MessagePack does not differentiate between positive
                     // signed integers and unsigned integers. Therefore, we used
                     // the code from the value_t::number_unsigned case here.
-                    if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) < 128)
+                    if (j.m_data.m_value.number_unsigned < 128)
                     {
                         // positive fixnum
                         write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint8_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
                     {
                         // uint 8
                         oa.write_character(to_char_type(0xCC));
                         write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint16_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
                     {
                         // uint 16
                         oa.write_character(to_char_type(0xCD));
                         write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint32_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
                     {
                         // uint 32
                         oa.write_character(to_char_type(0xCE));
@@ -410,31 +410,31 @@ class binary_writer
                 if (j.m_data.m_value.number_unsigned < 128)
                 {
                     // positive fixnum
-                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
                 {
                     // uint 8
                     oa.write_character(to_char_type(0xCC));
-                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
                 {
                     // uint 16
                     oa.write_character(to_char_type(0xCD));
-                    write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
                 {
                     // uint 32
                     oa.write_character(to_char_type(0xCE));
-                    write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));
                 }
                 else
                 {
                     // uint 64
                     oa.write_character(to_char_type(0xCF));
-                    write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
                 }
                 break;
             }

include/nlohmann/json.hpp

@@ -5547,6 +5547,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
+    template<typename T>
+    JSON_HEDLEY_WARN_UNUSED_RESULT
+    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bjdata(ptr, ptr + len))
+    static basic_json from_bjdata(const T* ptr, std::size_t len,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        return from_bjdata(ptr, ptr + len, strict, allow_exceptions);
+    }
+
     /// @brief create a JSON value from an input in BON8 format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bon8/
     template<typename InputType>
@@ -5584,6 +5594,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
+    template<typename T>
+    JSON_HEDLEY_WARN_UNUSED_RESULT
+    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bon8(ptr, ptr + len))
+    static basic_json from_bon8(const T* ptr, std::size_t len,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        return from_bon8(ptr, ptr + len, strict, allow_exceptions);
+    }
+
     /// @brief create a JSON value from an input in BSON format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
     template<typename InputType>

single_include/nlohmann/json.hpp

@@ -20667,24 +20667,24 @@ class binary_writer
                     // MessagePack does not differentiate between positive
                     // signed integers and unsigned integers. Therefore, we used
                     // the code from the value_t::number_unsigned case here.
-                    if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) < 128)
+                    if (j.m_data.m_value.number_unsigned < 128)
                     {
                         // positive fixnum
                         write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint8_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
                     {
                         // uint 8
                         oa.write_character(to_char_type(0xCC));
                         write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint16_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
                     {
                         // uint 16
                         oa.write_character(to_char_type(0xCD));
                         write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
                     }
-                    else if (static_cast<typename BasicJsonType::number_unsigned_t>(j.m_data.m_value.number_integer) <= (std::numeric_limits<std::uint32_t>::max)())
+                    else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
                     {
                         // uint 32
                         oa.write_character(to_char_type(0xCE));
@@ -20740,31 +20740,31 @@ class binary_writer
                 if (j.m_data.m_value.number_unsigned < 128)
                 {
                     // positive fixnum
-                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint8_t>::max)())
                 {
                     // uint 8
                     oa.write_character(to_char_type(0xCC));
-                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint8_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint16_t>::max)())
                 {
                     // uint 16
                     oa.write_character(to_char_type(0xCD));
-                    write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint16_t>(j.m_data.m_value.number_integer));
                 }
                 else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits<std::uint32_t>::max)())
                 {
                     // uint 32
                     oa.write_character(to_char_type(0xCE));
-                    write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint32_t>(j.m_data.m_value.number_integer));
                 }
                 else
                 {
                     // uint 64
                     oa.write_character(to_char_type(0xCF));
-                    write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_unsigned));
+                    write_number(static_cast<std::uint64_t>(j.m_data.m_value.number_integer));
                 }
                 break;
             }
@@ -31628,6 +31628,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
+    template<typename T>
+    JSON_HEDLEY_WARN_UNUSED_RESULT
+    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bjdata(ptr, ptr + len))
+    static basic_json from_bjdata(const T* ptr, std::size_t len,
+                                  const bool strict = true,
+                                  const bool allow_exceptions = true)
+    {
+        return from_bjdata(ptr, ptr + len, strict, allow_exceptions);
+    }
+
     /// @brief create a JSON value from an input in BON8 format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bon8/
     template<typename InputType>
@@ -31665,6 +31675,16 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         return result;
     }
 
+    template<typename T>
+    JSON_HEDLEY_WARN_UNUSED_RESULT
+    JSON_HEDLEY_DEPRECATED_FOR(3.13.0, from_bon8(ptr, ptr + len))
+    static basic_json from_bon8(const T* ptr, std::size_t len,
+                                const bool strict = true,
+                                const bool allow_exceptions = true)
+    {
+        return from_bon8(ptr, ptr + len, strict, allow_exceptions);
+    }
+
     /// @brief create a JSON value from an input in BSON format
     /// @sa https://json.nlohmann.me/api/basic_json/from_bson/
     template<typename InputType>

tests/src/unit-bjdata.cpp

@@ -4572,3 +4572,31 @@ TEST_CASE("BJData roundtrips" * doctest::skip())
         }
     }
 }
+
+TEST_CASE("issue #5648 - from_bjdata(ptr, len) must read len bytes, not treat ptr as a C string")
+{
+    // to_bjdata() encodes the integer 0 as the two bytes 'i' 0x00 (a BJData
+    // type marker followed by the value byte 0x00), so the packed data
+    // below contains a 0x00 byte before its end.
+    const json j = {{"a", 0}};
+    const std::vector<std::uint8_t> packed = json::to_bjdata(j);
+    bool contains_nul = false;
+    for (const auto byte : packed)
+    {
+        contains_nul |= (byte == 0x00);
+    }
+    REQUIRE(contains_nul);
+
+    // before the fix, from_bjdata had no (ptr, len) overload, so this call
+    // bound to from_bjdata(InputType&&, bool strict) instead: ptr was read
+    // as a NUL-terminated C string (stopping at the embedded 0x00 byte), and
+    // len was silently converted to the strict flag. The deprecated
+    // overload added for this issue forwards to from_bjdata(ptr, ptr + len,
+    // ...) instead, like from_ubjson's deprecated (ptr, len) overload does.
+    json result;
+    CHECK_NOTHROW(result = json::from_bjdata(packed.data(), packed.size()));
+    CHECK(result == j);
+
+    // len must not collapse into the strict flag either
+    CHECK(json::from_bjdata(packed.data(), packed.size(), false) == j);
+}

tests/src/unit-bon8.cpp

@@ -1010,6 +1010,34 @@ TEST_CASE("BON8 roundtrips" * doctest::skip())
     }
 }
 
+TEST_CASE("issue #5648 - from_bon8(ptr, len) must read len bytes, not treat ptr as a C string")
+{
+    // to_bon8() encodes the integer 40 as the two bytes 0xC2 0x00 (a BON8
+    // lead byte followed by a continuation byte of 0x00), so the packed data
+    // below contains a 0x00 byte before its end.
+    const json j = {{"a", 40}, {"b", "x"}};
+    const std::vector<std::uint8_t> packed = json::to_bon8(j);
+    bool contains_nul = false;
+    for (const auto byte : packed)
+    {
+        contains_nul |= (byte == 0x00);
+    }
+    REQUIRE(contains_nul);
+
+    // before the fix, from_bon8 had no (ptr, len) overload, so this call
+    // bound to from_bon8(InputType&&, bool strict) instead: ptr was read as
+    // a NUL-terminated C string (stopping at the embedded 0x00 byte), and
+    // len was silently converted to the strict flag. The deprecated
+    // overload added for this issue forwards to from_bon8(ptr, ptr + len,
+    // ...) instead, like from_cbor's deprecated (ptr, len) overload does.
+    json result;
+    CHECK_NOTHROW(result = json::from_bon8(packed.data(), packed.size()));
+    CHECK(result == j);
+
+    // len must not collapse into the strict flag either
+    CHECK(json::from_bon8(packed.data(), packed.size(), false) == j);
+}
+
 #ifdef JSON_HAS_CPP_17
 TEST_CASE("BON8 with std::byte")
 {

tests/src/unit-msgpack.cpp

@@ -2475,60 +2475,3 @@ TEST_CASE("MessagePack lengths beyond UINT32_MAX cannot be serialized")
     }
 #endif
 }
-
-TEST_CASE("MessagePack numbers use the active union member (see #5644)")
-{
-    // when number_integer_t is narrower than number_unsigned_t, to_msgpack()
-    // used to read the union member that was not the active one, writing
-    // wrong bytes for some values; std::int64_t/std::uint64_t (the default
-    // types, where both members have the same width) were not affected
-    using int32_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int32_t, std::uint64_t, double>;
-    using int16_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int16_t, std::uint64_t, double>;
-
-    SECTION("number_integer_t = std::int32_t")
-    {
-        SECTION("6442450944 (uint 64; the low 32 bits used to be sign-extended)")
-        {
-            const int32_json j = 6442450944ULL;
-            CHECK(j.is_number_unsigned());
-
-            std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00};
-            const auto result = int32_json::to_msgpack(j);
-            CHECK(result == expected);
-            CHECK(int32_json::from_msgpack(result) == j);
-        }
-
-        SECTION("4294967496 (uint 64; the low 32 bits used to be the whole value)")
-        {
-            const int32_json j = 4294967496ULL;
-            CHECK(j.is_number_unsigned());
-
-            std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
-            const auto result = int32_json::to_msgpack(j);
-            CHECK(result == expected);
-            CHECK(int32_json::from_msgpack(result) == j);
-        }
-    }
-
-    SECTION("number_integer_t = std::int16_t, 98304 (uint 32)")
-    {
-        const int16_json j = 98304ULL;
-        CHECK(j.is_number_unsigned());
-
-        std::vector<uint8_t> const expected{0xce, 0x00, 0x01, 0x80, 0x00};
-        const auto result = int16_json::to_msgpack(j);
-        CHECK(result == expected);
-        CHECK(int16_json::from_msgpack(result) == j);
-    }
-
-    SECTION("default types (std::int64_t/std::uint64_t) are unaffected")
-    {
-        const json j = 4294967496ULL;
-        CHECK(j.is_number_unsigned());
-
-        std::vector<uint8_t> const expected{0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xc8};
-        const auto result = json::to_msgpack(j);
-        CHECK(result == expected);
-        CHECK(json::from_msgpack(result) == j);
-    }
-}