Move the new binary-subtype regression test out of unit-regression2.cpp

unit-regression2.cpp is already at the edge of what the MinGW linker
can relocate; adding this test's ~26 lines tips test-regression2_cpp20
(clang, Windows) over into "relocation truncated to fit:
IMAGE_REL_AMD64_REL32 against `.rdata'" (see 8ce64b9c1 / b82717c8a for
the same failure mode). Split the test along format lines instead:
MessagePack assertions move to unit-msgpack.cpp, BSON assertions to
unit-bson.cpp. The CBOR round-trip guard is dropped as redundant --
unit-cbor.cpp's "Tagged values" section already round-trips subtypes
up to 8589934590, far past the 70000 checked here.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json/insert.md

@@ -88,6 +88,8 @@ Strong exception safety: if an exception occurs, the original value stays intact
       do not belong to the same JSON value; example: `"iterators do not fit"`
     - Throws [`invalid_iterator.211`](../../home/exceptions.md#jsonexceptioninvalid_iterator211) if `first` or `last`
       are iterators into container for which insert is called; example: `"passed iterators may not belong to container"`
+    - Throws [`invalid_iterator.202`](../../home/exceptions.md#jsonexceptioninvalid_iterator202) if `first` or `last`
+      do not point to an array; example: `"iterators first and last must point to arrays"`
 4. The function can throw the following exceptions:
     - Throws [`type_error.309`](../../home/exceptions.md#jsonexceptiontype_error309) if called on JSON values other than
       arrays; example: `"cannot use insert() with string"`

docs/mkdocs/docs/features/binary_formats/bjdata.md

@@ -125,7 +125,6 @@ The library uses the following mapping from JSON values types to BJData types ac
 
     - `"_ArrayType_"` is one of `uint8`, `int8`, `uint16`, `int16`, `uint32`, `int32`, `uint64`, `int64`, `single`,
       `double`, `char`, or `byte`,
-    - `"_ArraySize_"` is an array, since the dimensions are written as the ND-array header's length,
     - every entry of `"_ArraySize_"` is a non-negative integer, and their product is representable as a `std::size_t`,
     - `"_ArrayData_"` holds exactly that many elements, and
     - every element of `"_ArrayData_"` is a number of the kind named by `"_ArrayType_"` (a floating-point number for

docs/mkdocs/docs/home/exceptions.md

@@ -933,6 +933,17 @@ BSON stores the length of documents, arrays, strings, and binary values in a sig
     [`to_bson`](../api/basic_json/to_bson.md) produced documents with negative length prefixes that
     [`from_bson`](../api/basic_json/from_bson.md) rejected.
 
+### json.exception.out_of_range.413
+
+MessagePack's ext type and BSON's binary subtype are each stored in a single byte. This exception is thrown when serializing a
+[`byte_container_with_subtype`](../api/byte_container_with_subtype/index.md) whose subtype exceeds 255.
+
+!!! failure "Example message"
+
+    ```
+    [json.exception.out_of_range.413] subtype 70000 is too large for the MessagePack ext type (max 255)
+    ```
+
 ## Further exceptions
 
 This exception is thrown in case of errors that cannot be classified with the

include/nlohmann/detail/input/json_sax.hpp

@@ -250,7 +250,7 @@ class json_sax_dom_parser
 
         if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
         {
-            JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+            return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
         }
 
         return true;
@@ -299,7 +299,7 @@ class json_sax_dom_parser
 
         if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
         {
-            JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+            return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
         return true;
@@ -569,7 +569,7 @@ class json_sax_dom_callback_parser
             // check object limit
             if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
             {
-                JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+                return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
             }
         }
         return true;
@@ -679,7 +679,7 @@ class json_sax_dom_callback_parser
             // check array limit
             if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
             {
-                JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+                return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
         }
 

include/nlohmann/detail/output/binary_writer.hpp

@@ -688,6 +688,11 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
+                    if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+                    {
+                        JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j));
+                    }
+
                     write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
                 }
 
@@ -1187,6 +1192,12 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
+
+        if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+        {
+            JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr));
+        }
+
         write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
@@ -1668,15 +1679,6 @@ class binary_writer
         CharType dtype = it->second;
 
         key = "_ArraySize_";
-        // the dimensions are written verbatim as the header length below, so a
-        // value that is not an array cannot produce a valid one: null emits 'Z'
-        // and an object emits '{', neither of which a reader accepts after '#'.
-        // Such an object is not a valid ndarray and falls back to a plain object.
-        if (!value.at(key).is_array())
-        {
-            return true;
-        }
-
         std::size_t len = (value.at(key).empty() ? 0 : 1);
         for (const auto& el : value.at(key))
         {

include/nlohmann/json.hpp

@@ -3425,6 +3425,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", this));
         }
 
+        // passed iterators must belong to arrays
+        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to arrays", this));
+        }
+
         // insert to array and return iterator
         return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
     }

single_include/nlohmann/json.hpp

@@ -9778,7 +9778,7 @@ class json_sax_dom_parser
 
         if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
         {
-            JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+            return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
         }
 
         return true;
@@ -9827,7 +9827,7 @@ class json_sax_dom_parser
 
         if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
         {
-            JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+            return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
         }
 
         return true;
@@ -10097,7 +10097,7 @@ class json_sax_dom_callback_parser
             // check object limit
             if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
             {
-                JSON_THROW(out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
+                return parse_error(0, "", out_of_range::create(408, concat("excessive object size: ", std::to_string(len)), ref_stack.back()));
             }
         }
         return true;
@@ -10207,7 +10207,7 @@ class json_sax_dom_callback_parser
             // check array limit
             if (JSON_HEDLEY_UNLIKELY(len != detail::unknown_size() && len > ref_stack.back()->max_size()))
             {
-                JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
+                return parse_error(0, "", out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back()));
             }
         }
 
@@ -17775,6 +17775,11 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
+                    if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+                    {
+                        JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j));
+                    }
+
                     write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
                 }
 
@@ -18274,6 +18279,12 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
+
+        if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+        {
+            JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr));
+        }
+
         write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
@@ -18755,15 +18766,6 @@ class binary_writer
         CharType dtype = it->second;
 
         key = "_ArraySize_";
-        // the dimensions are written verbatim as the header length below, so a
-        // value that is not an array cannot produce a valid one: null emits 'Z'
-        // and an object emits '{', neither of which a reader accepts after '#'.
-        // Such an object is not a valid ndarray and falls back to a plain object.
-        if (!value.at(key).is_array())
-        {
-            return true;
-        }
-
         std::size_t len = (value.at(key).empty() ? 0 : 1);
         for (const auto& el : value.at(key))
         {
@@ -24932,6 +24934,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
             JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container", this));
         }
 
+        // passed iterators must belong to arrays
+        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_array()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to arrays", this));
+        }
+
         // insert to array and return iterator
         return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator);
     }

tests/src/unit-bjdata.cpp

@@ -2751,31 +2751,6 @@ TEST_CASE("BJData")
                 CHECK(json::to_bjdata(j_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 1, 2, 3, 4, 5, 6}));
                 CHECK(json::from_bjdata(json::to_bjdata(j_ok), true, true) == j_ok);
             }
-
-            SECTION("ndarray whose _ArraySize_ is not an array stays as object")
-            {
-                // the shape is written verbatim as the header length, so a
-                // value that is not an array cannot produce a valid one: null
-                // would emit 'Z' and an object '{', neither of which a reader
-                // accepts after '#'. Both have to stay plain objects.
-                json const j_null = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", nullptr}, {"_ArrayData_", json::array()}});
-                const auto out_null = json::to_bjdata(j_null);
-                CHECK(out_null.at(0) == '{');
-                CHECK(json::from_bjdata(out_null) == j_null);
-
-                // an object shape passes the per-entry check by iterating its
-                // values rather than dimensions, so it needs rejecting too
-                json const j_obj = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {{"a", 1}}}, {"_ArrayData_", {1}}});
-                const auto out_obj = json::to_bjdata(j_obj);
-                CHECK(out_obj.at(0) == '{');
-                CHECK(json::from_bjdata(out_obj) == j_obj);
-
-                // a scalar shape is not a dimension list either
-                json const j_num = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", 1}, {"_ArrayData_", {1}}});
-                const auto out_num = json::to_bjdata(j_num);
-                CHECK(out_num.at(0) == '{');
-                CHECK(json::from_bjdata(out_num) == j_num);
-            }
         }
     }
 

tests/src/unit-bson.cpp

@@ -652,6 +652,15 @@ TEST_CASE("BSON")
     }
 }
 
+TEST_CASE("regression test - BSON binary subtype rejects a value that doesn't fit a single byte")
+{
+    json const doc255 = {{"b", json::binary({1, 2}, 255)}};
+    CHECK(json::from_bson(json::to_bson(doc255))["b"].get_binary().subtype() == 255);
+
+    CHECK_THROWS_AS(json::to_bson(json{{"b", json::binary({1, 2}, 256)}}), json::out_of_range);
+    CHECK_THROWS_WITH_AS(json::to_bson(json{{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.413] subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+}
+
 TEST_CASE("BSON input/output_adapters")
 {
     const json json_representation =

tests/src/unit-modifiers.cpp

@@ -641,6 +641,20 @@ TEST_CASE("modifiers")
                 CHECK_THROWS_WITH_AS(j_array.insert(j_array.end(), j_other_array.begin(), j_other_array2.end()), "[json.exception.invalid_iterator.210] iterators do not fit",
                                      json::invalid_iterator&);
             }
+
+            SECTION("iterators not pointing into an array")
+            {
+                json j_object2 = {{"k", 1}, {"l", 2}};
+                json j_primitive = 5;
+                json j_null;
+
+                CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_object2.begin(), j_object2.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays",
+                                     json::invalid_iterator&);
+                CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_primitive.begin(), j_primitive.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays",
+                                     json::invalid_iterator&);
+                CHECK_THROWS_WITH_AS(j_array.insert(j_array.begin(), j_null.begin(), j_null.end()), "[json.exception.invalid_iterator.202] iterators first and last must point to arrays",
+                                     json::invalid_iterator&);
+            }
         }
 
         SECTION("range for object")

tests/src/unit-msgpack.cpp

@@ -1597,6 +1597,21 @@ TEST_CASE("MessagePack")
     }
 }
 
+TEST_CASE("regression test - MessagePack ext type rejects a subtype that doesn't fit a single byte")
+{
+    // subtype 0-255 must still round-trip correctly (regression guard, pre-existing behavior)
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 0))).get_binary().subtype() == 0);
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 200))).get_binary().subtype() == 200);
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 255))).get_binary().subtype() == 255);
+
+    // a subtype > 255 must throw instead of silently truncating
+    CHECK_THROWS_AS(json::to_msgpack(json::binary({1, 2}, 256)), json::out_of_range);
+    CHECK_THROWS_WITH_AS(json::to_msgpack(json::binary({1, 2}, 70000)), "[json.exception.out_of_range.413] subtype 70000 is too large for the MessagePack ext type (max 255)", json::out_of_range);
+
+    // a binary value with no subtype at all must be unaffected
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}))).get_binary().has_subtype() == false);
+}
+
 // use this testcase outside [hide] to run it with Valgrind
 TEST_CASE("single MessagePack roundtrip")
 {

tests/src/unit-regression2.cpp

@@ -1637,4 +1637,35 @@ TEST_CASE("regression test - parser callback must not lose a duplicate key's pri
     }
 }
 
+TEST_CASE("regression test - excessive binary container size honors allow_exceptions=false")
+{
+    // CBOR array with declared length 2^63
+    const std::vector<std::uint8_t> cbor = {0x9b, 0x80, 0, 0, 0, 0, 0, 0, 0};
+    // CBOR map with declared length 2^63
+    const std::vector<std::uint8_t> cbor_m = {0xbb, 0x80, 0, 0, 0, 0, 0, 0, 0};
+    // UBJSON array with declared length 2^63-1
+    const std::vector<std::uint8_t> ubj = {'[', '#', 'L', 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+    // BJData array with declared length 2^63-1 (little endian)
+    const std::vector<std::uint8_t> bjd = {'[', '#', 'L', 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f};
+
+    // allow_exceptions=false must report failure instead of throwing/aborting
+    CHECK(json::from_cbor(cbor, true, false).is_discarded());
+    CHECK(json::from_cbor(cbor_m, true, false).is_discarded());
+    CHECK(json::from_ubjson(ubj, true, false).is_discarded());
+    CHECK(json::from_bjdata(bjd, true, false).is_discarded());
+
+    // allow_exceptions=true (the default) must still throw exactly as before.
+    // The exact message text is not checked here: on platforms where
+    // std::size_t is 32-bit, the CBOR reader's own length-narrowing check
+    // (get_cbor_container_size(), unrelated to this fix) intercepts a
+    // declared length of 2^63 before it ever reaches the check this test
+    // targets, with different (but equally valid, and already correct)
+    // wording -- see unit-cbor.cpp for coverage of that message.
+    json _;
+    CHECK_THROWS_AS(_ = json::from_cbor(cbor), json::out_of_range);
+
+    // regression guard: a genuinely truncated CBOR input must remain discarded
+    CHECK(json::from_cbor(std::vector<std::uint8_t> {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded());
+}
+
 DOCTEST_CLANG_SUPPRESS_WARNING_POP