Throw type_error.321 for discarded values in to_bon8

write_bon8_value silently skipped discarded values, but the array/object
count marker still counted them, so [1, discarded] produced 82 91: a marker
announcing two elements followed by one. Throw type_error.321 like the other
binary writers do, at any nesting level.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/docset/generate_docset.py

@@ -337,17 +337,14 @@ def is_remote(url) -> bool:
 def download(url, docs) -> str:
     """Download url into assets/external and return the path relative to docs."""
     u = urllib.parse.urlparse(url if not url.startswith('//') else 'https:' + url)
-    if u.scheme.lower() not in ('http', 'https'):
-        raise ValueError(f'not an http(s) URL: {url}')
     req = urllib.request.Request(u.geturl(), headers={'User-Agent': USER_AGENT})
-    # (the scheme is checked above)
-    with urllib.request.urlopen(req, timeout=20) as r:  # nosec B310
+    with urllib.request.urlopen(req, timeout=20) as r:
         data = r.read()
         ctype = r.headers.get_content_type()
     path = urllib.parse.unquote(u.path).lstrip('/')
     ext = os.path.splitext(path)[1]
     if u.query or not ext or path.endswith('/'):
-        digest = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12]
+        digest = hashlib.sha1(url.encode()).hexdigest()[:12]
         path = os.path.join(os.path.dirname(path), digest + CONTENT_TYPE_EXT.get(ctype, ext or '.bin'))
     rel = os.path.normpath(os.path.join('assets', 'external', u.hostname, path))
     out = os.path.join(docs, rel)
@@ -388,8 +385,7 @@ def localize_images(docs) -> None:
 def load_mkdocs_yml() -> dict:
     """Load mkdocs.yml, ignoring tags like !ENV and !!python/name."""
     with open(MKDOCS_YML, encoding='utf-8') as f:
-        # (Loader is a yaml.SafeLoader)
-        return yaml.load(f, Loader=Loader)  # nosec B506
+        return yaml.load(f, Loader=Loader)
 
 
 def localize_site_urls(docs, site_url) -> None:

docs/mkdocs/docs/api/basic_json/to_bon8.md

@@ -41,6 +41,8 @@ With (2), the bytes written before the exception remain in the output adapter.
   above 9223372036854775807, which BON8 cannot represent
 - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if `j` contains a string that is not
   valid UTF-8
+- Throws [type_error.321](../../home/exceptions.md#jsonexceptiontype_error321) if `j` or a value nested in it is
+  discarded; example: `"cannot serialize discarded value to BON8"`
 
 ## Complexity
 

docs/mkdocs/docs/home/exceptions.md

@@ -810,7 +810,7 @@ does not list an enumerator and it is therefore converted like the first listed
 A discarded value (one created by [`parse()`](../api/basic_json/parse.md) with a callback that returns `false` for the
 value, or by default-constructing a [`basic_json`](../api/basic_json/index.md) with
 [`value_t::discarded`](../api/basic_json/value_t.md)) was passed to a binary serialization function, either directly or
-nested in an array or object. There is no way to represent a discarded value in CBOR, MessagePack, UBJSON, BJData, or BSON.
+nested in an array or object. There is no way to represent a discarded value in CBOR, MessagePack, UBJSON, BJData, BSON, or BON8.
 
 !!! failure "Example message"
 

include/nlohmann/detail/output/binary_writer.hpp

@@ -809,6 +809,8 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
+
+    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8(const BasicJsonType& j)
     {
@@ -878,9 +880,9 @@ class binary_writer
         }
     }
 
-    /// @brief a CBOR, MessagePack, or BON8 array or object whose elements
-    ///        @ref write_cbor_iterative, @ref write_msgpack_iterative, or
-    ///        @ref write_bon8_iterative is still writing
+    /// @brief a CBOR or MessagePack array or object whose elements
+    ///        @ref write_cbor_iterative or @ref write_msgpack_iterative is
+    ///        still writing
     struct binary_container_frame
     {
         explicit binary_container_frame(const BasicJsonType* value_) noexcept
@@ -2570,26 +2572,11 @@ class binary_writer
     @param[in] j                JSON value to serialize
     @param[in,out] string_open  whether the output ends with a non-empty
                                 string that has not been terminated with 0xFF
-    @param[in] depth            nesting level of @a j, counted from the
-                                top-level value passed to @ref basic_json::to_bon8
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
-    @throw out_of_range.407 if an unsigned integer does not fit int64
 
-    Values nested deeper than @ref recursion_depth_limit are written by
-    @ref write_bon8_iterative without the call stack.
-
-    @sa @ref write_cbor
-    @sa https://github.com/nlohmann/json/issues/5392
+    @throw type_error.321 if @a j or a value nested in it is discarded
     */
-    void write_bon8_value(const BasicJsonType& j, bool& string_open, const std::size_t depth = 0)
+    void write_bon8_value(const BasicJsonType& j, bool& string_open)
     {
-        if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit()) && (j.is_array() || j.is_object()))
-        {
-            write_bon8_iterative(j, string_open);
-            return;
-        }
-
         switch (j.type())
         {
             case value_t::null:
@@ -2643,7 +2630,7 @@ class binary_writer
 
                 for (const auto& el : *j.m_data.m_value.array)
                 {
-                    write_bon8_value(el, string_open, depth + 1);
+                    write_bon8_value(el, string_open);
                 }
 
                 if (N > 4)
@@ -2662,7 +2649,7 @@ class binary_writer
                 for (const auto& el : *j.m_data.m_value.object)
                 {
                     write_bon8_string(el.first, string_open, j);
-                    write_bon8_value(el.second, string_open, depth + 1);
+                    write_bon8_value(el.second, string_open);
                 }
 
                 if (N > 4)
@@ -2695,84 +2682,7 @@ class binary_writer
 
             case value_t::discarded:
             default:
-                break;
-        }
-    }
-
-    /*!
-    @brief write @a j with @ref write_bon8_value, or write its marker and push
-           a frame for @ref write_bon8_iterative to continue with its elements
-
-    A scalar, and an empty array or object, are written out in full and not
-    pushed.
-
-    @sa @ref write_cbor_value_or_push
-    */
-    void write_bon8_value_or_push(const BasicJsonType& j, bool& string_open, std::vector<binary_container_frame>& stack)
-    {
-        if (j.is_array() || j.is_object())
-        {
-            // arrays use the markers 0x80..0x85, objects 0x86..0x8B; the last
-            // one stands for more than four elements, closed later with 0xFE
-            const auto N = j.size();
-            const std::size_t base = j.is_array() ? 0x80 : 0x86;
-            write_bon8_marker(static_cast<std::uint8_t>(base + (N <= 4 ? N : 5)), string_open);
-            if (N != 0)
-            {
-                stack.emplace_back(&j);
-            }
-            return;
-        }
-
-        write_bon8_value(j, string_open);
-    }
-
-    /*!
-    @brief write out @a root and everything below it without the call stack
-
-    A container with more than four elements is closed with 0xFE once its
-    last element is written, as in @ref write_bon8_value.
-
-    @sa @ref write_cbor_iterative
-    */
-    void write_bon8_iterative(const BasicJsonType& root, bool& string_open)
-    {
-        // only a container with elements is ever pushed; see write_bon8_value_or_push
-        std::vector<binary_container_frame> stack;
-        write_bon8_value_or_push(root, string_open, stack);
-
-        while (!stack.empty())
-        {
-            const binary_container_frame current = stack.back();
-            const bool is_array = current.value->is_array();
-            const bool at_end = is_array
-                                ? current.array_it == current.value->m_data.m_value.array->cend()
-                                : current.object_it == current.value->m_data.m_value.object->cend();
-
-            if (at_end)
-            {
-                if (current.value->size() > 4)
-                {
-                    write_bon8_marker(0xFE, string_open);
-                }
-                stack.pop_back();
-                continue;
-            }
-
-            // read the child before pushing: entering it can move every frame
-            const BasicJsonType* child = nullptr;
-            if (is_array)
-            {
-                child = &(*current.array_it);
-                ++stack.back().array_it;
-            }
-            else
-            {
-                write_bon8_string(current.object_it->first, string_open, *current.value);
-                child = &(current.object_it->second);
-                ++stack.back().object_it;
-            }
-            write_bon8_value_or_push(*child, string_open, stack);
+                throw_on_discarded(j, "BON8");
         }
     }
 

single_include/nlohmann/json.hpp

@@ -22426,6 +22426,8 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
+
+    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8(const BasicJsonType& j)
     {
@@ -22495,9 +22497,9 @@ class binary_writer
         }
     }
 
-    /// @brief a CBOR, MessagePack, or BON8 array or object whose elements
-    ///        @ref write_cbor_iterative, @ref write_msgpack_iterative, or
-    ///        @ref write_bon8_iterative is still writing
+    /// @brief a CBOR or MessagePack array or object whose elements
+    ///        @ref write_cbor_iterative or @ref write_msgpack_iterative is
+    ///        still writing
     struct binary_container_frame
     {
         explicit binary_container_frame(const BasicJsonType* value_) noexcept
@@ -24187,26 +24189,11 @@ class binary_writer
     @param[in] j                JSON value to serialize
     @param[in,out] string_open  whether the output ends with a non-empty
                                 string that has not been terminated with 0xFF
-    @param[in] depth            nesting level of @a j, counted from the
-                                top-level value passed to @ref basic_json::to_bon8
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
-    @throw out_of_range.407 if an unsigned integer does not fit int64
 
-    Values nested deeper than @ref recursion_depth_limit are written by
-    @ref write_bon8_iterative without the call stack.
-
-    @sa @ref write_cbor
-    @sa https://github.com/nlohmann/json/issues/5392
+    @throw type_error.321 if @a j or a value nested in it is discarded
     */
-    void write_bon8_value(const BasicJsonType& j, bool& string_open, const std::size_t depth = 0)
+    void write_bon8_value(const BasicJsonType& j, bool& string_open)
     {
-        if (JSON_HEDLEY_UNLIKELY(depth >= recursion_depth_limit()) && (j.is_array() || j.is_object()))
-        {
-            write_bon8_iterative(j, string_open);
-            return;
-        }
-
         switch (j.type())
         {
             case value_t::null:
@@ -24260,7 +24247,7 @@ class binary_writer
 
                 for (const auto& el : *j.m_data.m_value.array)
                 {
-                    write_bon8_value(el, string_open, depth + 1);
+                    write_bon8_value(el, string_open);
                 }
 
                 if (N > 4)
@@ -24279,7 +24266,7 @@ class binary_writer
                 for (const auto& el : *j.m_data.m_value.object)
                 {
                     write_bon8_string(el.first, string_open, j);
-                    write_bon8_value(el.second, string_open, depth + 1);
+                    write_bon8_value(el.second, string_open);
                 }
 
                 if (N > 4)
@@ -24312,84 +24299,7 @@ class binary_writer
 
             case value_t::discarded:
             default:
-                break;
-        }
-    }
-
-    /*!
-    @brief write @a j with @ref write_bon8_value, or write its marker and push
-           a frame for @ref write_bon8_iterative to continue with its elements
-
-    A scalar, and an empty array or object, are written out in full and not
-    pushed.
-
-    @sa @ref write_cbor_value_or_push
-    */
-    void write_bon8_value_or_push(const BasicJsonType& j, bool& string_open, std::vector<binary_container_frame>& stack)
-    {
-        if (j.is_array() || j.is_object())
-        {
-            // arrays use the markers 0x80..0x85, objects 0x86..0x8B; the last
-            // one stands for more than four elements, closed later with 0xFE
-            const auto N = j.size();
-            const std::size_t base = j.is_array() ? 0x80 : 0x86;
-            write_bon8_marker(static_cast<std::uint8_t>(base + (N <= 4 ? N : 5)), string_open);
-            if (N != 0)
-            {
-                stack.emplace_back(&j);
-            }
-            return;
-        }
-
-        write_bon8_value(j, string_open);
-    }
-
-    /*!
-    @brief write out @a root and everything below it without the call stack
-
-    A container with more than four elements is closed with 0xFE once its
-    last element is written, as in @ref write_bon8_value.
-
-    @sa @ref write_cbor_iterative
-    */
-    void write_bon8_iterative(const BasicJsonType& root, bool& string_open)
-    {
-        // only a container with elements is ever pushed; see write_bon8_value_or_push
-        std::vector<binary_container_frame> stack;
-        write_bon8_value_or_push(root, string_open, stack);
-
-        while (!stack.empty())
-        {
-            const binary_container_frame current = stack.back();
-            const bool is_array = current.value->is_array();
-            const bool at_end = is_array
-                                ? current.array_it == current.value->m_data.m_value.array->cend()
-                                : current.object_it == current.value->m_data.m_value.object->cend();
-
-            if (at_end)
-            {
-                if (current.value->size() > 4)
-                {
-                    write_bon8_marker(0xFE, string_open);
-                }
-                stack.pop_back();
-                continue;
-            }
-
-            // read the child before pushing: entering it can move every frame
-            const BasicJsonType* child = nullptr;
-            if (is_array)
-            {
-                child = &(*current.array_it);
-                ++stack.back().array_it;
-            }
-            else
-            {
-                write_bon8_string(current.object_it->first, string_open, *current.value);
-                child = &(current.object_it->second);
-                ++stack.back().object_it;
-            }
-            write_bon8_value_or_push(*child, string_open, stack);
+                throw_on_discarded(j, "BON8");
         }
     }
 

tests/src/unit-bon8.cpp

@@ -82,9 +82,56 @@ TEST_CASE("BON8")
     {
         SECTION("discarded")
         {
-            // discarded values are not serialized
+            // a discarded value cannot be serialized to BON8
             const json j = json::value_t::discarded;
-            CHECK(json::to_bon8(j).empty());
+            CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
+        }
+
+        SECTION("discarded values nested in a container")
+        {
+            const json discarded = json::value_t::discarded;
+
+            SECTION("in a small array")
+            {
+                const json j = {1, discarded};
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/1) cannot serialize discarded value to BON8", json::type_error&);
+#else
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
+#endif
+            }
+
+            SECTION("as a small object value")
+            {
+                json j;
+                j["a"] = discarded;
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/a) cannot serialize discarded value to BON8", json::type_error&);
+#else
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
+#endif
+            }
+
+            SECTION("in a large array")
+            {
+                const json j = {1, 2, 3, 4, 5, discarded};
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/5) cannot serialize discarded value to BON8", json::type_error&);
+#else
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
+#endif
+            }
+
+            SECTION("as a large object value")
+            {
+                json j = {{"a", 1}, {"b", 2}, {"c", 3}, {"d", 4}, {"e", 5}};
+                j["f"] = discarded;
+#if JSON_DIAGNOSTICS
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/f) cannot serialize discarded value to BON8", json::type_error&);
+#else
+                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
+#endif
+            }
         }
 
         SECTION("null")

tests/src/unit-large_json.cpp

@@ -400,31 +400,26 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
         CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array);
         CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array);
         CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array);
-        CHECK(json::from_bon8(json::to_bon8(deep_array)) == deep_array);
 
         CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object);
         CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object);
         CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object);
         CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object);
         CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object);
-        CHECK(json::from_bon8(json::to_bon8(deep_object)) == deep_object);
 
         CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array);
         CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array);
         CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array);
         CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array);
-        CHECK(json::from_bon8(json::to_bon8(empty_array)) == empty_array);
 
         CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object);
         CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object);
         CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object);
-        CHECK(json::from_bon8(json::to_bon8(empty_object)) == empty_object);
 
         CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed);
         CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed);
         CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed);
         CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed);
-        CHECK(json::from_bon8(json::to_bon8(mixed)) == mixed);
     }
 
     SECTION("the two ways of writing a value meet at the bound")
@@ -437,13 +432,11 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
             CHECK(json::from_cbor(json::to_cbor(array)) == array);
             CHECK(json::from_msgpack(json::to_msgpack(array)) == array);
             CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array);
-            CHECK(json::from_bon8(json::to_bon8(array)) == array);
 
             const json object = nested_object(depth, json(7));
             CHECK(json::from_cbor(json::to_cbor(object)) == object);
             CHECK(json::from_msgpack(json::to_msgpack(object)) == object);
             CHECK(json::from_bjdata(json::to_bjdata(object)) == object);
-            CHECK(json::from_bon8(json::to_bon8(object)) == object);
         }
     }
 
@@ -488,10 +481,6 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
             expected_ubjson.append(depth, ']');
             const auto packed_ubjson = json::to_ubjson(array);
             CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson);
-
-            std::vector<std::uint8_t> expected_bon8(depth, 0x81);
-            expected_bon8.push_back(0x90);
-            CHECK(json::to_bon8(array) == expected_bon8);
         }
     }
 
@@ -504,7 +493,6 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
         CHECK(json::from_msgpack(json::to_msgpack(object)) == object);
         CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object);
         CHECK(json::from_bjdata(json::to_bjdata(object)) == object);
-        CHECK(json::from_bon8(json::to_bon8(object)) == object);
 
         const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}});
         const json deep_ndarray = nested_array(depth, ndarray);
@@ -535,27 +523,6 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
         CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), (prefix + "cannot serialize discarded value to BJData").c_str(), json::type_error);
     }
 
-    SECTION("BON8 past the recursion bound: discarded values and errors")
-    {
-        const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50;
-
-        // BON8 writes nothing for a discarded value, deep or not
-        const json deep_discarded = nested_array(depth, json(json::value_t::discarded));
-        CHECK(json::to_bon8(deep_discarded) == std::vector<std::uint8_t>(depth, 0x81));
-
-        // errors are thrown from below the bound as from above it
-        const json deep_invalid_string = nested_array(depth, json(std::string("\x80")));
-        CHECK_THROWS_AS(json::to_bon8(deep_invalid_string), json::type_error);
-
-        json deep_invalid_key = json::object();
-        deep_invalid_key[std::string("\x80")] = 1;
-        deep_invalid_key = nested_object(depth, deep_invalid_key);
-        CHECK_THROWS_AS(json::to_bon8(deep_invalid_key), json::type_error);
-
-        const json deep_too_large = nested_array(depth, json(9223372036854775808u));
-        CHECK_THROWS_AS(json::to_bon8(deep_too_large), json::out_of_range);
-    }
-
     SECTION("does not overflow the C++ stack")
     {
         const std::size_t depth = 100000;
@@ -576,27 +543,6 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
 
         CHECK_NOTHROW(packed = json::to_bjdata(j));
         CHECK(json::from_bjdata(packed) == j);
-
-        CHECK_NOTHROW(packed = json::to_bon8(j));
-        CHECK(json::from_bon8(packed) == j);
-    }
-
-    SECTION("BON8 containers with more than four elements past the recursion bound")
-    {
-        // such containers are closed with 0xFE, which the iterative writer
-        // must emit when it leaves them; strings next to each other must
-        // still be separated with 0xFF
-        const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50;
-        json wide_array = json::array({1, "a", "b", json::array(), 5});
-        json wide_object = json({{"k1", "v"}, {"k2", 2}, {"k3", ""}, {"k4", json::object()}, {"k5", nullptr}});
-        for (std::size_t i = 0; i < depth; ++i)
-        {
-            wide_array = json::array({"s", wide_array, "t", wide_object, true});
-            wide_object = json({{"a", wide_object}, {"b", "x"}, {"c", "y"}, {"d", wide_array.size()}, {"e", "z"}});
-        }
-
-        CHECK(json::from_bon8(json::to_bon8(wide_array)) == wide_array);
-        CHECK(json::from_bon8(json::to_bon8(wide_object)) == wide_object);
     }
 
     SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014")
@@ -615,14 +561,5 @@ TEST_CASE("issue #5392 - binary writers on deeply nested values")
         const json j_msgpack = json::from_msgpack(v_msgpack);
         CHECK(json::to_msgpack(j_msgpack) == v_msgpack);
     }
-
-    SECTION("regression test for https://issues.oss-fuzz.com/issues/572238015")
-    {
-        // the BON8 analogue: 200000 nested one-element arrays down to null
-        std::vector<std::uint8_t> v(200000, 0x81);
-        v.push_back(0xFA);
-        const json j = json::from_bon8(v);
-        CHECK(json::to_bon8(j) == v);
-    }
 }