Check the URL scheme before downloading in generate_docset.py (#5803)

Codacy flagged two Bandit findings in the docset generator added in
#5799: B310 (urlopen with an unchecked scheme) and B506 (yaml.load).
download() now rejects anything but http(s) URLs before opening them,
and the yaml.load call is marked, since its Loader derives from
yaml.SafeLoader. The SHA-1 used to name downloaded files is marked as
not used for security.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/docset/generate_docset.py

@@ -337,14 +337,17 @@ def is_remote(url) -> bool:
 def download(url, docs) -> str:
     """Download url into assets/external and return the path relative to docs."""
     u = urllib.parse.urlparse(url if not url.startswith('//') else 'https:' + url)
+    if u.scheme.lower() not in ('http', 'https'):
+        raise ValueError(f'not an http(s) URL: {url}')
     req = urllib.request.Request(u.geturl(), headers={'User-Agent': USER_AGENT})
-    with urllib.request.urlopen(req, timeout=20) as r:
+    # (the scheme is checked above)
+    with urllib.request.urlopen(req, timeout=20) as r:  # nosec B310
         data = r.read()
         ctype = r.headers.get_content_type()
     path = urllib.parse.unquote(u.path).lstrip('/')
     ext = os.path.splitext(path)[1]
     if u.query or not ext or path.endswith('/'):
-        digest = hashlib.sha1(url.encode()).hexdigest()[:12]
+        digest = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12]
         path = os.path.join(os.path.dirname(path), digest + CONTENT_TYPE_EXT.get(ctype, ext or '.bin'))
     rel = os.path.normpath(os.path.join('assets', 'external', u.hostname, path))
     out = os.path.join(docs, rel)
@@ -385,7 +388,8 @@ def localize_images(docs) -> None:
 def load_mkdocs_yml() -> dict:
     """Load mkdocs.yml, ignoring tags like !ENV and !!python/name."""
     with open(MKDOCS_YML, encoding='utf-8') as f:
-        return yaml.load(f, Loader=Loader)
+        # (Loader is a yaml.SafeLoader)
+        return yaml.load(f, Loader=Loader)  # nosec B506
 
 
 def localize_site_urls(docs, site_url) -> None:

docs/mkdocs/docs/api/basic_json/to_bon8.md

@@ -41,8 +41,6 @@ With (2), the bytes written before the exception remain in the output adapter.
   above 9223372036854775807, which BON8 cannot represent
 - Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if `j` contains a string that is not
   valid UTF-8
-- Throws [type_error.321](../../home/exceptions.md#jsonexceptiontype_error321) if `j` or a value nested in it is
-  discarded; example: `"cannot serialize discarded value to BON8"`
 
 ## Complexity
 

docs/mkdocs/docs/home/exceptions.md

@@ -810,7 +810,7 @@ does not list an enumerator and it is therefore converted like the first listed
 A discarded value (one created by [`parse()`](../api/basic_json/parse.md) with a callback that returns `false` for the
 value, or by default-constructing a [`basic_json`](../api/basic_json/index.md) with
 [`value_t::discarded`](../api/basic_json/value_t.md)) was passed to a binary serialization function, either directly or
-nested in an array or object. There is no way to represent a discarded value in CBOR, MessagePack, UBJSON, BJData, BSON, or BON8.
+nested in an array or object. There is no way to represent a discarded value in CBOR, MessagePack, UBJSON, BJData, or BSON.
 
 !!! failure "Example message"
 

include/nlohmann/detail/output/binary_writer.hpp

@@ -809,8 +809,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-
-    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8(const BasicJsonType& j)
     {
@@ -2572,8 +2570,6 @@ class binary_writer
     @param[in] j                JSON value to serialize
     @param[in,out] string_open  whether the output ends with a non-empty
                                 string that has not been terminated with 0xFF
-
-    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8_value(const BasicJsonType& j, bool& string_open)
     {
@@ -2682,7 +2678,7 @@ class binary_writer
 
             case value_t::discarded:
             default:
-                throw_on_discarded(j, "BON8");
+                break;
         }
     }
 

single_include/nlohmann/json.hpp

@@ -22426,8 +22426,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-
-    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8(const BasicJsonType& j)
     {
@@ -24189,8 +24187,6 @@ class binary_writer
     @param[in] j                JSON value to serialize
     @param[in,out] string_open  whether the output ends with a non-empty
                                 string that has not been terminated with 0xFF
-
-    @throw type_error.321 if @a j or a value nested in it is discarded
     */
     void write_bon8_value(const BasicJsonType& j, bool& string_open)
     {
@@ -24299,7 +24295,7 @@ class binary_writer
 
             case value_t::discarded:
             default:
-                throw_on_discarded(j, "BON8");
+                break;
         }
     }
 

tests/src/unit-bon8.cpp

@@ -82,56 +82,9 @@ TEST_CASE("BON8")
     {
         SECTION("discarded")
         {
-            // a discarded value cannot be serialized to BON8
+            // discarded values are not serialized
             const json j = json::value_t::discarded;
-            CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
-        }
-
-        SECTION("discarded values nested in a container")
-        {
-            const json discarded = json::value_t::discarded;
-
-            SECTION("in a small array")
-            {
-                const json j = {1, discarded};
-#if JSON_DIAGNOSTICS
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/1) cannot serialize discarded value to BON8", json::type_error&);
-#else
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
-#endif
-            }
-
-            SECTION("as a small object value")
-            {
-                json j;
-                j["a"] = discarded;
-#if JSON_DIAGNOSTICS
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/a) cannot serialize discarded value to BON8", json::type_error&);
-#else
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
-#endif
-            }
-
-            SECTION("in a large array")
-            {
-                const json j = {1, 2, 3, 4, 5, discarded};
-#if JSON_DIAGNOSTICS
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/5) cannot serialize discarded value to BON8", json::type_error&);
-#else
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
-#endif
-            }
-
-            SECTION("as a large object value")
-            {
-                json j = {{"a", 1}, {"b", 2}, {"c", 3}, {"d", 4}, {"e", 5}};
-                j["f"] = discarded;
-#if JSON_DIAGNOSTICS
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] (/f) cannot serialize discarded value to BON8", json::type_error&);
-#else
-                CHECK_THROWS_WITH_AS(json::to_bon8(j), "[json.exception.type_error.321] cannot serialize discarded value to BON8", json::type_error&);
-#endif
-            }
+            CHECK(json::to_bon8(j).empty());
         }
 
         SECTION("null")