Check the URL scheme before downloading in generate_docset.py (#5803)

Codacy flagged two Bandit findings in the docset generator added in
#5799: B310 (urlopen with an unchecked scheme) and B506 (yaml.load).
download() now rejects anything but http(s) URLs before opening them,
and the yaml.load call is marked, since its Loader derives from
yaml.SafeLoader. The SHA-1 used to name downloaded files is marked as
not used for security.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/docset/generate_docset.py

@@ -337,14 +337,17 @@ def is_remote(url) -> bool:
 def download(url, docs) -> str:
     """Download url into assets/external and return the path relative to docs."""
     u = urllib.parse.urlparse(url if not url.startswith('//') else 'https:' + url)
+    if u.scheme.lower() not in ('http', 'https'):
+        raise ValueError(f'not an http(s) URL: {url}')
     req = urllib.request.Request(u.geturl(), headers={'User-Agent': USER_AGENT})
-    with urllib.request.urlopen(req, timeout=20) as r:
+    # (the scheme is checked above)
+    with urllib.request.urlopen(req, timeout=20) as r:  # nosec B310
         data = r.read()
         ctype = r.headers.get_content_type()
     path = urllib.parse.unquote(u.path).lstrip('/')
     ext = os.path.splitext(path)[1]
     if u.query or not ext or path.endswith('/'):
-        digest = hashlib.sha1(url.encode()).hexdigest()[:12]
+        digest = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12]
         path = os.path.join(os.path.dirname(path), digest + CONTENT_TYPE_EXT.get(ctype, ext or '.bin'))
     rel = os.path.normpath(os.path.join('assets', 'external', u.hostname, path))
     out = os.path.join(docs, rel)
@@ -385,7 +388,8 @@ def localize_images(docs) -> None:
 def load_mkdocs_yml() -> dict:
     """Load mkdocs.yml, ignoring tags like !ENV and !!python/name."""
     with open(MKDOCS_YML, encoding='utf-8') as f:
-        return yaml.load(f, Loader=Loader)
+        # (Loader is a yaml.SafeLoader)
+        return yaml.load(f, Loader=Loader)  # nosec B506
 
 
 def localize_site_urls(docs, site_url) -> None:

docs/mkdocs/docs/api/basic_json/at.md

@@ -241,6 +241,4 @@ Strong exception safety: if an exception occurs, the original value stays intact
 3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
    already supported by [`operator[]`](operator[].md), [`value`](value.md), [`find`](find.md), and other lookup
    functions.
-4. Added in version 2.0.0. Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of
-[`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is not
-a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones.
+4. Added in version 2.0.0.

docs/mkdocs/docs/api/basic_json/operator[].md

@@ -286,6 +286,4 @@ Strong exception safety: if an exception occurs, the original value stays intact
 3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
    already supported by [`at`](at.md), [`value`](value.md), [`find`](find.md), and other lookup functions.
 4. Added in version 2.0.0. A missing array index in the const version is guarded by a runtime assertion since
-   version 3.13.0. Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of
-[`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is not
-a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones.
+   version 3.13.0.

docs/mkdocs/docs/api/basic_json/patch.md

@@ -112,6 +112,3 @@ is thrown. In any case, the original value is not changed: the patch is applied
   location has a non-object/non-array parent in version 3.13.0.
 - Added [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) and rejected a "move" operation whose "from" location is a proper
   prefix of its "path" location instead of silently producing a corrupted result in version 3.13.0.
-- Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of
-  [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is
-  not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones.

docs/mkdocs/docs/api/basic_json/patch_inplace.md

@@ -110,6 +110,3 @@ function throws an exception.
   location has a non-object/non-array parent in version 3.13.0.
 - Added [`out_of_range.414`](../../home/exceptions.md#jsonexceptionout_of_range414) and rejected a "move" operation whose "from" location is a proper
   prefix of its "path" location instead of silently producing a corrupted result in version 3.13.0.
-- Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of
-  [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is
-  not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones.

docs/mkdocs/docs/api/basic_json/unflatten.md

@@ -36,9 +36,8 @@ The function can throw the following exceptions:
 - Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) if an array index in a key is not a
   number; example: `"array index 'one' is not a number"`
 - Throws [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) if a level becomes an array
-  (because one of its keys is `0`) and another key at that level begins with a digit but is not a valid array index
-  (such as `1a`), or is `-`; example:
-  `"unresolved reference token '-'"`
+  (because one of its keys is `0`) and another key at that level cannot be an array index; example:
+  `"unresolved reference token 'x'"`
 
 ## Complexity
 
@@ -81,6 +80,3 @@ Apart from these two cases, for a JSON value `j`, the following is always true:
 
 - Added in version 2.0.0.
 - Made the array/object decision independent of the object's iteration order in version 3.13.0.
-- Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109) instead of
-  [`out_of_range.404`](../../home/exceptions.md#jsonexceptionout_of_range404) for a one-character array index that is
-  not a digit (e.g., `/x`) in version 3.13.0, as it already did for longer ones.

docs/mkdocs/docs/api/basic_json/value.md

@@ -227,6 +227,4 @@ changes to any JSON value.
    [`operator[]`](operator[].md), [`at`](at.md), [`find`](find.md), and other lookup functions.
 3. Added in version 2.0.2. Extended to work with arrays in version 3.13.0, including fixing an issue where resolving
    `ptr` through an array unexpectedly threw `out_of_range` instead of returning the resolved element (or
-   `default_value`, as documented). Throws [`parse_error.109`](../../home/exceptions.md#jsonexceptionparse_error109)
-   instead of returning `default_value` for a one-character array index that is not a digit (e.g., `/x`) in version
-   3.13.0, as it already did for longer ones.
+   `default_value`, as documented).

docs/mkdocs/docs/home/exceptions.md

@@ -278,9 +278,7 @@ In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
 
 ### json.exception.parse_error.109
 
-A JSON Pointer array index must be a number. This exception is thrown for an array index that does not begin with a
-digit, except for `-` and the empty reference token, which throw [`out_of_range.404`](#jsonexceptionout_of_range404)
-where they cannot be resolved.
+A JSON Pointer array index must be a number.
 
 !!! failure "Example messages"
 
@@ -291,11 +289,6 @@ where they cannot be resolved.
     [json.exception.parse_error.109] parse error: array index '+1' is not a number
     ```
 
-!!! note
-
-    Before version 3.13.0, a one-character array index that is not a digit (e.g., `x`) threw
-    [`out_of_range.404`](#jsonexceptionout_of_range404) instead.
-
 ### json.exception.parse_error.110
 
 When parsing a [binary format](../features/binary_formats/index.md), the byte vector ends before the complete value has
@@ -878,8 +871,7 @@ The provided key was not found in the JSON object.
 
 ### json.exception.out_of_range.404
 
-A reference token in a JSON Pointer could not be resolved, for instance an array index that begins with a digit but
-contains other characters (e.g., `1a`), or `-` where it cannot be used.
+A reference token in a JSON Pointer could not be resolved.
 
 !!! failure "Example message"
 

include/nlohmann/detail/json_pointer.hpp

@@ -255,7 +255,7 @@ class json_pointer
     {
         ok,                ///< @a s is a valid, representable array index
         leading_zero,      ///< @a s begins with '0' but has more than one character
-        not_a_number,      ///< @a s is neither empty nor "-" and does not begin with a digit
+        not_a_number,      ///< @a s does not begin with a digit
         unresolved,        ///< @a s could not be converted to an integer
         exceeds_size_type  ///< @a s converts to an integer that exceeds size_type
     };
@@ -282,10 +282,8 @@ class json_pointer
             return array_index_status::leading_zero;
         }
 
-        // error condition (cf. RFC 6901, Sect. 4); this also covers single-
-        // character tokens, so "/x" and "/xy" fail alike; "-" and the empty
-        // token are left to the conversion below and are reported as unresolved
-        if (JSON_HEDLEY_UNLIKELY(!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9')))
+        // error condition (cf. RFC 6901, Sect. 4)
+        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))
         {
             return array_index_status::not_a_number;
         }

include/nlohmann/ordered_map.hpp

@@ -247,14 +247,6 @@ public:
         //               ^        ^
         //             first    last
 
-        // Note on conformance: before C++20, [basic.life]/8 did not allow an
-        // object of a type with a const member (like value_type's const Key)
-        // to transparently replace the destroyed one, so strictly, accessing
-        // it through the vector's existing pointers would have required
-        // std::launder (which does not exist before C++17). C++20 dropped that
-        // condition (P1971R0, NB comment US 041). Compilers have always treated
-        // this pattern as intended, so it is kept deliberately.
-
         // Since we cannot move const Keys, we re-construct them in place.
         // We start at first and re-construct (viz. copy) the elements from
         // the back of the vector. Example for the first iteration:

single_include/nlohmann/json.hpp

@@ -20173,7 +20173,7 @@ class json_pointer
     {
         ok,                ///< @a s is a valid, representable array index
         leading_zero,      ///< @a s begins with '0' but has more than one character
-        not_a_number,      ///< @a s is neither empty nor "-" and does not begin with a digit
+        not_a_number,      ///< @a s does not begin with a digit
         unresolved,        ///< @a s could not be converted to an integer
         exceeds_size_type  ///< @a s converts to an integer that exceeds size_type
     };
@@ -20200,10 +20200,8 @@ class json_pointer
             return array_index_status::leading_zero;
         }
 
-        // error condition (cf. RFC 6901, Sect. 4); this also covers single-
-        // character tokens, so "/x" and "/xy" fail alike; "-" and the empty
-        // token are left to the conversion below and are reported as unresolved
-        if (JSON_HEDLEY_UNLIKELY(!s.empty() && s != "-" && !(s[0] >= '0' && s[0] <= '9')))
+        // error condition (cf. RFC 6901, Sect. 4)
+        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))
         {
             return array_index_status::not_a_number;
         }
@@ -27782,14 +27780,6 @@ public:
         //               ^        ^
         //             first    last
 
-        // Note on conformance: before C++20, [basic.life]/8 did not allow an
-        // object of a type with a const member (like value_type's const Key)
-        // to transparently replace the destroyed one, so strictly, accessing
-        // it through the vector's existing pointers would have required
-        // std::launder (which does not exist before C++17). C++20 dropped that
-        // condition (P1971R0, NB comment US 041). Compilers have always treated
-        // this pattern as intended, so it is kept deliberately.
-
         // Since we cannot move const Keys, we re-construct them in place.
         // We start at first and re-construct (viz. copy) the elements from
         // the back of the vector. Example for the first iteration:

tests/src/unit-element_access2.cpp

@@ -535,8 +535,6 @@ TEST_CASE_TEMPLATE("element access 2", Json, nlohmann::json, nlohmann::ordered_j
                     // Test malformed index (non-numeric) throws parse_error
                     CHECK_THROWS_WITH_AS(j_array.value("/foo"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'foo' is not a number", typename Json::parse_error&);
                     CHECK_THROWS_WITH_AS(j_array_const.value("/foo"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'foo' is not a number", typename Json::parse_error&);
-                    CHECK_THROWS_WITH_AS(j_array.value("/x"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'x' is not a number", typename Json::parse_error&);
-                    CHECK_THROWS_WITH_AS(j_array_const.value("/x"_json_pointer, 1), "[json.exception.parse_error.109] parse error: array index 'x' is not a number", typename Json::parse_error&);
 
                     // Test leading-zero index throws parse_error
                     CHECK_THROWS_WITH_AS(j_array.value("/01"_json_pointer, 1), "[json.exception.parse_error.106] parse error: array index '01' must not begin with '0'", typename Json::parse_error&);

tests/src/unit-json_patch.cpp

@@ -1713,21 +1713,6 @@ TEST_CASE("JSON patch - move where 'from' is a proper prefix of 'path' (regressi
         CHECK(doc.patch(patch) == R"({"b": 1})"_json);
     }
 
-    SECTION("array index tokens that are not a number")
-    {
-        json const doc = R"({"a": [1, 2]})"_json;
-
-        // "-" cannot be removed and is reported as unresolved
-        json const patch_dash = {{{"op", "remove"}, {"path", "/a/-"}}};
-        CHECK_THROWS_WITH_AS(doc.patch(patch_dash), "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&);
-
-        // a single character is reported like a longer token
-        json const patch_x = {{{"op", "remove"}, {"path", "/a/x"}}};
-        CHECK_THROWS_WITH_AS(doc.patch(patch_x), "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-        json const patch_xy = {{{"op", "remove"}, {"path", "/a/xy"}}};
-        CHECK_THROWS_WITH_AS(doc.patch(patch_xy), "[json.exception.parse_error.109] parse error: array index 'xy' is not a number", json::parse_error&);
-    }
-
     SECTION("the array-append token '-' is an ordinary child token")
     {
         // "-" (append-to-array) addresses a location *inside* the array,

tests/src/unit-json_pointer.cpp

@@ -421,27 +421,6 @@ TEST_CASE("JSON pointers")
             CHECK_THROWS_WITH_AS(json({{"/list/0", 1}, {"/list/1", 2}, {"/list/three", 3}}).unflatten(),
             "[json.exception.parse_error.109] parse error: array index 'three' is not a number", json::parse_error&);
 
-            // a single-character token that is not a digit is reported like a
-            // longer one (parse_error.109), not as unresolved (out_of_range.404)
-            CHECK_THROWS_WITH_AS(j["/x"_json_pointer] = 1,
-                                 "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-            CHECK_THROWS_WITH_AS(j_const["/x"_json_pointer] == 1,
-                                 "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-            CHECK_THROWS_WITH_AS(j.at("/x"_json_pointer) = 1,
-                                 "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-            CHECK_THROWS_WITH_AS(j_const.at("/x"_json_pointer) == 1,
-                                 "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-            CHECK_THROWS_WITH_AS(j.at("/+"_json_pointer),
-                                 "[json.exception.parse_error.109] parse error: array index '+' is not a number", json::parse_error&);
-            CHECK(!j.contains("/x"_json_pointer));
-            CHECK(!j_const.contains("/x"_json_pointer));
-            CHECK_THROWS_WITH_AS(json({{"/list/0", 1}, {"/list/x", 2}}).unflatten(),
-            "[json.exception.parse_error.109] parse error: array index 'x' is not a number", json::parse_error&);
-
-            // "-" is a valid reference token, so it is still reported as unresolved
-            CHECK_THROWS_WITH_AS(json({{"/list/0", 1}, {"/list/-", 2}}).unflatten(),
-            "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&);
-
             // assign to "-"
             j["/-"_json_pointer] = 99;
             CHECK(j == json({1, 13, 3, 33, nullptr, 55, 99}));