Read the node array base after emit() in the view builder's open()

emit() moves the node array when it grows, and the subtraction read base
in the same expression, so the order was unspecified. MSVC Release builds
without forced inlining read the old base; the container index then
pointed outside the array and close() wrote out of bounds.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/docset/generate_docset.py

@@ -337,14 +337,17 @@ def is_remote(url) -> bool:
 def download(url, docs) -> str:
     """Download url into assets/external and return the path relative to docs."""
     u = urllib.parse.urlparse(url if not url.startswith('//') else 'https:' + url)
+    if u.scheme.lower() not in ('http', 'https'):
+        raise ValueError(f'not an http(s) URL: {url}')
     req = urllib.request.Request(u.geturl(), headers={'User-Agent': USER_AGENT})
-    with urllib.request.urlopen(req, timeout=20) as r:
+    # (the scheme is checked above)
+    with urllib.request.urlopen(req, timeout=20) as r:  # nosec B310
         data = r.read()
         ctype = r.headers.get_content_type()
     path = urllib.parse.unquote(u.path).lstrip('/')
     ext = os.path.splitext(path)[1]
     if u.query or not ext or path.endswith('/'):
-        digest = hashlib.sha1(url.encode()).hexdigest()[:12]
+        digest = hashlib.sha1(url.encode(), usedforsecurity=False).hexdigest()[:12]
         path = os.path.join(os.path.dirname(path), digest + CONTENT_TYPE_EXT.get(ctype, ext or '.bin'))
     rel = os.path.normpath(os.path.join('assets', 'external', u.hostname, path))
     out = os.path.join(docs, rel)
@@ -385,7 +388,8 @@ def localize_images(docs) -> None:
 def load_mkdocs_yml() -> dict:
     """Load mkdocs.yml, ignoring tags like !ENV and !!python/name."""
     with open(MKDOCS_YML, encoding='utf-8') as f:
-        return yaml.load(f, Loader=Loader)
+        # (Loader is a yaml.SafeLoader)
+        return yaml.load(f, Loader=Loader)  # nosec B506
 
 
 def localize_site_urls(docs, site_url) -> None:

include/nlohmann/detail/view/builder.hpp

@@ -821,7 +821,10 @@ indent_done:
 
         NLOHMANN_VIEW_ALWAYS_INLINE void open(value_t k)
         {
-            const auto idx = static_cast<std::uint32_t>(emit(k, 0, 0, static_cast<std::size_t>(p - b), 0) - base);
+            // (base is read after emit(), which moves the node array when it
+            // grows; in one expression the order of the two is unspecified)
+            const node* const n = emit(k, 0, 0, static_cast<std::size_t>(p - b), 0);
+            const auto idx = static_cast<std::uint32_t>(n - base);
             if (depth != 0)
             {
                 const frame f = {cur_idx, cur_count, cur_is_object};

include/nlohmann/detail/view/errors.hpp

@@ -55,7 +55,7 @@ template<typename BasicJsonType>
 {
     if (f.code == error_code::input_too_large)
     {
-        // (the limit is detail::view::max_input_size: 4 GiB minus 16 bytes)
+        // (the limit is detail::view::max_input_size(): 4 GiB minus 16 bytes)
         NLOHMANN_VIEW_THROW(out_of_range::create(416, "input of 4294967280 bytes or more is not supported by json_document", nullptr));
     }
     const BasicJsonType accepted = BasicJsonType::parse(src, src + size, nullptr, true, ignore_comments, ignore_trailing_commas);

include/nlohmann/detail/view/node.hpp

@@ -32,7 +32,10 @@ static_assert(static_cast<std::uint8_t>(value_t::null) == 0 && static_cast<std::
 /// The largest input a document accepts, in bytes. Offsets and node counts are
 /// 32 bits wide; the limit keeps 16 bytes (the width of the scanner's steps)
 /// below 2^32, so that a position one step past the end of the text fits.
-static constexpr std::size_t max_input_size = 0xFFFFFFEFu;
+constexpr std::size_t max_input_size() noexcept
+{
+    return 0xFFFFFFEFu;
+}
 
 /// node flags
 struct node_flags

include/nlohmann/json_view.hpp

@@ -24,7 +24,8 @@
 #ifndef INCLUDE_NLOHMANN_JSON_VIEW_HPP_
 #define INCLUDE_NLOHMANN_JSON_VIEW_HPP_
 
-#include <cstddef> // size_t
+#include <array> // array
+#include <cstddef> // nullptr_t, size_t // IWYU pragma: keep
 #include <cstring> // memcpy, strlen
 #include <iterator> // distance, input_iterator_tag, iterator_traits
 #include <memory> // unique_ptr
@@ -32,7 +33,21 @@
 #include <type_traits> // enable_if, integral_constant, is_base_of, is_integral, is_same, remove_cv, remove_extent
 #include <utility> // forward, move
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json.hpp> // IWYU pragma: export
+
+// json.hpp provides the library's types and macros used below (it includes
+// the headers that define them); json_view.hpp must not include them again,
+// because the amalgamated json_view.hpp only includes json.hpp
+// IWYU pragma: no_include <version>
+// IWYU pragma: no_include "nlohmann/detail/abi_config.hpp"
+// IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp"
+// IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp"
+// IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp"
+// IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp"
+// IWYU pragma: no_include "nlohmann/detail/string_concat.hpp"
+// IWYU pragma: no_include "nlohmann/detail/value_t.hpp"
+// IWYU pragma: no_include "nlohmann/json.hpp"
+// IWYU pragma: no_include "nlohmann/json_fwd.hpp"
 
 // the view builds on internals of the library: both must be the same version
 #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 || NLOHMANN_JSON_VERSION_PATCH != 0
@@ -50,9 +65,6 @@
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 
-template<typename BasicJsonType>
-class basic_json_document;
-
 /*!
 @brief read-only handle to one value of a basic_json_document
 
@@ -475,7 +487,7 @@ class basic_json_document
         d.discarded = true;
         detail::view::parse_failure failure;
         bool ok = false;
-        if (NLOHMANN_VIEW_UNLIKELY(size > detail::view::max_input_size))
+        if (NLOHMANN_VIEW_UNLIKELY(size > detail::view::max_input_size()))
         {
             failure.code = detail::view::error_code::input_too_large;
         }
@@ -607,6 +619,6 @@ using ordered_json_view = basic_json_view<ordered_json>;
 
 NLOHMANN_JSON_NAMESPACE_END
 
-#include <nlohmann/detail/view/macro_unscope.hpp>
+#include <nlohmann/detail/view/macro_unscope.hpp> // IWYU pragma: keep
 
 #endif  // INCLUDE_NLOHMANN_JSON_VIEW_HPP_

single_include/nlohmann/json_view.hpp

@@ -24,7 +24,8 @@
 #ifndef INCLUDE_NLOHMANN_JSON_VIEW_HPP_
 #define INCLUDE_NLOHMANN_JSON_VIEW_HPP_
 
-#include <cstddef> // size_t
+#include <array> // array
+#include <cstddef> // nullptr_t, size_t // IWYU pragma: keep
 #include <cstring> // memcpy, strlen
 #include <iterator> // distance, input_iterator_tag, iterator_traits
 #include <memory> // unique_ptr
@@ -32,7 +33,21 @@
 #include <type_traits> // enable_if, integral_constant, is_base_of, is_integral, is_same, remove_cv, remove_extent
 #include <utility> // forward, move
 
-#include <nlohmann/json.hpp>
+#include <nlohmann/json.hpp> // IWYU pragma: export
+
+// json.hpp provides the library's types and macros used below (it includes
+// the headers that define them); json_view.hpp must not include them again,
+// because the amalgamated json_view.hpp only includes json.hpp
+// IWYU pragma: no_include <version>
+// IWYU pragma: no_include "nlohmann/detail/abi_config.hpp"
+// IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp"
+// IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp"
+// IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp"
+// IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp"
+// IWYU pragma: no_include "nlohmann/detail/string_concat.hpp"
+// IWYU pragma: no_include "nlohmann/detail/value_t.hpp"
+// IWYU pragma: no_include "nlohmann/json.hpp"
+// IWYU pragma: no_include "nlohmann/json_fwd.hpp"
 
 // the view builds on internals of the library: both must be the same version
 #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 12 || NLOHMANN_JSON_VERSION_PATCH != 0
@@ -189,7 +204,10 @@ static_assert(static_cast<std::uint8_t>(value_t::null) == 0 && static_cast<std::
 /// The largest input a document accepts, in bytes. Offsets and node counts are
 /// 32 bits wide; the limit keeps 16 bytes (the width of the scanner's steps)
 /// below 2^32, so that a position one step past the end of the text fits.
-static constexpr std::size_t max_input_size = 0xFFFFFFEFu;
+constexpr std::size_t max_input_size() noexcept
+{
+    return 0xFFFFFFEFu;
+}
 
 /// node flags
 struct node_flags
@@ -1402,7 +1420,10 @@ indent_done:
 
         NLOHMANN_VIEW_ALWAYS_INLINE void open(value_t k)
         {
-            const auto idx = static_cast<std::uint32_t>(emit(k, 0, 0, static_cast<std::size_t>(p - b), 0) - base);
+            // (base is read after emit(), which moves the node array when it
+            // grows; in one expression the order of the two is unspecified)
+            const node* const n = emit(k, 0, 0, static_cast<std::size_t>(p - b), 0);
+            const auto idx = static_cast<std::uint32_t>(n - base);
             if (depth != 0)
             {
                 const frame f = {cur_idx, cur_count, cur_is_object};
@@ -1690,7 +1711,7 @@ template<typename BasicJsonType>
 {
     if (f.code == error_code::input_too_large)
     {
-        // (the limit is detail::view::max_input_size: 4 GiB minus 16 bytes)
+        // (the limit is detail::view::max_input_size(): 4 GiB minus 16 bytes)
         NLOHMANN_VIEW_THROW(out_of_range::create(416, "input of 4294967280 bytes or more is not supported by json_document", nullptr));
     }
     const BasicJsonType accepted = BasicJsonType::parse(src, src + size, nullptr, true, ignore_comments, ignore_trailing_commas);
@@ -2145,9 +2166,6 @@ NLOHMANN_JSON_NAMESPACE_END
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 
-template<typename BasicJsonType>
-class basic_json_document;
-
 /*!
 @brief read-only handle to one value of a basic_json_document
 
@@ -2570,7 +2588,7 @@ class basic_json_document
         d.discarded = true;
         detail::view::parse_failure failure;
         bool ok = false;
-        if (NLOHMANN_VIEW_UNLIKELY(size > detail::view::max_input_size))
+        if (NLOHMANN_VIEW_UNLIKELY(size > detail::view::max_input_size()))
         {
             failure.code = detail::view::error_code::input_too_large;
         }
@@ -2724,6 +2742,6 @@ NLOHMANN_JSON_NAMESPACE_END
 #undef NLOHMANN_VIEW_THROW
 #undef NLOHMANN_VIEW_LITTLE_ENDIAN
 #undef NLOHMANN_VIEW_REPEAT16
-
+// IWYU pragma: keep
 
 #endif  // INCLUDE_NLOHMANN_JSON_VIEW_HPP_

tests/src/unit-json_view.cpp

@@ -457,7 +457,7 @@ TEST_CASE("json_view")
     {
         // 32-bit offsets: the limit is 4 GiB minus 16 bytes (a margin below 2^32),
         // which is what the exception message and the documentation say
-        const std::size_t limit = nlohmann::detail::view::max_input_size;
+        const std::size_t limit = nlohmann::detail::view::max_input_size();
         CHECK(limit == std::size_t{4294967279u});
 
         const oversized_input input{limit + 1};