Read BSON documents without recursing per nesting level

An embedded document (record type 0x03) or array (0x04) was read by calling
back into the document reader, which read its element list, which called the
element reader again for the next embedded one. The native call stack
therefore grew with the nesting depth of the input, and about seven bytes buy
a level, so a document of a few hundred kilobytes crashes the process
(#5104). This is the last of the four binary formats to still do that.

Apply the same shape as the other three: open_bson_document() reads the size
prefix and opens the document, parse_bson_element_internal() calls it for both
record types instead of recursing, and parse_bson_internal() loops over the
element list of whichever document is innermost, closing it when its
terminator is reached and resuming the one below.

check_bson_document_size() is unchanged, and so is when it runs: a document is
still measured from the byte before its size prefix to the byte after its
terminator, and still reported before the end event. The frame carries those
two values, which is what a per-document check needs once the reads are
interleaved rather than nested. Nothing else about the element reader changes.

unit-bson passes unchanged. Round trips through to_bson of nested objects,
arrays, arrays of objects and mixed nesting are identical to the previous
commit, as are the errors for a truncated document, an unsupported record
type, a negative size and a size that does not match, including their byte
offsets. A 30,000-level document built by to_bson is now read to completion
where it used to crash.

Note for sequencing: #5185 changes parse_bson_internal(), the element list and
the array reader, which are the functions this commit restructures. It should
land first; this commit then keeps its checks and moves them onto the loop.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -199,11 +199,21 @@ class binary_reader
     */
     struct container_frame
     {
-        container_frame(const std::size_t remaining_, const bool is_object_) noexcept
+        container_frame(const std::size_t remaining_, const bool is_object_,
-            : remaining(remaining_), is_object(is_object_) {}
+                        const char_int_type type_marker_ = 0) noexcept
+            : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {}
 
-        /// number of elements that have not been read yet
+        /// number of elements that have not been read yet, or npos when the
+        /// container is not sized and ends at a marker instead
         std::size_t remaining;
+        /// BSON: value of chars_read before this document's size prefix, which
+        /// check_bson_document_size() needs once the document has been read
+        std::size_t start_position = 0;
+        /// UBJSON/BJData: the type marker of an optimized container, so that
+        /// its elements are read without one of their own; 0 otherwise
+        char_int_type type_marker;
+        /// BSON: the size this document declares, in bytes
+        std::int32_t declared_size = 0;
         /// whether to close this container with end_object() or end_array()
         bool is_object;
     };
@@ -220,27 +230,28 @@ class binary_reader
 
     @return whether the SAX parser accepted the start event
     */
-    bool enter_container(const bool is_object, const std::size_t len)
+    bool enter_container(const bool is_object, const std::size_t len,
+                         const char_int_type type_marker = 0)
     {
         if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len)))
         {
             return false;
         }
 
-        container_stack.emplace_back(len, is_object);
+        container_stack.emplace_back(len, is_object, type_marker);
         return true;
     }
 
     /// @copydoc enter_container
-    bool enter_array(const std::size_t len)
+    bool enter_array(const std::size_t len, const char_int_type type_marker = 0)
     {
-        return enter_container(/*is_object*/false, len);
+        return enter_container(/*is_object*/false, len, type_marker);
     }
 
     /// @copydoc enter_container
-    bool enter_object(const std::size_t len)
+    bool enter_object(const std::size_t len, const char_int_type type_marker = 0)
     {
-        return enter_container(/*is_object*/true, len);
+        return enter_container(/*is_object*/true, len, type_marker);
     }
 
     //////////
@@ -277,8 +288,10 @@ class binary_reader
     @brief Reads in a BSON-object and passes it to the SAX-parser.
     @return whether a valid BSON-value was passed to the SAX parser
     */
-    bool parse_bson_internal()
+    bool open_bson_document(const bool is_object)
     {
+        // recorded before the size prefix is read, because
+        // check_bson_document_size() measures the document from here
         const std::size_t document_start = chars_read;
         std::int32_t document_size{};
         if (!get_number<std::int32_t, true>(input_format_t::bson, document_size))
@@ -286,22 +299,89 @@ class binary_reader
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
+        if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size())))
         {
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false)))
+        container_frame& frame = container_stack.back();
+        frame.start_position = document_start;
+        frame.declared_size = document_size;
+        return true;
+    }
+
+    /*!
+    @brief read a BSON document and everything nested inside it
+
+    Reads elements until the document that was begun here is complete,
+    resuming the enclosing document each time an embedded one ends, so that
+    the nesting depth of the input costs heap rather than native stack
+    (see #5104).
+
+    @return whether reading the document succeeded
+    */
+    bool parse_bson_internal()
+    {
+        if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true)))
         {
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
+        // the key currently being read; hoisted out of the loop so that its
+        // capacity is reused across elements and across nesting levels
+        string_t key;
+
+        while (true)
+        {
+            const auto element_type = get();
+
+            if (element_type == 0) // end of the innermost document
+            {
+                const container_frame& top = container_stack.back();
+                const bool is_object = top.is_object;
+
+                if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size)))
                 {
                     return false;
                 }
 
-        return sax->end_object();
+                container_stack.pop_back();
+                if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array()))
+                {
+                    return false;
+                }
+                // the document begun here is complete once it is not inside one
+                if (container_stack.empty())
+                {
+                    return true;
+                }
+                continue;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list")))
+            {
+                return false;
+            }
+
+            const std::size_t element_type_parse_position = chars_read;
+            key.clear();
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
+            {
+                return false;
+            }
+
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on
+            if (container_stack.back().is_object && !sax->key(key))
+            {
+                return false;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
+            {
+                return false;
+            }
+        }
     }
 
     /*!
@@ -413,12 +493,12 @@ class binary_reader
 
             case 0x03: // object
             {
-                return parse_bson_internal();
+                return open_bson_document(/*is_object*/true);
             }
 
             case 0x04: // array
             {
-                return parse_bson_array();
+                return open_bson_document(/*is_object*/false);
             }
 
             case 0x05: // binary
@@ -468,82 +548,7 @@ class binary_reader
         }
     }
 
-    /*!
-    @brief Read a BSON element list (as specified in the BSON-spec)
 
-    The same binary layout is used for objects and arrays, hence it must be
-    indicated with the argument @a is_array which one is expected
-    (true --> array, false --> object).
-
-    @param[in] is_array Determines if the element list being read is to be
-                        treated as an object (@a is_array == false), or as an
-                        array (@a is_array == true).
-    @return whether a valid BSON-object/array was passed to the SAX parser
-    */
-    bool parse_bson_element_list(const bool is_array)
-    {
-        string_t key;
-
-        while (auto element_type = get())
-        {
-            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list")))
-            {
-                return false;
-            }
-
-            const std::size_t element_type_parse_position = chars_read;
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
-            {
-                return false;
-            }
-
-            if (!is_array && !sax->key(key))
-            {
-                return false;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
-            {
-                return false;
-            }
-
-            // get_bson_cstr only appends
-            key.clear();
-        }
-
-        return true;
-    }
-
-    /*!
-    @brief Reads an array from the BSON input and passes it to the SAX-parser.
-    @return whether a valid BSON-array was passed to the SAX parser
-    */
-    bool parse_bson_array()
-    {
-        const std::size_t document_start = chars_read;
-        std::int32_t document_size{};
-        if (!get_number<std::int32_t, true>(input_format_t::bson, document_size))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true)))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
-        {
-            return false;
-        }
-
-        return sax->end_array();
-    }
 
     //////////
     // CBOR //
@@ -2169,7 +2174,103 @@ class binary_reader
     */
     bool parse_ubjson_internal(const bool get_char = true)
     {
-        return get_ubjson_value(get_char ? get_ignore_noop() : current);
+        // the key currently being read; hoisted out of the loop so that its
+        // capacity is reused across elements and across nesting levels
+        string_t key;
+
+        // the type marker of the value to read next
+        char_int_type prefix = get_char ? get_ignore_noop() : current;
+
+        while (true)
+        {
+            const std::size_t depth = container_stack.size();
+
+            if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix)))
+            {
+                return false;
+            }
+
+            // the value begun here is complete once it is not inside anything
+            if (container_stack.empty())
+            {
+                return true;
+            }
+
+            // a value was completed rather than a container opened; a
+            // container that ends at a marker needs the next byte to test
+            if (container_stack.size() == depth && container_stack.back().remaining == npos)
+            {
+                get_ignore_noop();
+            }
+
+            // advance to the next element, closing the containers that ended.
+            // The reference is not held across get_ubjson_value() above, which
+            // can push onto the stack and reallocate it.
+            for (;;)
+            {
+                container_frame& top = container_stack.back();
+
+                if (top.remaining != npos)
+                {
+                    if (top.remaining != 0)
+                    {
+                        --top.remaining;
+                        if (top.is_object)
+                        {
+                            key.clear();
+                            if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+                            {
+                                return false;
+                            }
+                        }
+                        // an optimized container gives its elements no marker
+                        prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop();
+                        break;
+                    }
+                }
+                // the end marker is compared against a literal rather than
+                // against a conditional expression, because char_int_type is
+                // unsigned for some input adapters and MSVC then reports the
+                // comparison as a signed/unsigned mismatch
+                else if (top.is_object ? (current != '}') : (current != ']'))
+                {
+                    // a container that ends at a marker is never optimized, so
+                    // every element carries its own marker; for an object the
+                    // byte tested above is the first byte of the key
+                    if (top.is_object)
+                    {
+                        key.clear();
+                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
+                        {
+                            return false;
+                        }
+                        prefix = get_ignore_noop();
+                    }
+                    else
+                    {
+                        prefix = current;
+                    }
+                    break;
+                }
+
+                const bool is_object = top.is_object;
+                container_stack.pop_back();
+                if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array()))
+                {
+                    return false;
+                }
+                if (container_stack.empty())
+                {
+                    return true;
+                }
+                // the container that just ended was an element of the one
+                // below it, which may need the next byte for its own test
+                if (container_stack.back().remaining == npos)
+                {
+                    get_ignore_noop();
+                }
+            }
+        }
     }
 
     /*!
@@ -2941,53 +3042,22 @@ class binary_reader
                                         exception_message(input_format, "excessive array size", "size"), nullptr));
             }
 
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first)))
+            if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second)))
             {
                 return false;
             }
 
-            if (size_and_type.second != 0)
+            if (size_and_type.second == 'N')
             {
-                if (size_and_type.second != 'N')
+                // a no-op is not a value, so a container of them holds none;
-                {
+                // the declared size has already been passed to the SAX parser
-                    for (std::size_t i = 0; i < size_and_type.first; ++i)
+                container_stack.back().remaining = 0;
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
-                        {
-                            return false;
-                        }
-                    }
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                }
-            }
-        }
-        else
-        {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
-            {
-                return false;
             }
 
-            while (current != ']')
+            return true;
-            {
-                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false)))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-            }
         }
 
-        return sax->end_array();
+        return enter_array(detail::unknown_size());
     }
 
     /*!
@@ -3009,68 +3079,12 @@ class binary_reader
                                     exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr));
         }
 
-        string_t key;
         if (size_and_type.first != npos)
         {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first)))
+            return enter_object(size_and_type.first, size_and_type.second);
-            {
-                return false;
         }
 
-            if (size_and_type.second != 0)
+        return enter_object(detail::unknown_size());
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-        }
-        else
-        {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
-            {
-                return false;
-            }
-
-            while (current != '}')
-            {
-                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
-                {
-                    return false;
-                }
-                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-                key.clear();
-            }
-        }
-
-        return sax->end_object();
     }
 
     // Note, no reader for UBJSON binary types is implemented because they do

single_include/nlohmann/json.hpp

@@ -10886,11 +10886,21 @@ class binary_reader
     */
     struct container_frame
     {
-        container_frame(const std::size_t remaining_, const bool is_object_) noexcept
+        container_frame(const std::size_t remaining_, const bool is_object_,
-            : remaining(remaining_), is_object(is_object_) {}
+                        const char_int_type type_marker_ = 0) noexcept
+            : remaining(remaining_), type_marker(type_marker_), is_object(is_object_) {}
 
-        /// number of elements that have not been read yet
+        /// number of elements that have not been read yet, or npos when the
+        /// container is not sized and ends at a marker instead
         std::size_t remaining;
+        /// BSON: value of chars_read before this document's size prefix, which
+        /// check_bson_document_size() needs once the document has been read
+        std::size_t start_position = 0;
+        /// UBJSON/BJData: the type marker of an optimized container, so that
+        /// its elements are read without one of their own; 0 otherwise
+        char_int_type type_marker;
+        /// BSON: the size this document declares, in bytes
+        std::int32_t declared_size = 0;
         /// whether to close this container with end_object() or end_array()
         bool is_object;
     };
@@ -10907,27 +10917,28 @@ class binary_reader
 
     @return whether the SAX parser accepted the start event
     */
-    bool enter_container(const bool is_object, const std::size_t len)
+    bool enter_container(const bool is_object, const std::size_t len,
+                         const char_int_type type_marker = 0)
     {
         if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->start_object(len) : !sax->start_array(len)))
         {
             return false;
         }
 
-        container_stack.emplace_back(len, is_object);
+        container_stack.emplace_back(len, is_object, type_marker);
         return true;
     }
 
     /// @copydoc enter_container
-    bool enter_array(const std::size_t len)
+    bool enter_array(const std::size_t len, const char_int_type type_marker = 0)
     {
-        return enter_container(/*is_object*/false, len);
+        return enter_container(/*is_object*/false, len, type_marker);
     }
 
     /// @copydoc enter_container
-    bool enter_object(const std::size_t len)
+    bool enter_object(const std::size_t len, const char_int_type type_marker = 0)
     {
-        return enter_container(/*is_object*/true, len);
+        return enter_container(/*is_object*/true, len, type_marker);
     }
 
     //////////
@@ -10964,8 +10975,10 @@ class binary_reader
     @brief Reads in a BSON-object and passes it to the SAX-parser.
     @return whether a valid BSON-value was passed to the SAX parser
     */
-    bool parse_bson_internal()
+    bool open_bson_document(const bool is_object)
     {
+        // recorded before the size prefix is read, because
+        // check_bson_document_size() measures the document from here
         const std::size_t document_start = chars_read;
         std::int32_t document_size{};
         if (!get_number<std::int32_t, true>(input_format_t::bson, document_size))
@@ -10973,22 +10986,89 @@ class binary_reader
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
+        if (JSON_HEDLEY_UNLIKELY(!enter_container(is_object, detail::unknown_size())))
         {
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/false)))
+        container_frame& frame = container_stack.back();
+        frame.start_position = document_start;
+        frame.declared_size = document_size;
+        return true;
+    }
+
+    /*!
+    @brief read a BSON document and everything nested inside it
+
+    Reads elements until the document that was begun here is complete,
+    resuming the enclosing document each time an embedded one ends, so that
+    the nesting depth of the input costs heap rather than native stack
+    (see #5104).
+
+    @return whether reading the document succeeded
+    */
+    bool parse_bson_internal()
+    {
+        if (JSON_HEDLEY_UNLIKELY(!open_bson_document(/*is_object*/true)))
         {
             return false;
         }
 
-        if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
+        // the key currently being read; hoisted out of the loop so that its
+        // capacity is reused across elements and across nesting levels
+        string_t key;
+
+        while (true)
+        {
+            const auto element_type = get();
+
+            if (element_type == 0) // end of the innermost document
+            {
+                const container_frame& top = container_stack.back();
+                const bool is_object = top.is_object;
+
+                if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(top.start_position, top.declared_size)))
                 {
                     return false;
                 }
 
-        return sax->end_object();
+                container_stack.pop_back();
+                if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array()))
+                {
+                    return false;
+                }
+                // the document begun here is complete once it is not inside one
+                if (container_stack.empty())
+                {
+                    return true;
+                }
+                continue;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list")))
+            {
+                return false;
+            }
+
+            const std::size_t element_type_parse_position = chars_read;
+            key.clear();
+            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
+            {
+                return false;
+            }
+
+            // an array's elements are named "0", "1", ... in the wire format,
+            // and those names are not passed on
+            if (container_stack.back().is_object && !sax->key(key))
+            {
+                return false;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
+            {
+                return false;
+            }
+        }
     }
 
     /*!
@@ -11100,12 +11180,12 @@ class binary_reader
 
             case 0x03: // object
             {
-                return parse_bson_internal();
+                return open_bson_document(/*is_object*/true);
             }
 
             case 0x04: // array
             {
-                return parse_bson_array();
+                return open_bson_document(/*is_object*/false);
             }
 
             case 0x05: // binary
@@ -11155,82 +11235,7 @@ class binary_reader
         }
     }
 
-    /*!
-    @brief Read a BSON element list (as specified in the BSON-spec)
 
-    The same binary layout is used for objects and arrays, hence it must be
-    indicated with the argument @a is_array which one is expected
-    (true --> array, false --> object).
-
-    @param[in] is_array Determines if the element list being read is to be
-                        treated as an object (@a is_array == false), or as an
-                        array (@a is_array == true).
-    @return whether a valid BSON-object/array was passed to the SAX parser
-    */
-    bool parse_bson_element_list(const bool is_array)
-    {
-        string_t key;
-
-        while (auto element_type = get())
-        {
-            if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::bson, "element list")))
-            {
-                return false;
-            }
-
-            const std::size_t element_type_parse_position = chars_read;
-            if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
-            {
-                return false;
-            }
-
-            if (!is_array && !sax->key(key))
-            {
-                return false;
-            }
-
-            if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_internal(element_type, element_type_parse_position)))
-            {
-                return false;
-            }
-
-            // get_bson_cstr only appends
-            key.clear();
-        }
-
-        return true;
-    }
-
-    /*!
-    @brief Reads an array from the BSON input and passes it to the SAX-parser.
-    @return whether a valid BSON-array was passed to the SAX parser
-    */
-    bool parse_bson_array()
-    {
-        const std::size_t document_start = chars_read;
-        std::int32_t document_size{};
-        if (!get_number<std::int32_t, true>(input_format_t::bson, document_size))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!parse_bson_element_list(/*is_array*/true)))
-        {
-            return false;
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!check_bson_document_size(document_start, document_size)))
-        {
-            return false;
-        }
-
-        return sax->end_array();
-    }
 
     //////////
     // CBOR //
@@ -12856,7 +12861,103 @@ class binary_reader
     */
     bool parse_ubjson_internal(const bool get_char = true)
     {
-        return get_ubjson_value(get_char ? get_ignore_noop() : current);
+        // the key currently being read; hoisted out of the loop so that its
+        // capacity is reused across elements and across nesting levels
+        string_t key;
+
+        // the type marker of the value to read next
+        char_int_type prefix = get_char ? get_ignore_noop() : current;
+
+        while (true)
+        {
+            const std::size_t depth = container_stack.size();
+
+            if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(prefix)))
+            {
+                return false;
+            }
+
+            // the value begun here is complete once it is not inside anything
+            if (container_stack.empty())
+            {
+                return true;
+            }
+
+            // a value was completed rather than a container opened; a
+            // container that ends at a marker needs the next byte to test
+            if (container_stack.size() == depth && container_stack.back().remaining == npos)
+            {
+                get_ignore_noop();
+            }
+
+            // advance to the next element, closing the containers that ended.
+            // The reference is not held across get_ubjson_value() above, which
+            // can push onto the stack and reallocate it.
+            for (;;)
+            {
+                container_frame& top = container_stack.back();
+
+                if (top.remaining != npos)
+                {
+                    if (top.remaining != 0)
+                    {
+                        --top.remaining;
+                        if (top.is_object)
+                        {
+                            key.clear();
+                            if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
+                            {
+                                return false;
+                            }
+                        }
+                        // an optimized container gives its elements no marker
+                        prefix = (top.type_marker != 0) ? top.type_marker : get_ignore_noop();
+                        break;
+                    }
+                }
+                // the end marker is compared against a literal rather than
+                // against a conditional expression, because char_int_type is
+                // unsigned for some input adapters and MSVC then reports the
+                // comparison as a signed/unsigned mismatch
+                else if (top.is_object ? (current != '}') : (current != ']'))
+                {
+                    // a container that ends at a marker is never optimized, so
+                    // every element carries its own marker; for an object the
+                    // byte tested above is the first byte of the key
+                    if (top.is_object)
+                    {
+                        key.clear();
+                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
+                        {
+                            return false;
+                        }
+                        prefix = get_ignore_noop();
+                    }
+                    else
+                    {
+                        prefix = current;
+                    }
+                    break;
+                }
+
+                const bool is_object = top.is_object;
+                container_stack.pop_back();
+                if (JSON_HEDLEY_UNLIKELY(is_object ? !sax->end_object() : !sax->end_array()))
+                {
+                    return false;
+                }
+                if (container_stack.empty())
+                {
+                    return true;
+                }
+                // the container that just ended was an element of the one
+                // below it, which may need the next byte for its own test
+                if (container_stack.back().remaining == npos)
+                {
+                    get_ignore_noop();
+                }
+            }
+        }
     }
 
     /*!
@@ -13628,53 +13729,22 @@ class binary_reader
                                         exception_message(input_format, "excessive array size", "size"), nullptr));
             }
 
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first)))
+            if (JSON_HEDLEY_UNLIKELY(!enter_array(size_and_type.first, size_and_type.second)))
             {
                 return false;
             }
 
-            if (size_and_type.second != 0)
+            if (size_and_type.second == 'N')
             {
-                if (size_and_type.second != 'N')
+                // a no-op is not a value, so a container of them holds none;
-                {
+                // the declared size has already been passed to the SAX parser
-                    for (std::size_t i = 0; i < size_and_type.first; ++i)
+                container_stack.back().remaining = 0;
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
-                        {
-                            return false;
-                        }
-                    }
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                }
-            }
-        }
-        else
-        {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_array(detail::unknown_size())))
-            {
-                return false;
             }
 
-            while (current != ']')
+            return true;
-            {
-                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal(false)))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-            }
         }
 
-        return sax->end_array();
+        return enter_array(detail::unknown_size());
     }
 
     /*!
@@ -13696,68 +13766,12 @@ class binary_reader
                                     exception_message(input_format, "BJData object does not support ND-array size in optimized format", "object"), nullptr));
         }
 
-        string_t key;
         if (size_and_type.first != npos)
         {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(size_and_type.first)))
+            return enter_object(size_and_type.first, size_and_type.second);
-            {
-                return false;
         }
 
-            if (size_and_type.second != 0)
+        return enter_object(detail::unknown_size());
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second)))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-            else
-            {
-                for (std::size_t i = 0; i < size_and_type.first; ++i)
-                {
-                    if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key) || !sax->key(key)))
-                    {
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                    {
-                        return false;
-                    }
-                    key.clear();
-                }
-            }
-        }
-        else
-        {
-            if (JSON_HEDLEY_UNLIKELY(!sax->start_object(detail::unknown_size())))
-            {
-                return false;
-            }
-
-            while (current != '}')
-            {
-                if (JSON_HEDLEY_UNLIKELY(!get_ubjson_string(key, false) || !sax->key(key)))
-                {
-                    return false;
-                }
-                if (JSON_HEDLEY_UNLIKELY(!parse_ubjson_internal()))
-                {
-                    return false;
-                }
-                get_ignore_noop();
-                key.clear();
-            }
-        }
-
-        return sax->end_object();
     }
 
     // Note, no reader for UBJSON binary types is implemented because they do

tests/src/unit-bson.cpp

@@ -1011,6 +1011,91 @@ TEST_CASE("BSON document size mismatch")
     }
 }
 
+TEST_CASE("BSON nesting does not consume the call stack")
+{
+    // An embedded document or array used to be read by calling back into the
+    // document reader, so the native call stack grew with the nesting depth of
+    // the input (#5104). The open documents are kept on a heap stack now.
+    //
+    // Deeply nested values must not be compared, copied or dumped here: those
+    // operations are still recursive and would reintroduce the crash.
+
+    // A document nested deeply enough to have crashed. The bytes are built
+    // here rather than with to_bson(), because the writer still recurses once
+    // per level and would overflow the stack before the reader is ever
+    // reached. Every level is
+    //     <int32 size> 0x03 'a' 0x00 <inner document> 0x00
+    // so a level is eight bytes larger than the one it holds, and the sizes
+    // can be filled in from the outside in.
+    const std::size_t depth = 30000;
+    std::vector<uint8_t> input;
+    input.reserve(5 + (8 * depth));
+    for (std::size_t i = 0; i < depth; ++i)
+    {
+        const auto size = static_cast<std::uint32_t>(5 + (8 * (depth - i)));
+        input.push_back(static_cast<uint8_t>(size & 0xFF));
+        input.push_back(static_cast<uint8_t>((size >> 8) & 0xFF));
+        input.push_back(static_cast<uint8_t>((size >> 16) & 0xFF));
+        input.push_back(static_cast<uint8_t>((size >> 24) & 0xFF));
+        input.push_back(0x03); // embedded document
+        input.push_back('a');
+        input.push_back(0x00);
+    }
+    // the innermost document is empty, then one terminator closes each level
+    input.insert(input.end(), {0x05, 0x00, 0x00, 0x00, 0x00});
+    input.insert(input.end(), depth, 0x00);
+
+    SECTION("a well-formed deep document is read through the SAX interface")
+    {
+        SaxCountdown accept_all(1000000);
+        CHECK(json::sax_parse(input, &accept_all, json::input_format_t::bson));
+    }
+
+    SECTION("a well-formed deep document is read into a value")
+    {
+        json j = json::from_bson(input);
+
+        // walked rather than compared: comparing, copying or dumping a value
+        // this deep is still recursive
+        std::size_t measured = 0;
+        const json* q = &j;
+        while (q->is_object() && !q->empty())
+        {
+            q = &q->begin().value();
+            ++measured;
+        }
+        CHECK(measured == depth);
+    }
+
+    SECTION("embedded documents and arrays are still read the same way")
+    {
+        const json values = {{"a", {{"b", {{"c", 1}}}}}};
+        CHECK(json::from_bson(json::to_bson(values)) == values);
+
+        const json array = {{"a", {1, 2, 3}}};
+        CHECK(json::from_bson(json::to_bson(array)) == array);
+
+        const json mixed = {{"a", {json{{"x", 1}}, json{{"y", 2}}}}};
+        CHECK(json::from_bson(json::to_bson(mixed)) == mixed);
+
+        CHECK(json::from_bson(json::to_bson(json::object())) == json::object());
+    }
+
+    SECTION("a size that does not match is still reported per document")
+    {
+        // the embedded document claims one byte too many
+        std::vector<uint8_t> const bad =
+        {
+            0x15, 0x00, 0x00, 0x00, 0x03, 'a', 0x00,
+            0x0D, 0x00, 0x00, 0x00, 0x08, 'b', 0x00, 0x01, 0x00,
+            0x00
+        };
+        json _;
+        CHECK_THROWS_AS(_ = json::from_bson(bad), json::parse_error&);
+        CHECK(json::from_bson(bad, true, false).is_discarded());
+    }
+}
+
 TEST_CASE("BSON numerical data")
 {
     SECTION("number")

tests/src/unit-ubjson.cpp

@@ -2149,6 +2149,111 @@ TEST_CASE("UBJSON")
     }
 }
 
+TEST_CASE("UBJSON nesting does not consume the call stack")
+{
+    // Containers used to be read by calling back into the value reader once
+    // per element, so the native call stack grew with the nesting depth of the
+    // input. '[' alone opens a container, so a payload of repeated '[' crashed
+    // the process (#5104), as did the optimized forms, which reach the same
+    // path through a type or size annotation. The containers are kept on a
+    // heap stack now.
+    //
+    // Deeply nested values must not be compared, copied or dumped here: those
+    // operations are still recursive and would reintroduce the crash.
+    json _;
+
+    SECTION("containers that end at a marker")
+    {
+        const std::vector<uint8_t> input(500000, '[');
+        CHECK_THROWS_WITH_AS(_ = json::from_ubjson(input), "[json.exception.parse_error.110] parse error at byte 500001: syntax error while parsing UBJSON value: unexpected end of input", json::parse_error&);
+        CHECK(json::from_ubjson(input, true, false).is_discarded());
+    }
+
+    SECTION("containers with a size")
+    {
+        std::vector<uint8_t> input;
+        for (std::size_t i = 0; i < 100000; ++i)
+        {
+            input.push_back('[');
+            input.push_back('#');
+            input.push_back('i');
+            input.push_back(1);
+        }
+        CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
+        CHECK(json::from_ubjson(input, true, false).is_discarded());
+    }
+
+    SECTION("containers with a type and a size")
+    {
+        // '[' is a permitted optimized type in UBJSON, so each element of such
+        // a container is itself a container, read without a marker of its own
+        std::vector<uint8_t> input;
+        for (std::size_t i = 0; i < 100000; ++i)
+        {
+            const std::vector<uint8_t> level = {'[', '$', '[', '#', 'i', 1};
+            input.insert(input.end(), level.begin(), level.end());
+        }
+        CHECK_THROWS_AS(_ = json::from_ubjson(input), json::parse_error&);
+        CHECK(json::from_ubjson(input, true, false).is_discarded());
+    }
+
+    SECTION("a well-formed deep value is read through the SAX interface")
+    {
+        std::vector<uint8_t> input(100000, '[');
+        input.insert(input.end(), 100000, ']');
+
+        SaxCountdown accept_all(1000000);
+        CHECK(json::sax_parse(input, &accept_all, json::input_format_t::ubjson));
+    }
+
+    SECTION("a well-formed deep value is read into a value")
+    {
+        const std::size_t depth = 10000;
+        std::vector<uint8_t> input(depth, '[');
+        input.insert(input.end(), depth, ']');
+
+        json j = json::from_ubjson(input);
+
+        std::size_t measured = 0;
+        const json* p = &j;
+        while (p->is_array() && !p->empty())
+        {
+            p = &p->front();
+            ++measured;
+        }
+        // the innermost array is empty, so the descent stops one level short
+        CHECK(measured == depth - 1);
+    }
+
+    SECTION("containers are still read the same way")
+    {
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', ']'})) == json::array());
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '}'})) == json::object());
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 0})) == json::array());
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '#', 'i', 0})) == json::object());
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 2, 'i', 1, 'i', 2})) == json({1, 2}));
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'{', '$', 'i', '#', 'i', 1, 'i', 1, 'a', 1})) == json({{"a", 1}}));
+        // a no-op is not a value, so a container of them holds none
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', 'N', '#', 'i', 2})) == json::array());
+        // sized and unsized forms nested inside one another
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '[', '#', 'i', 2, 'i', 1, 'i', 2, ']'})) == json({{1, 2}}));
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '#', 'i', 1, '[', 'i', 1, ']'})) == json({{1}}));
+        // an optimized container of containers
+        CHECK(json::from_ubjson(std::vector<uint8_t>({'[', '$', '[', '#', 'i', 2, 'i', 1, ']', 'i', 2, ']'})) == json({{1}, {2}}));
+    }
+
+    SECTION("BJData containers are still read the same way")
+    {
+        // the ND-array wrapper and the binary shortcut are complete values,
+        // not containers the reader descends into
+        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6})) ==
+        json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}));
+        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '$', 'i', '#', 'i', 2, 1, 2})) == json({1, 2}));
+        CHECK(json::from_bjdata(std::vector<uint8_t>({'[', '[', 'i', 1, ']', ']'})) == json({{1}}));
+    }
+}
+
 TEST_CASE("UBJSON optimized arrays of a valueless type are bounded")
 {
     // An element of type 'Z', 'T' or 'F' is encoded by its marker alone, so an