Copy nested values into a document without recursion

Counting and copying the nodes of a view or a basic_json value into an
editable document recursed once per nesting level, so that a deeply
nested value overflowed the stack. The four functions now walk the value
with an explicit stack, as materialize() does.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/edit.hpp

@@ -17,6 +17,7 @@
 #include <string> // string, to_string
 #include <type_traits> // decay, enable_if, integral_constant, is_arithmetic, is_convertible, is_floating_point, is_same, is_signed
 #include <utility> // forward
+#include <vector> // vector
 
 #include <nlohmann/json.hpp>
 #include <nlohmann/detail/view/document_data.hpp>
@@ -663,100 +664,197 @@ class editor
         }
     }
 
+    // The subtrees are walked with an explicit stack (as materialize() does):
+    // the nesting depth is limited by memory only, not by the call stack.
+
     /// number of nodes of a subtree (containers, keys, scalars)
     template<bool E>
     static std::size_t count_nodes(const document_data& d, const node* n)
     {
-        if (!is_container(*n))
+        using walk = navigation<E>;
+        struct frame
         {
-            return 1;
+            const node* pos; ///< next element, or key of the next member
-        }
+            const node* end;
-        const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
+            bool object;
-        std::size_t r = 1;
+        };
-        for (const node* c = navigation<E>::first(d, n), *end = navigation<E>::end(d, n); c != end;)
+        std::vector<frame> open;
+        std::size_t r = 0;
+        for (;;)
         {
-            const node* const v = object ? c + 1 : c;
+            ++r;
-            r += (object ? 1 : 0) + count_nodes<E>(d, navigation<E>::value(v));
+            if (is_container(*n))
-            c = document_data::after(v);
+            {
+                open.push_back(frame{walk::first(d, n), walk::end(d, n), n->kind == static_cast<std::uint8_t>(value_t::object)});
+            }
+            // the next value: close finished containers, then step over the key
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return r;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    open.pop_back();
+                    continue;
+                }
+                const node* v = f.pos;
+                if (f.object)
+                {
+                    ++r; // the key
+                    ++v;
+                }
+                f.pos = document_data::after(v);
+                n = walk::value(v);
+                break;
+            }
         }
-        return r;
     }
 
-    /// copy a subtree (of any document) as a contiguous sequence; returns its end
+    /// copy a subtree (of any document) as a contiguous sequence of
+    /// count_nodes() nodes
     template<bool E>
-    node* fill_nodes(const document_data& d, const node* n, node* out)
+    void fill_nodes(const document_data& d, const node* n, node* out)
     {
-        if (!is_container(*n))
+        using walk = navigation<E>;
+        struct frame
         {
-            *out = copy_scalar(d, *n);
+            const node* pos; ///< next element, or key of the next member
-            return out + 1;
+            const node* end;
-        }
+            bool object;
-        node* const self = out++;
+            node* self;      ///< the container in the copy
-        *self = plain_node(static_cast<value_t>(n->kind));
+        };
-        self->len = n->len;
+        std::vector<frame> open;
-        const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
+        for (;;)
-        for (const node* c = navigation<E>::first(d, n), *end = navigation<E>::end(d, n); c != end;)
         {
-            if (object)
+            if (is_container(*n))
             {
-                *out++ = copy_scalar(d, *c);
+                node* const self = out++;
-                ++c;
+                *self = plain_node(static_cast<value_t>(n->kind));
+                self->len = n->len;
+                open.push_back(frame{walk::first(d, n), walk::end(d, n), n->kind == static_cast<std::uint8_t>(value_t::object), self});
+            }
+            else
+            {
+                *out++ = copy_scalar(d, *n);
+            }
+            // the next value: close finished containers, then copy the key
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    f.self->next = static_cast<std::uint32_t>(out - f.self);
+                    open.pop_back();
+                    continue;
+                }
+                const node* v = f.pos;
+                if (f.object)
+                {
+                    *out++ = copy_scalar(d, *v);
+                    ++v;
+                }
+                f.pos = document_data::after(v);
+                n = walk::value(v);
+                break;
             }
-            out = fill_nodes<E>(d, navigation<E>::value(c), out);
-            c = document_data::after(c);
         }
-        self->next = static_cast<std::uint32_t>(out - self);
-        return out;
     }
 
     static std::size_t count_nodes(const BasicJsonType& j)
     {
-        std::size_t r = 1;
+        using iterator = typename BasicJsonType::const_iterator;
-        if (j.is_object())
+        struct frame
         {
-            for (const auto& member : j.items())
+            iterator pos;
+            iterator end;
+            bool object;
+        };
+        std::vector<frame> open;
+        const BasicJsonType* n = &j;
+        std::size_t r = 0;
+        for (;;)
+        {
+            ++r;
+            if (n->is_structured())
             {
-                r += 1 + count_nodes(member.value());
+                open.push_back(frame{n->cbegin(), n->cend(), n->is_object()});
+            }
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return r;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    open.pop_back();
+                    continue;
+                }
+                r += f.object ? 1 : 0; // the key
+                n = &*f.pos;
+                ++f.pos;
+                break;
             }
         }
-        else if (j.is_array())
-        {
-            for (const auto& e : j)
-            {
-                r += count_nodes(e);
-            }
-        }
-        return r;
     }
 
-    node* fill_nodes(const BasicJsonType& j, node* out)
+    void fill_nodes(const BasicJsonType& j, node* out)
     {
-        if (!j.is_structured())
+        using iterator = typename BasicJsonType::const_iterator;
+        struct frame
         {
-            *out = json_scalar(j);
+            iterator pos;
-            return out + 1;
+            iterator end;
-        }
+            bool object;
-        node* const self = out++;
+            node* self; ///< the container in the copy
-        *self = plain_node(j.type());
+        };
-        self->len = static_cast<std::uint32_t>(j.size());
+        std::vector<frame> open;
-        if (j.is_object())
+        const BasicJsonType* n = &j;
+        for (;;)
         {
-            for (const auto& member : j.items())
+            if (n->is_structured())
             {
-                check_utf8(member.key().data(), member.key().size());
+                node* const self = out++;
-                *out++ = string_node(member.key().data(), member.key().size());
+                *self = plain_node(n->type());
-                out = fill_nodes(member.value(), out);
+                self->len = static_cast<std::uint32_t>(n->size());
+                open.push_back(frame{n->cbegin(), n->cend(), n->is_object(), self});
+            }
+            else
+            {
+                *out++ = json_scalar(*n);
+            }
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    f.self->next = static_cast<std::uint32_t>(out - f.self);
+                    open.pop_back();
+                    continue;
+                }
+                if (f.object)
+                {
+                    const auto& key = f.pos.key();
+                    check_utf8(key.data(), key.size());
+                    *out++ = string_node(key.data(), key.size());
+                }
+                n = &*f.pos;
+                ++f.pos;
+                break;
             }
         }
-        else
-        {
-            for (const auto& e : j)
-            {
-                out = fill_nodes(e, out);
-            }
-        }
-        self->next = static_cast<std::uint32_t>(out - self);
-        return out;
     }
 
     document_data& m_doc;

tests/src/unit-json_view_edit.cpp

@@ -563,3 +563,73 @@ TEST_CASE("json_view edits: views and values")
         CHECK(d.root().dump() == "[true,false]");
     }
 }
+
+TEST_CASE("json_view edits: deeply nested values")
+{
+    // copying a value into a document must not recurse per nesting level
+    const std::size_t depth = 100000;
+    const std::string brackets = std::string(depth, '[') + std::string(depth, ']');
+    std::string braces;
+    for (std::size_t i = 0; i < depth; ++i)
+    {
+        braces += "{\"a\":";
+    }
+    braces += '1';
+    braces += std::string(depth, '}');
+
+    SECTION("a view of a read-only document")
+    {
+        const json_document source = json_document::parse(brackets);
+        json_editable_document d = json_editable_document::parse("[]");
+        d.push_back(d.root(), source.root());
+        CHECK(d.root().dump() == "[" + brackets + "]");
+    }
+
+    SECTION("a view of an editable document")
+    {
+        const json_editable_document source = json_editable_document::parse(braces);
+        json_editable_document d = json_editable_document::parse("{}");
+        d.set(d.root(), "deep", source.root());
+        CHECK(d.root().dump() == "{\"deep\":" + braces + "}");
+    }
+
+    SECTION("a view of an edited document (values behind links)")
+    {
+        const json_document source = json_document::parse(brackets);
+        json_editable_document edited = json_editable_document::parse("[[]]");
+        edited.push_back(edited.root()[0], source.root());
+        edited.push_back(edited.root(), source.root());
+        json_editable_document d = json_editable_document::parse("null");
+        d.set(d.root(), edited.root());
+        CHECK(d.root().dump() == "[[" + brackets + "]," + brackets + "]");
+    }
+
+    SECTION("a basic_json value")
+    {
+        json deep = json::array();
+        json* inner = &deep;
+        for (std::size_t i = 1; i < depth; ++i)
+        {
+            inner->push_back(json::array());
+            inner = &inner->back();
+        }
+        json_editable_document d = json_editable_document::parse("[]");
+        d.push_back(d.root(), deep);
+        CHECK(d.root().dump() == "[" + brackets + "]");
+    }
+
+    SECTION("a basic_json value with objects")
+    {
+        json deep = 1;
+        for (std::size_t i = 0; i < depth; ++i)
+        {
+            json outer = json::object();
+            outer["a"] = std::move(deep);
+            deep = std::move(outer);
+        }
+        json_editable_document d = json_editable_document::parse("{}");
+        d.set(d.root(), "deep", deep);
+        CHECK(d.root().dump() == "{\"deep\":" + braces + "}");
+    }
+}
+