Merge branch 'json-view/19-edit-set' into json-view/21-images

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/builder.hpp

@@ -9,7 +9,7 @@
 
 #pragma once
 
-#include <algorithm> // find, find_if, max
+#include <algorithm> // find, find_if, max, min
 #include <array> // array
 #include <cstddef> // size_t, ptrdiff_t
 #include <cstdint> // int64_t, uint8_t, uint16_t, uint32_t, uint64_t
@@ -202,8 +202,18 @@ class builder
         const std::uint64_t done = static_cast<std::uint64_t>(at - b) + 1;
         const std::uint64_t guess = static_cast<std::uint64_t>(n) * static_cast<std::uint64_t>(e - b + 1) / done;
         const std::uint64_t grown = guess + (guess / 4) + 64; // a variable: GCC calls a cast of the sum useless where std::uint64_t is std::size_t
+        // (n is below 2^32: the input is smaller than 4 GiB; the sum cannot wrap)
+        const std::uint64_t wanted = (std::max)(grown, static_cast<std::uint64_t>(n) + (n / 2) + 64);
+        const std::uint64_t limit = document_data::max_nodes();
         doc.tape_size = n;
-        doc.reserve((std::max)(static_cast<std::size_t>(grown), n + (n / 2) + 64));
+        // LCOV_EXCL_START (a node array that fills the address space)
+        if (NLOHMANN_VIEW_UNLIKELY(n >= limit))
+        {
+            document_data::throw_bad_alloc(); // no room for another node
+        }
+        // LCOV_EXCL_STOP
+        // (a count beyond the limit is cut: the index does not grow beyond what can be addressed)
+        doc.reserve(static_cast<std::size_t>((std::min)(wanted, limit)));
         return doc.tape;
     }
 
@@ -816,7 +826,10 @@ indent_done:
             n->flags = flags;
             n->extra = extra;
             n->off = static_cast<std::uint32_t>(off);
-            set_integer_bits(*n, second);
+            // len is the low half of the second word, next the high half
+            // (not a native word over both, which swaps them on big-endian)
+            n->len = static_cast<std::uint32_t>(second);
+            n->next = static_cast<std::uint32_t>(second >> 32);
 #endif
             return n;
         }

include/nlohmann/detail/view/document_data.hpp

@@ -13,9 +13,10 @@
 #include <cstdint> // uint8_t, uint32_t
 #include <cstring> // memcpy
 #include <functional> // less
+#include <limits> // numeric_limits
 #include <map> // map
 #include <memory> // unique_ptr
-#include <new> // operator new, placement new
+#include <new> // bad_alloc, operator new, placement new
 #include <string> // string
 #include <vector> // vector
 
@@ -123,13 +124,30 @@ struct document_data
         tape_cap = inline_cap;
     }
 
-    /// make room for n nodes; keeps the first tape_size nodes
+    /// the largest node count whose size in bytes fits a std::size_t
+    static constexpr std::size_t max_nodes() noexcept
+    {
+        return (std::numeric_limits<std::size_t>::max)() / sizeof(node);
+    }
+
+    [[noreturn]] NLOHMANN_VIEW_NOINLINE static void throw_bad_alloc()
+    {
+        NLOHMANN_VIEW_THROW(std::bad_alloc());
+    }
+
+    /// make room for n nodes; keeps the first tape_size nodes (throws
+    /// std::bad_alloc for a count that does not fit the address space,
+    /// instead of wrapping around in n * sizeof(node))
     void reserve(std::size_t n)
     {
         if (n <= tape_cap)
         {
             return;
         }
+        if (NLOHMANN_VIEW_UNLIKELY(n > max_nodes()))
+        {
+            throw_bad_alloc();
+        }
         node* fresh = static_cast<node*>(::operator new (n * sizeof(node)));
         if (tape_size != 0)
         {

include/nlohmann/detail/view/edit.hpp

@@ -17,6 +17,7 @@
 #include <string> // string, to_string
 #include <type_traits> // decay, enable_if, integral_constant, is_arithmetic, is_convertible, is_floating_point, is_same, is_signed
 #include <utility> // forward
+#include <vector> // vector
 
 #include <nlohmann/json.hpp>
 #include <nlohmann/detail/view/document_data.hpp>
@@ -151,25 +152,23 @@ class editor
         {
             become_empty(o, value_t::object);
         }
-        // an existing member: assign it (and drop later duplicates, so that
-        // lookups, iteration, and materialize() agree)
+        // an existing member: assign the one that lookups find (the last
+        // one, should the key occur more than once), and drop the others, so
+        // that lookups, iteration, and materialize() agree. The key stays at
+        // the position of its first occurrence, as materialize() puts it.
         node* slot = nullptr;
-        bool duplicates = false;
+        std::size_t matches = 0;
         for (const node* k = nav::first(m_doc, o), *end = nav::end(m_doc, o); k != end; k = document_data::after(k + 1))
         {
             if (key_equals(*k, key))
             {
-                if (slot != nullptr)
-                {
-                    duplicates = true;
-                    break;
-                }
                 slot = const_cast<node*>(nav::value(k + 1)); // NOLINT(cppcoreguidelines-pro-type-const-cast): the nodes belong to this document
+                ++matches;
             }
         }
         if (slot != nullptr)
         {
-            if (duplicates)
+            if (matches > 1)
             {
                 erase_members(o, key, true);
             }
@@ -315,27 +314,38 @@ class editor
         return k.len == key.size() && (key.size() == 0 || std::memcmp(m_doc.str(k), key.data(), key.size()) == 0);
     }
 
-    /// remove the members with this key (all, or all but the first) from an object
-    std::size_t erase_members(node* o, string_view_t key, bool keep_first)
+    /// Remove the members with this key from an object: all of them, or all
+    /// but one. That one stays where the first occurrence is, but holds the
+    /// value of the last (the one that lookups find, which views may refer to).
+    std::size_t erase_members(node* o, string_view_t key, bool keep_one)
     {
         node* const h = block_of(m_doc, o, 0);
+        node last_value{}; // the entry of the value of the last member
+        node* const end = h + h->next;
+        if (keep_one)
+        {
+            for (node* r = h + 1; r != end; r += 2)
+            {
+                if (key_equals(*r, key))
+                {
+                    last_value = r[1];
+                }
+            }
+        }
         node* w = h + 1;
         std::size_t erased = 0;
         bool kept = false;
-        for (node* r = h + 1, *end = h + h->next; r != end; r += 2)
+        for (node* r = h + 1; r != end; r += 2)
         {
             const bool match = key_equals(*r, key);
-            if (match && (kept || !keep_first))
+            if (match && (kept || !keep_one))
             {
                 ++erased;
                 continue;
             }
+            w[0] = r[0];
+            w[1] = match ? last_value : r[1];
             kept = kept || match;
-            if (w != r)
-            {
-                w[0] = r[0];
-                w[1] = r[1];
-            }
             w += 2;
         }
         h->next = static_cast<std::uint32_t>(w - h);
@@ -347,9 +357,10 @@ class editor
     /// turn a null into an empty array/object in place
     static void become_empty(node* n, value_t k) noexcept
     {
+        const std::uint8_t linked = n->flags & node_flags::linked;
         *n = node{};
         n->kind = static_cast<std::uint8_t>(k);
-        n->flags = node_flags::is_new;
+        n->flags = static_cast<std::uint8_t>(node_flags::is_new | linked);
         n->next = 1;
     }
 
@@ -357,13 +368,17 @@ class editor
     /// include slot (if known).
     void assign(node* slot, const encoded& e, node* parent, bool parent_known)
     {
+        // an entry of a moved sequence links to the slot: it can take any extent
+        const std::uint8_t linked = slot->flags & node_flags::linked;
         if (e.region == nullptr)
         {
-            if (is_container(*slot) && slot->next > 1 && slot != m_doc.tape)
+            if (is_container(*slot) && slot->next > 1 && slot != m_doc.tape && linked == 0)
             {
                 // The slot spans its old elements in the enclosing sequence, but
                 // a scalar is one node: the enclosing container first switches to
-                // links (then the extent of the slot no longer matters).
+                // links (then the extent of the slot no longer matters). Looking
+                // for the container is linear in the size of the document, so
+                // links (which are marked in the slot) avoid it.
                 node* const p = parent_known ? parent : find_parent(m_doc, slot);
                 if (p != nullptr && ((p->flags & node_flags::moved) == 0 || moved_capacity(m_doc, p) == 0))
                 {
@@ -371,6 +386,7 @@ class editor
                 }
             }
             *slot = e.scalar;
+            slot->flags = static_cast<std::uint8_t>(slot->flags | linked);
             return;
         }
         // an array/object: the slot keeps its extent (so that the enclosing
@@ -379,11 +395,21 @@ class editor
         const node* const r = e.region;
         const std::uint32_t extent = is_container(*slot) ? slot->next : 1;
         const bool was_moved = (slot->flags & node_flags::moved) != 0;
+        // Everything that can throw happens before the slot changes: a slot
+        // that is a container without the moved flag would show its old
+        // elements. reserve_moved() makes the set_moved() below, which sets
+        // the flag, safe; the entry of `regions` exists already (encode()
+        // added it), so that the assignment at the end does not allocate.
+        if (!was_moved)
+        {
+            reserve_moved(m_doc);
+        }
         slot->kind = r->kind;
         slot->extra = 0;
         slot->len = r->len;
         slot->next = extent;
-        slot->flags = was_moved ? static_cast<std::uint8_t>(node_flags::moved | node_flags::is_new) : std::uint8_t{0};
+        // (set_moved() adds the moved flag to a slot that does not have it yet)
+        slot->flags = static_cast<std::uint8_t>((was_moved ? node_flags::moved | node_flags::is_new : 0) | linked);
         set_moved(m_doc, slot, e.region, 0);
         edit_state_of(m_doc).regions[e.region] = slot;
     }
@@ -600,6 +626,8 @@ class editor
         switch (static_cast<value_t>(n.kind))
         {
             case value_t::string:
+                // (an editable document only holds valid UTF-8, whatever the check of the other document was)
+                check_utf8(from.str(n), n.len);
                 return string_node(from.str(n), n.len);
             case value_t::number_integer:
             case value_t::number_unsigned:
@@ -663,100 +691,197 @@ class editor
         }
     }
 
+    // The subtrees are walked with an explicit stack (as materialize() does):
+    // the nesting depth is limited by memory only, not by the call stack.
+
     /// number of nodes of a subtree (containers, keys, scalars)
     template<bool E>
     static std::size_t count_nodes(const document_data& d, const node* n)
     {
-        if (!is_container(*n))
+        using walk = navigation<E>;
+        struct frame
         {
-            return 1;
-        }
-        const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
-        std::size_t r = 1;
-        for (const node* c = navigation<E>::first(d, n), *end = navigation<E>::end(d, n); c != end;)
+            const node* pos; ///< next element, or key of the next member
+            const node* end;
+            bool object;
+        };
+        std::vector<frame> open;
+        std::size_t r = 0;
+        for (;;)
         {
-            const node* const v = object ? c + 1 : c;
-            r += (object ? 1 : 0) + count_nodes<E>(d, navigation<E>::value(v));
-            c = document_data::after(v);
+            ++r;
+            if (is_container(*n))
+            {
+                open.push_back(frame{walk::first(d, n), walk::end(d, n), n->kind == static_cast<std::uint8_t>(value_t::object)});
+            }
+            // the next value: close finished containers, then step over the key
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return r;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    open.pop_back();
+                    continue;
+                }
+                const node* v = f.pos;
+                if (f.object)
+                {
+                    ++r; // the key
+                    ++v;
+                }
+                f.pos = document_data::after(v);
+                n = walk::value(v);
+                break;
+            }
         }
-        return r;
     }
 
-    /// copy a subtree (of any document) as a contiguous sequence; returns its end
+    /// copy a subtree (of any document) as a contiguous sequence of
+    /// count_nodes() nodes
     template<bool E>
-    node* fill_nodes(const document_data& d, const node* n, node* out)
+    void fill_nodes(const document_data& d, const node* n, node* out)
     {
-        if (!is_container(*n))
+        using walk = navigation<E>;
+        struct frame
         {
-            *out = copy_scalar(d, *n);
-            return out + 1;
-        }
-        node* const self = out++;
-        *self = plain_node(static_cast<value_t>(n->kind));
-        self->len = n->len;
-        const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
-        for (const node* c = navigation<E>::first(d, n), *end = navigation<E>::end(d, n); c != end;)
+            const node* pos; ///< next element, or key of the next member
+            const node* end;
+            bool object;
+            node* self;      ///< the container in the copy
+        };
+        std::vector<frame> open;
+        for (;;)
         {
-            if (object)
+            if (is_container(*n))
             {
-                *out++ = copy_scalar(d, *c);
-                ++c;
+                node* const self = out++;
+                *self = plain_node(static_cast<value_t>(n->kind));
+                self->len = n->len;
+                open.push_back(frame{walk::first(d, n), walk::end(d, n), n->kind == static_cast<std::uint8_t>(value_t::object), self});
+            }
+            else
+            {
+                *out++ = copy_scalar(d, *n);
+            }
+            // the next value: close finished containers, then copy the key
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    f.self->next = static_cast<std::uint32_t>(out - f.self);
+                    open.pop_back();
+                    continue;
+                }
+                const node* v = f.pos;
+                if (f.object)
+                {
+                    *out++ = copy_scalar(d, *v);
+                    ++v;
+                }
+                f.pos = document_data::after(v);
+                n = walk::value(v);
+                break;
             }
-            out = fill_nodes<E>(d, navigation<E>::value(c), out);
-            c = document_data::after(c);
         }
-        self->next = static_cast<std::uint32_t>(out - self);
-        return out;
     }
 
     static std::size_t count_nodes(const BasicJsonType& j)
     {
-        std::size_t r = 1;
-        if (j.is_object())
+        using iterator = typename BasicJsonType::const_iterator;
+        struct frame
         {
-            for (const auto& member : j.items())
+            iterator pos;
+            iterator end;
+            bool object;
+        };
+        std::vector<frame> open;
+        const BasicJsonType* n = &j;
+        std::size_t r = 0;
+        for (;;)
+        {
+            ++r;
+            if (n->is_structured())
             {
-                r += 1 + count_nodes(member.value());
+                open.push_back(frame{n->cbegin(), n->cend(), n->is_object()});
+            }
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return r;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    open.pop_back();
+                    continue;
+                }
+                r += f.object ? 1 : 0; // the key
+                n = &*f.pos;
+                ++f.pos;
+                break;
             }
         }
-        else if (j.is_array())
-        {
-            for (const auto& e : j)
-            {
-                r += count_nodes(e);
-            }
-        }
-        return r;
     }
 
-    node* fill_nodes(const BasicJsonType& j, node* out)
+    void fill_nodes(const BasicJsonType& j, node* out)
     {
-        if (!j.is_structured())
+        using iterator = typename BasicJsonType::const_iterator;
+        struct frame
         {
-            *out = json_scalar(j);
-            return out + 1;
-        }
-        node* const self = out++;
-        *self = plain_node(j.type());
-        self->len = static_cast<std::uint32_t>(j.size());
-        if (j.is_object())
+            iterator pos;
+            iterator end;
+            bool object;
+            node* self; ///< the container in the copy
+        };
+        std::vector<frame> open;
+        const BasicJsonType* n = &j;
+        for (;;)
         {
-            for (const auto& member : j.items())
+            if (n->is_structured())
             {
-                check_utf8(member.key().data(), member.key().size());
-                *out++ = string_node(member.key().data(), member.key().size());
-                out = fill_nodes(member.value(), out);
+                node* const self = out++;
+                *self = plain_node(n->type());
+                self->len = static_cast<std::uint32_t>(n->size());
+                open.push_back(frame{n->cbegin(), n->cend(), n->is_object(), self});
+            }
+            else
+            {
+                *out++ = json_scalar(*n);
+            }
+            for (;;)
+            {
+                if (open.empty())
+                {
+                    return;
+                }
+                frame& f = open.back();
+                if (f.pos == f.end)
+                {
+                    f.self->next = static_cast<std::uint32_t>(out - f.self);
+                    open.pop_back();
+                    continue;
+                }
+                if (f.object)
+                {
+                    const auto& key = f.pos.key();
+                    check_utf8(key.data(), key.size());
+                    *out++ = string_node(key.data(), key.size());
+                }
+                n = &*f.pos;
+                ++f.pos;
+                break;
             }
         }
-        else
-        {
-            for (const auto& e : j)
-            {
-                out = fill_nodes(e, out);
-            }
-        }
-        self->next = static_cast<std::uint32_t>(out - self);
-        return out;
     }
 
     document_data& m_doc;

include/nlohmann/detail/view/edit_storage.hpp

@@ -63,6 +63,23 @@ inline node* alloc_nodes(document_data& d, std::size_t k)
     return r;
 }
 
+/// The capacity of the edit arena after it grows by n bytes (`used` of `cap`
+/// are taken): doubled, or what is needed plus some room, but never more than
+/// the 4 GiB - 1 bytes that the 32-bit offsets of nodes can address. An error
+/// if n more bytes do not fit even then.
+inline std::size_t text_capacity(std::size_t cap, std::size_t used, std::size_t n)
+{
+    constexpr std::size_t limit = 0xFFFFFFFFu;
+    if (NLOHMANN_VIEW_UNLIKELY(used > limit || n > limit - used))
+    {
+        throw_out_of_range(416, "edits of 4 GiB or more are not supported by json_document");
+    }
+    const std::size_t needed = used + n;
+    const std::size_t wanted = needed + (std::min)(limit - needed, std::size_t{256});
+    const std::size_t doubled = cap > limit / 2 ? limit : cap * 2;
+    return (std::max)(doubled, wanted);
+}
+
 /// copy n bytes into the edit arena and return their offset; a new buffer
 /// leaves the old one alive, so that string views into it remain valid
 inline std::uint32_t append_text(document_data& d, const char* s, std::size_t n)
@@ -70,11 +87,7 @@ inline std::uint32_t append_text(document_data& d, const char* s, std::size_t n)
     document_data::edit_state& e = edit_state_of(d);
     if (NLOHMANN_VIEW_UNLIKELY(e.text_cap - e.text_used < n))
     {
-        const std::size_t cap = (std::max)(e.text_cap * 2, e.text_used + n + 256);
-        if (cap > 0xFFFFFFFFu)
-        {
-            throw_out_of_range(416, "edits of 4 GiB or more are not supported by json_document"); // LCOV_EXCL_LINE (4 GiB)
-        }
+        const std::size_t cap = text_capacity(e.text_cap, e.text_used, n);
         std::unique_ptr<char[]> fresh(new char[cap]); // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
         if (e.text_used != 0)
         {
@@ -101,16 +114,12 @@ inline std::size_t moved_capacity(const document_data& d, const node* n) noexcep
     return d.edits->moved_cap[n->off];
 }
 
-/// let container n take its elements from `seq` (header node first)
-inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap)
+/// Make room for one more moved container. This is the part of set_moved()
+/// that can throw: a caller that changes a node before it calls set_moved()
+/// calls this first, so that a failure leaves the node as it was.
+inline void reserve_moved(document_data& d)
 {
     document_data::edit_state& e = edit_state_of(d);
-    if ((n->flags & node_flags::moved) != 0)
-    {
-        e.moved[n->off] = seq;
-        e.moved_cap[n->off] = cap;
-        return;
-    }
     if (e.moved.size() >= 0xFFFFFFFFu)
     {
         throw_out_of_range(416, "more than 4294967295 edited arrays and objects are not supported by json_document"); // LCOV_EXCL_LINE
@@ -121,6 +130,20 @@ inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap)
         e.moved.reserve((2 * e.moved.size()) + 16);
         e.moved_cap.reserve((2 * e.moved.size()) + 16);
     }
+}
+
+/// let container n take its elements from `seq` (header node first); cannot
+/// throw if n is moved already or reserve_moved() was called
+inline void set_moved(document_data& d, node* n, node* seq, std::size_t cap)
+{
+    document_data::edit_state& e = edit_state_of(d);
+    if ((n->flags & node_flags::moved) != 0)
+    {
+        e.moved[n->off] = seq;
+        e.moved_cap[n->off] = cap;
+        return;
+    }
+    reserve_moved(d);
     e.moved.push_back(seq);
     e.moved_cap.push_back(cap);
     n->off = static_cast<std::uint32_t>(e.moved.size() - 1);
@@ -146,6 +169,7 @@ inline node* block_of(document_data& d, node* n, std::size_t extra)
         set_moved(d, n, nh, cap);
         return nh;
     }
+    reserve_moved(d); // (so that set_moved() below cannot throw: the links are marked before)
     const bool object = n->kind == static_cast<std::uint8_t>(value_t::object);
     const std::size_t used = 1 + (static_cast<std::size_t>(n->len) * (object ? 2 : 1));
     const std::size_t cap = used + extra;
@@ -161,7 +185,7 @@ inline node* block_of(document_data& d, node* n, std::size_t extra)
         {
             *o++ = *c++; // the key
         }
-        make_link(*o, document_data::deref(c));
+        make_link(*o, const_cast<node*>(document_data::deref(c))); // NOLINT(cppcoreguidelines-pro-type-const-cast): the nodes belong to the document
         ++o;
         c = document_data::after(c);
     }

include/nlohmann/detail/view/errors.hpp

@@ -66,9 +66,8 @@ template<typename BasicJsonType>
 {
     if (f.code == error_code::input_too_large)
     {
-        // LCOV_EXCL_START (4 GiB)
-        NLOHMANN_VIEW_THROW(out_of_range::create(416, "input of 4 GiB or more is not supported by json_document", nullptr));
-        // LCOV_EXCL_STOP
+        // (the limit is detail::view::max_input_size: 4 GiB minus 16 bytes)
+        NLOHMANN_VIEW_THROW(out_of_range::create(416, "input of 4294967280 bytes or more is not supported by json_document", nullptr));
     }
     const BasicJsonType accepted = BasicJsonType::parse(src, src + size, nullptr, true, ignore_comments, ignore_trailing_commas);
     // LCOV_EXCL_START (only if parse() accepts what the view rejects: a bug)

include/nlohmann/detail/view/input.hpp

@@ -9,7 +9,7 @@
 #pragma once
 
 #include <string> // basic_string, char_traits, string
-#include <type_traits> // decay, integral_constant, is_array, is_lvalue_reference, is_pointer, is_same, remove_reference
+#include <type_traits> // decay, integral_constant, is_array, is_const, is_integral, is_lvalue_reference, is_pointer, is_same, remove_reference
 #include <utility> // forward
 
 #include <nlohmann/json.hpp>
@@ -28,12 +28,12 @@ namespace view
 /// how a document takes its input
 enum class input_kind
 {
-    move_string,  ///< rvalue std::string: owned without a copy
+    move_string,  ///< non-const rvalue std::string: owned without a copy
     c_string,     ///< const char* (NUL-terminated): borrowed
     char_array,   ///< char array (e.g. a string literal): borrowed
     borrow_range, ///< lvalue contiguous byte container, or std::string_view: borrowed
-    copy_range,   ///< rvalue contiguous byte container: copied
-    adapter,      ///< anything else parse() accepts (streams, wide strings, ...): read into a buffer
+    copy_range,   ///< rvalue contiguous byte container (a const rvalue std::string too): copied
+    adapter,      ///< streams, wide strings, and the rest of what the library's input adapter reads: read into a buffer
 };
 
 template<typename InputType>
@@ -52,13 +52,20 @@ struct classify_input
     static constexpr input_kind value =
         std::is_array<R>::value ? input_kind::char_array
         : std::is_pointer<D>::value ? input_kind::c_string
-        : (is_rvalue && std::is_same<D, std::string>::value) ? input_kind::move_string
+        : (is_rvalue && !std::is_const<R>::value && std::is_same<D, std::string>::value) ? input_kind::move_string
         : (is_bytes && (!is_rvalue || is_string_view)) ? input_kind::borrow_range
         : is_bytes ? input_kind::copy_range
         : input_kind::adapter;
     // NOLINTEND(readability-avoid-nested-conditional-operator)
 };
 
+/// an integer type other than bool: a length passed where a flag is expected
+template<typename T>
+struct is_integer_not_bool : std::is_integral<T> {};
+
+template<>
+struct is_integer_not_bool<bool> : std::false_type {};
+
 /// std::basic_string guarantees a NUL at data()[size()] (the parser's sentinel)
 template<typename T>
 struct is_std_string : std::false_type {};

include/nlohmann/detail/view/lookup.hpp

@@ -11,6 +11,8 @@
 #include <cstddef> // size_t
 #include <cstdint> // uint16_t, uint32_t, uint64_t
 #include <cstring> // memcmp, memcpy
+#include <limits> // numeric_limits
+#include <type_traits> // integral_constant, is_integral, is_same
 
 #include <nlohmann/json.hpp>
 #include <nlohmann/detail/view/document_data.hpp>
@@ -82,8 +84,9 @@ class short_key
     std::uint64_t m_b = 0;
 };
 
-/// the key node of the first member of an object with the given key, or
-/// nullptr; most keys are rejected by their length, from the index alone
+/// the key node of the last member of an object with the given key, or
+/// nullptr (the last one, as materialize() and parse() keep it); most keys are
+/// rejected by their length, from the index alone
 template<bool Editable>
 const node* find_member(const document_data& d, const node* object, const char* key, std::size_t n) noexcept
 {
@@ -94,6 +97,7 @@ const node* find_member(const document_data& d, const node* object, const char*
     }
     const node* const end = nav::end(d, object);
     const auto* const k = reinterpret_cast<const unsigned char*>(key); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+    const node* last = nullptr;
     if (NLOHMANN_VIEW_LIKELY(n <= 16))
     {
         const short_key probe(k, n);
@@ -101,19 +105,37 @@ const node* find_member(const document_data& d, const node* object, const char*
         {
             if (m->len == n && probe.matches(reinterpret_cast<const unsigned char*>(d.str(*m)))) // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
             {
-                return m;
+                last = m;
             }
         }
-        return nullptr;
+        return last;
     }
     for (const node* m = nav::first(d, object); m != end; m = document_data::after(m + 1))
     {
         if (m->len == n && std::memcmp(d.str(*m), key, n) == 0)
         {
-            return m;
+            last = m;
         }
     }
-    return nullptr;
+    return last;
+}
+
+/// whether an integer type is accepted as an array index by the view's
+/// operator[] and at(): every integer type but bool and size_t, which has its
+/// own overload
+template<typename T>
+struct is_index_type : std::integral_constant < bool,
+    std::is_integral<T>::value && !std::is_same<T, bool>::value && !std::is_same<T, std::size_t>::value >
+{};
+
+/// an integer as an index: negative values, and values that do not fit a
+/// size_t, map to the largest size_t (out of range for every array)
+template<typename SizeType, typename IntegerType>
+SizeType to_index(IntegerType idx) noexcept
+{
+    const IntegerType zero = 0;
+    const auto result = static_cast<SizeType>(idx);
+    return (idx < zero || static_cast<IntegerType>(result) != idx) ? (std::numeric_limits<SizeType>::max)() : result;
 }
 
 /// the entry of the element of an array at an index below its size (a link

include/nlohmann/detail/view/node.hpp

@@ -29,6 +29,11 @@ static_assert(static_cast<std::uint8_t>(value_t::null) == 0 && static_cast<std::
               && static_cast<std::uint8_t>(value_t::number_unsigned) == 6 && static_cast<std::uint8_t>(value_t::number_float) == 7,
               "the node format depends on the numbering of value_t");
 
+/// The largest input a document accepts, in bytes. Offsets and node counts are
+/// 32 bits wide; the limit keeps 16 bytes (the width of the scanner's steps)
+/// below 2^32, so that a position one step past the end of the text fits.
+static constexpr std::size_t max_input_size = 0xFFFFFFEFu;
+
 /// node flags
 struct node_flags
 {
@@ -38,6 +43,7 @@ struct node_flags
     static constexpr std::uint8_t is_true = 4; ///< boolean value
     static constexpr std::uint8_t moved = 8;   ///< array/object: the elements live in a separate sequence (editable documents)
     static constexpr std::uint8_t is_new = 16; ///< written by an edit: no source position
+    static constexpr std::uint8_t linked = 32; ///< an entry of a moved sequence links to this value (editable documents): its extent in the parsed layout no longer matters
 };
 
 /// kind of an entry of an edited sequence that stands for a value stored
@@ -52,7 +58,7 @@ struct node
 {
     std::uint8_t kind;   ///< value_t, or kind_link
     std::uint8_t flags;  ///< node_flags
-    std::uint16_t extra; ///< numbers: integer digits (low byte) and fraction digits (high byte), 255 = "many"; objects: number of the hash index; otherwise 0
+    std::uint16_t extra; ///< numbers: integer digits (low byte) and fraction digits (high byte), 255 = "many"; objects: number of the hash index (1-based, 0 = none); otherwise 0
     std::uint32_t off;   ///< source offset (string content, number token, literal, bracket); arena offset if escaped/edited; number of the element sequence if moved
     std::uint32_t len;   ///< string: decoded bytes; float: token bytes; array/object: element count
     std::uint32_t next;  ///< array/object: number of nodes of the subtree (its extent in the enclosing sequence)
@@ -72,24 +78,36 @@ NLOHMANN_VIEW_ALWAYS_INLINE const node* link_target(const node& n) noexcept
     return t;
 }
 
-inline void make_link(node& n, const node* target) noexcept
+/// let the entry n stand for the value at target (and mark the value)
+inline void make_link(node& n, node* target) noexcept
 {
+    target->flags = static_cast<std::uint8_t>(target->flags | node_flags::linked);
     n = node{};
     n.kind = kind_link;
     std::memcpy(reinterpret_cast<unsigned char*>(&n) + 8, static_cast<const void*>(&target), sizeof(const node*)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
 }
 
-/// the converted value of an integer node (stored in len/next)
+/// the converted value of an integer node: len is its low half, next its high
+/// half (on little-endian targets the two words are the value in memory)
 NLOHMANN_VIEW_ALWAYS_INLINE std::uint64_t integer_bits(const node& n) noexcept
 {
+#if NLOHMANN_VIEW_LITTLE_ENDIAN
     std::uint64_t v = 0;
     std::memcpy(&v, reinterpret_cast<const unsigned char*>(&n) + 8, 8); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
     return v;
+#else
+    return static_cast<std::uint64_t>(n.len) | (static_cast<std::uint64_t>(n.next) << 32);
+#endif
 }
 
 NLOHMANN_VIEW_ALWAYS_INLINE void set_integer_bits(node& n, std::uint64_t v) noexcept
 {
+#if NLOHMANN_VIEW_LITTLE_ENDIAN
     std::memcpy(reinterpret_cast<unsigned char*>(&n) + 8, &v, 8); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+#else
+    n.len = static_cast<std::uint32_t>(v);
+    n.next = static_cast<std::uint32_t>(v >> 32);
+#endif
 }
 
 /// token length of a number node

include/nlohmann/detail/view/object_index.hpp

@@ -22,8 +22,14 @@
 // large objects). An object with document_data::index_min_members members or
 // more gets an open-addressing table after parsing; its node stores the
 // number of the table (1-based) in `extra`. A slot holds the offset of a key
-// node from its object node (0: empty). Of duplicate keys, the first is kept,
-// as for the linear search.
+// node from its object node (0: empty). Of duplicate keys, the last is kept,
+// as for the linear search, and as basic_json::parse() does.
+//
+// The hash is not seeded, so keys chosen to collide could make the build
+// quadratic. A key therefore sits at most index_max_displacement slots away
+// from its home slot; if a key would sit further away, the table is dropped
+// and the object is searched linearly (like a small one). For the same
+// reason, a lookup visits at most index_max_displacement + 1 slots.
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
@@ -31,6 +37,11 @@ namespace detail
 namespace view
 {
 
+/// the farthest a key may sit from its home slot (a table with at most half of
+/// its slots in use gives random keys a distance of about 50 for millions of
+/// members; and every member costs at most this many steps while building)
+constexpr std::size_t index_max_displacement = 64;
+
 /// hash of a key: its bytes, eight at a time, in a fixed byte order
 inline std::uint64_t key_hash(const char* s, std::size_t n) noexcept
 {
@@ -68,27 +79,44 @@ inline void build_object_index(document_data& d, node* obj)
     d.index_slots.resize(start + cap, 0);
     std::uint32_t* const slots = d.index_slots.data() + start;
     const std::size_t mask = cap - 1;
+    bool degenerate = false;
     for (const node* k = document_data::first_child(obj), *end = document_data::child_end(obj); k != end; k = document_data::after(k + 1))
     {
         const char* const key = d.str(*k);
         const std::uint64_t hash = key_hash(key, k->len); // (a cast of the call would be useless where std::uint64_t is std::size_t)
         std::size_t i = static_cast<std::size_t>(hash) & mask;
         bool duplicate = false;
+        std::size_t distance = 0;
         while (slots[i] != 0)
         {
             const node* const other = obj + slots[i];
             if (other->len == k->len && (k->len == 0 || std::memcmp(d.str(*other), key, k->len) == 0))
             {
-                duplicate = true; // keep the first
+                duplicate = true; // keep the last: the key's slot now leads to this member
+                slots[i] = static_cast<std::uint32_t>(k - obj);
+                break;
+            }
+            if (++distance > index_max_displacement)
+            {
+                degenerate = true; // too many keys share a home region
                 break;
             }
             i = (i + 1) & mask;
         }
+        if (degenerate)
+        {
+            break;
+        }
         if (!duplicate)
         {
             slots[i] = static_cast<std::uint32_t>(k - obj);
         }
     }
+    if (degenerate)
+    {
+        d.index_slots.resize(start); // no table: the object is searched linearly
+        return;
+    }
     d.indexes.push_back(document_data::object_index{start, static_cast<std::uint32_t>(mask)});
     obj->extra = static_cast<std::uint16_t>(d.indexes.size());
 }
@@ -102,7 +130,7 @@ inline void build_object_indexes(document_data& d)
     }
 }
 
-/// the key node of the first member with this key of an indexed object, or
+/// the key node of the last member with this key of an indexed object, or
 /// nullptr
 inline const node* find_indexed(const document_data& d, const node* obj, const char* key, std::size_t n) noexcept
 {
@@ -110,7 +138,7 @@ inline const node* find_indexed(const document_data& d, const node* obj, const c
     const std::uint32_t* const slots = d.index_slots.data() + ix.start;
     const std::uint64_t hash = key_hash(key, n); // (a cast of the call would be useless where std::uint64_t is std::size_t)
     std::size_t i = static_cast<std::size_t>(hash) & ix.mask;
-    for (;;)
+    for (std::size_t distance = 0; distance <= index_max_displacement; ++distance)
     {
         const std::uint32_t s = slots[i];
         if (s == 0)
@@ -124,6 +152,7 @@ inline const node* find_indexed(const document_data& d, const node* obj, const c
         }
         i = (i + 1) & ix.mask;
     }
+    return nullptr; // (no key sits further from its home slot)
 }
 
 }  // namespace view

include/nlohmann/detail/view/serializer.hpp

@@ -44,7 +44,13 @@ class output_buffer
 
     void finish()
     {
-        m_out.resize(static_cast<std::size_t>(m_pos - m_out.data()));
+        const auto size = static_cast<std::size_t>(m_pos - m_out.data());
+        m_out.resize(size);
+        // do not keep a buffer that was sized for a much larger output
+        if (m_out.capacity() > 1024 && m_out.capacity() / 2 > size)
+        {
+            m_out.shrink_to_fit();
+        }
     }
 
     NLOHMANN_VIEW_ALWAYS_INLINE void reserve(std::size_t n)

include/nlohmann/detail/view/simd.hpp

@@ -35,7 +35,8 @@
 #else
     #define NLOHMANN_VIEW_NEON 0
 #endif
-#if !defined(JSON_VIEW_NO_SIMD) && !NLOHMANN_VIEW_NEON && (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2))
+// (x86 only: other targets can define __SSE2__ as well, e.g., WebAssembly with -msse2, but have no <cpuid.h>)
+#if !defined(JSON_VIEW_NO_SIMD) && !NLOHMANN_VIEW_NEON && (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86)) && (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2))
     #include <emmintrin.h>
     #define NLOHMANN_VIEW_SSE2 1
 #else