From fa8134ae85095f7fa70a0153a3e051c1ced464b9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 21 Oct 2026 22:57:51 +0200 Subject: [PATCH] Compare json_view values without a vector per container and a basic_json per number operator== of views (view/view, view/json, and editable documents) built two vectors of members for every object it entered, sorted them with stable_sort for every object, and converted each number to a basic_json value. It now: - keeps the members of all open objects in one vector (reserved with geometric growth, so that nested objects do not reallocate), and orders them by insertion sort for up to 16 members, or not at all when the keys ascend already; - compares numbers of one type directly, and builds a basic_json only for numbers of different types; - compares short texts (keys, numbers, most strings) with two overlapping loads (bytes_equal(), in node.hpp); - first walks both documents in lockstep, in the order of their text, which finds most pairs of equal values without the work of the comparison. The walk only enters an object if the comparison would visit its members in the same order (the object type keeps the order, the keys ascend, or all values are scalars), so a difference is never found later than before, and it records the containers it found different, so that the comparison does not walk them again; - rejects values of different types and arrays of different sizes before any of this, and, for an object type that keeps the order, objects whose first (up to four) members have different keys, which ends a comparison that differs near the start without preparing the members. Results are unchanged: numbers compare by value across types, duplicate keys resolve as parse() resolves them, and objects compare in document order for ordered_json and by key otherwise. Instructions of one comparison of a document with an equal copy, macOS arm64 (develop -> now), json_document / ordered_json_document / editable: twitter 16.1M -> 7.0M 17.7M -> 1.5M 16.6M -> 7.3M citm 45.0M -> 3.4M 47.0M -> 3.2M 45.7M -> 4.3M canada 111.4M -> 13.7M 111.4M -> 13.7M 113.3M -> 15.0M jeopardy 26.2M -> 3.7M 27.3M -> 3.7M 26.6M -> 12.1M and of a view with a basic_json value: twitter 11.4M -> 5.9M 11.2M -> 7.3M 11.6M -> 5.9M citm 33.9M -> 9.7M 32.6M -> 9.1M 34.6M -> 9.8M canada 92.9M -> 32.1M 92.9M -> 32.1M 93.9M -> 32.8M and Linux instruction counts (GCC 13 / Clang 18) of the same comparisons: document with an equal copy json_document ordered_json_document editable twitter -49 % / -52 % -89 % / -89 % -48 % / -50 % citm -90 % / -89 % -91 % / -90 % -88 % / -87 % jeopardy (4,000 questions) -85 % / -85 % -86 % / -86 % -46 % / -49 % view with a basic_json value twitter -41 % / -45 % -28 % / -35 % -45 % / -44 % citm -63 % / -65 % -63 % / -66 % -65 % / -64 % jeopardy -39 % / -43 % -14 % / -24 % -44 % / -43 % A sweep that places one difference at about 2,100 positions of eight documents (a token at an even spacing, the first and last tokens, the first, middle and last member of the root removed, one appended, the document wrapped) in five kinds of comparison (view/view of read-only and of editable documents, view/json, json_document, editable/view) finds the same results as before and no placement slower on the Mac (Clang); on Linux no placement is slower than 3 % (Clang and GCC). Signed-off-by: Niels Lohmann --- include/nlohmann/detail/view/compare.hpp | 886 ++++++++++-- include/nlohmann/detail/view/node.hpp | 68 + include/nlohmann/json_view.hpp | 4 + single_include/nlohmann/json_view.hpp | 1642 ++++++++++++++++------ tests/src/unit-json_view_dump.cpp | 150 ++ 5 files changed, 2250 insertions(+), 500 deletions(-) diff --git a/include/nlohmann/detail/view/compare.hpp b/include/nlohmann/detail/view/compare.hpp index 8ca6058f0..626f90238 100644 --- a/include/nlohmann/detail/view/compare.hpp +++ b/include/nlohmann/detail/view/compare.hpp @@ -8,14 +8,21 @@ #pragma once -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector #include +#include #include +#include +#include +#include NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -30,17 +37,260 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys; small ones are sorted by insertion, which keeps equal keys in order +/// and needs no memory. +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + std::vector value_at(n, none); + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + value_at[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + i = last + 1; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (value_at[i] != none) + { + order[w++] = value_at[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence + std::vector order(n); + for (std::size_t i = 0; i < n; ++i) + { + order[i] = i; + } + std::stable_sort(order.begin(), order.end(), [first](std::size_t x, std::size_t y) + { + return key_less(first[x].first, first[y].first); + }); + std::vector drop(n, false); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = true; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (!drop[i]) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -56,6 +306,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -80,60 +356,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void open(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -146,6 +409,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -167,40 +441,394 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void open(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (!EA && a->next == 1 + (2 * static_cast(a->len))) + { + return true; // (scalar values only) + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + string_view_t ka[limit]; + string_view_t kb[limit]; + a.first_keys(ka, n); + b.first_keys(kb, n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; @@ -214,22 +842,54 @@ bool equal(const A& a0, const B& b0) using string_view_t = typename A::string_view_t; const bool ordered = is_ordered_map::value; + constexpr std::size_t npos = static_cast(-1); struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = npos; ///< their order in the comparison: from this position in indices (npos: as they are) + std::size_t index_b = npos; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -240,7 +900,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -257,31 +917,87 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ordered && ta == value_t::object && keys_differ_early(a, b)) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.open(f.cursor_a); + b.open(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + if (ta == value_t::number_integer ? a.integer() != b.integer() + : ta == value_t::number_unsigned ? a.unsigned_integer() != b.unsigned_integer() + : !floats_equal(a, b)) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -289,28 +1005,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == npos ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == npos ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } diff --git a/include/nlohmann/detail/view/node.hpp b/include/nlohmann/detail/view/node.hpp index 7a05c8e0a..0d3c5182a 100644 --- a/include/nlohmann/detail/view/node.hpp +++ b/include/nlohmann/detail/view/node.hpp @@ -120,6 +120,74 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t number_length(const node& n) noexcept : (n.extra & 0xFFu) + (n.kind == static_cast(value_t::number_integer) ? 1u : 0u); } +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + std::size_t i = 0; + for (; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + /// estimated number of nodes for an input of `size` bytes (one node per ~12 /// bytes covers typical documents without regrowth) inline std::size_t estimate_nodes(std::size_t size) noexcept diff --git a/include/nlohmann/json_view.hpp b/include/nlohmann/json_view.hpp index f0fc58dc9..9c7026606 100644 --- a/include/nlohmann/json_view.hpp +++ b/include/nlohmann/json_view.hpp @@ -723,8 +723,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/single_include/nlohmann/json_view.hpp b/single_include/nlohmann/json_view.hpp index 6220270c7..c36adf457 100644 --- a/single_include/nlohmann/json_view.hpp +++ b/single_include/nlohmann/json_view.hpp @@ -317,6 +317,74 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t number_length(const node& n) noexcept : (n.extra & 0xFFu) + (n.kind == static_cast(value_t::number_integer) ? 1u : 0u); } +/// whether n bytes are equal; the short texts of JSON (keys, numbers, most +/// strings) are compared by two overlapping loads instead of a library call +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept +{ + const auto load8 = [](const char* p) + { + std::uint64_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + const auto load4 = [](const char* p) + { + std::uint32_t v = 0; + std::memcpy(&v, p, sizeof(v)); + return v; + }; + if (n >= 8) + { + std::size_t i = 0; + for (; i + 8 < n; i += 8) + { + if (load8(x + i) != load8(y + i)) + { + return false; + } + } + return load8(x + n - 8) == load8(y + n - 8); + } + if (n >= 4) + { + return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4); + } + for (std::size_t i = 0; i < n; ++i) + { + if (x[i] != y[i]) + { + return false; + } + } + return true; +} + +/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first); +/// the first difference is found eight bytes at a time +NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept +{ + const std::size_t n = n1 < n2 ? n1 : n2; + std::size_t i = 0; + for (; i + 8 <= n; i += 8) + { + const std::uint64_t u = read_eight_bytes(x + i); + const std::uint64_t v = read_eight_bytes(y + i); + if (u != v) + { + const auto shift = static_cast(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs) + return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu); + } + } + for (; i < n; ++i) + { + if (x[i] != y[i]) + { + return static_cast(x[i]) < static_cast(y[i]); + } + } + return n1 < n2; +} + /// estimated number of nodes for an input of `size` bytes (one node per ~12 /// bytes covers typical documents without regrowth) inline std::size_t estimate_nodes(std::size_t size) noexcept @@ -2283,15 +2351,366 @@ NLOHMANN_JSON_NAMESPACE_END -#include // sort, stable_sort +#include // is_sorted, sort, stable_sort +#include // array #include // size_t +#include // int64_t, uint32_t, uint64_t +#include // equal_to #include // string -#include // move, pair +#include // move, pair, swap #include // vector // #include +// #include + // #include +// #include + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // size_t +#include // int64_t, uint64_t +#include // numeric_limits +#include // string +#include // integral_constant + +// #include +// #include + +// #include + +// #include + +// #include + + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +/*! +@brief locate the decimal point and the end of the mantissa of a float token + +Also checks that the token is a JSON number. Tokens of the parser and of edits +always are; an image loaded with image_check::bounds can hold any bytes, which +must not reach the conversion (it expects a well-formed token). +*/ +inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept +{ + const auto digit = [last](const char* q) + { + return q != last && is_digit(static_cast(*q)); + }; + const char* p = first; + p += (p != last && *p == '-') ? 1 : 0; + if (!digit(p) || (*p == '0' && digit(p + 1))) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + dot = std::string::npos; + if (p != last && *p == '.') + { + dot = static_cast(p - first); + if (!digit(++p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + mantissa_end = static_cast(p - first); + if (p != last && (*p == 'e' || *p == 'E')) + { + ++p; + p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; + if (!digit(p)) + { + return false; + } + do + { + ++p; + } + while (digit(p)); + } + return p == last; +} + +/*! +@brief the value of the float token of a node, as parse() converts it + +Uses the lexer's conversion (detail::convert_float), so that the values are +bit-identical to parse(): float and double are converted without allocation +and independent of the locale. A token that is not a JSON number (only in a +damaged image loaded with image_check::bounds) yields 0. +*/ +template +NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) +{ + const char* const last = first + n.len; + std::size_t dot = 0; + std::size_t mantissa_end = 0; + if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) + { + return FloatType{}; + } + return convert_float(first, last, dot, mantissa_end); +} + +/*! +@brief the digits of a float token with at most 19 digits, from its layout + +The digit layout recorded while parsing says where the integer digits, the +fraction digits, and the exponent are, so the digits are read eight at a +time without scanning. + +@param[in] p first character of the token +@param[in] e end of the token +@param[in] limit end of the readable memory (the source text) +*/ +NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + const bool negative = *p == '-'; + p += negative ? 1 : 0; + std::uint64_t w = parse_upto19(p, int_digits, limit); + p += int_digits; + std::int64_t q = 0; + if (frac_digits != 0) + { + w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); + p += 1 + frac_digits; + q = -static_cast(frac_digits); + } + if (p != e) + { + // [eE][+-]digits; huge exponents saturate (the parser rejected + // overflow). The token is not read beyond e, and the digits are taken + // as unsigned, so that a token that is not well-formed (a damaged + // image loaded with image_check::bounds) yields a wrong value, but no + // overflow. + ++p; + const bool exp_negative = p != e && *p == '-'; + p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; + std::int64_t exp_value = 0; + for (; p != e; ++p) + { + if (exp_value < 0x10000000) + { + exp_value = (exp_value * 10) + static_cast(*p - '0'); + } + } + q += exp_negative ? -exp_value : exp_value; + } + + float_significand d; + d.w = w; + d.exponent = q; + d.negative = negative; + return d; +} + +/*! +@brief the value of a float token with at most 19 digits, from its layout + +The result is correctly rounded by the lexer's conversion +(detail::decimal_to_float(): Clinger's fast path where both operands are +exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 +digits), so it is the value parse() produces. +*/ +template +NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept +{ + return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); +} + +/// the value of a float set by an edit: its token (the shortest round-trip +/// text, or "nan", "inf", "-inf") in the edit arena +template +NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) +{ + if (token[0] == 'n') + { + return std::numeric_limits::quiet_NaN(); + } + if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) + { + return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); + } + return float_value(token, n); +} + +/// the value of the float token of a node, as parse() converts it; floats and +/// doubles with at most 19 digits are converted from the digit layout +template +FloatType float_value(const document_data& d, const node& n) +{ + if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) + { + return edited_float(d.str(n), n); + } + return float_value(d, n, std::integral_constant::value> {}); +} + +template +FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) +{ + const unsigned int_digits = n.extra & 0xFFu; + const unsigned frac_digits = n.extra >> 8u; + if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") + { + // (a float token not written by an edit is in the text) + const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) + } + return float_value(d.str(n), n); +} + +template +FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) +{ + return float_value(d.str(n), n); +} + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + + + +#include // min +#include // size_t +#include // memcmp, strlen +#include // basic_string + +// #include +// #include + + +#if NLOHMANN_VIEW_HAS_CPP_17 + #include // string_view +#endif +#ifndef JSON_NO_IO + #include // ostream +#endif + +NLOHMANN_JSON_NAMESPACE_BEGIN +namespace detail +{ +namespace view +{ + +#if NLOHMANN_VIEW_HAS_CPP_17 +using string_ref = std::string_view; +#else +/// minimal C++11 stand-in for std::string_view +class string_ref +{ + public: + using size_type = std::size_t; + using const_iterator = const char*; + + string_ref() noexcept = default; + // s must be null-terminated, as for std::string_view(const char*) + // flawfinder: ignore + string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} + template + string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) + + const char* data() const noexcept + { + return m_data; + } + std::size_t size() const noexcept + { + return m_size; + } + std::size_t length() const noexcept + { + return m_size; + } + bool empty() const noexcept + { + return m_size == 0; + } + const char* begin() const noexcept + { + return m_data; + } + const char* end() const noexcept + { + return m_data + m_size; + } + char operator[](std::size_t i) const noexcept + { + return m_data[i]; + } + + template + explicit operator std::basic_string() const + { + return std::basic_string(m_data, m_size); + } + + friend bool operator==(string_ref a, string_ref b) noexcept + { + return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); + } + friend bool operator!=(string_ref a, string_ref b) noexcept + { + return !(a == b); + } + friend bool operator<(string_ref a, string_ref b) noexcept + { + const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); + return c != 0 ? c < 0 : a.m_size < b.m_size; + } +#ifndef JSON_NO_IO + friend std::ostream& operator<<(std::ostream& o, string_ref s) + { + return o.write(s.m_data, static_cast(s.m_size)); + } +#endif + + private: + const char* m_data = ""; + std::size_t m_size = 0; +}; +#endif + +} // namespace view +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END + NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail @@ -2306,17 +2725,260 @@ namespace view // value, at the position of the first occurrence), and in document order if // the object type keeps an order (ordered_json), by key otherwise. +/// The members of an object as parse() keeps them: one per key, the last +/// value at the position of the first occurrence, in the order of the +/// comparison (sorted by key, or, if the object type keeps the order, the order +/// of first occurrence). The members m[0..n) are in document order; this +/// appends to out the indices of the member of each key that has its value +/// (its last one), in the order of the comparison. Most objects have unique +/// keys; small ones are sorted by insertion, which keeps equal keys in order +/// and needs no memory. +/// whether the key a is before the key b +template +NLOHMANN_VIEW_ALWAYS_INLINE bool key_less(const Key& a, const Key& b) noexcept +{ + return bytes_less(a.data(), a.size(), b.data(), b.size()); +} + +template +void resolve_indices(const Entry* m, std::size_t n, bool ordered, std::vector& out) +{ + const std::size_t start = out.size(); + out.resize(start + n); + std::uint32_t* const order = out.data() + start; + for (std::size_t i = 0; i < n; ++i) + { + order[i] = static_cast(i); + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + const std::uint32_t x = order[i]; + std::size_t j = i; + for (; j > 0 && key_less(m[x].first, m[order[j - 1]].first); --j) + { + order[j] = order[j - 1]; + } + order[j] = x; + } + } + else + { + std::stable_sort(order, order + n, [m](std::uint32_t x, std::uint32_t y) + { + return key_less(m[x].first, m[y].first); + }); + } + if (!ordered) + { + // the last of each run of equal keys + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + order[w++] = order[last]; + i = last + 1; + } + out.resize(start + w); + return; + } + // (document order) the value of the last occurrence at the first occurrence of each key + constexpr std::uint32_t none = 0xFFFFFFFFu; + std::vector value_at(n, none); + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && m[order[last + 1]].first == m[order[i]].first) + { + ++last; + } + value_at[order[i]] = order[last]; // (stable: order[i] is the first occurrence) + i = last + 1; + } + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (value_at[i] != none) + { + order[w++] = value_at[i]; + } + } + out.resize(start + w); +} + +/// whether the keys of n members are the same, in the same order +template +bool same_keys(const EntryA* a, const EntryB* b, std::size_t n) +{ + for (std::size_t i = 0; i < n; ++i) + { + if (!(a[i].first == b[i].first)) + { + return false; + } + } + return true; +} + +/// The members of an object as parse() keeps them (one per key: the last +/// value at the position of the first occurrence), sorted by key or, if the +/// object type keeps the order, in the order of first occurrence. The +/// members are given in document order, at the end of m (from the position +/// base on); most objects have unique keys, in which case they are only sorted +/// (for small objects with an insertion sort, which keeps equal keys in order +/// and needs no memory). +template +void resolve_members(std::vector>& m, std::size_t base, bool ordered) +{ + using entry = std::pair; + const std::size_t n = m.size() - base; + entry* const first = m.data() + base; + if (ordered) + { + // (document order) later values of a key go to its first occurrence + std::vector order(n); + for (std::size_t i = 0; i < n; ++i) + { + order[i] = i; + } + std::stable_sort(order.begin(), order.end(), [first](std::size_t x, std::size_t y) + { + return key_less(first[x].first, first[y].first); + }); + std::vector drop(n, false); + bool any = false; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[order[last + 1]].first == first[order[i]].first) + { + ++last; + } + if (last != i) + { + first[order[i]].second = first[order[last]].second; + for (std::size_t k = i + 1; k <= last; ++k) + { + drop[order[k]] = true; + } + any = true; + } + i = last + 1; + } + if (any) + { + std::size_t w = 0; + for (std::size_t i = 0; i < n; ++i) + { + if (!drop[i]) + { + if (w != i) + { + first[w] = std::move(first[i]); + } + ++w; + } + } + m.erase(m.begin() + static_cast(base + w), m.end()); + } + return; + } + // sorted by key; equal keys stay in document order + if (n <= 16) + { + for (std::size_t i = 1; i < n; ++i) + { + entry x = std::move(first[i]); + std::size_t j = i; + for (; j > 0 && key_less(x.first, first[j - 1].first); --j) + { + first[j] = std::move(first[j - 1]); + } + first[j] = std::move(x); + } + } + else + { + std::stable_sort(first, first + n, [](const entry & x, const entry & y) + { + return key_less(x.first, y.first); + }); + } + // the first key of a run with the last value + std::size_t w = 0; + for (std::size_t i = 0; i < n;) + { + std::size_t last = i; + while (last + 1 < n && first[last + 1].first == first[i].first) + { + ++last; + } + if (w != i || last != i) + { + first[w] = entry(first[i].first, std::move(first[last].second)); + } + ++w; + i = last + 1; + } + m.erase(m.begin() + static_cast(base + w), m.end()); +} + +/// room for k more entries, growing geometrically (reserve(size() + k) alone would +/// reallocate for each object nested in another) +template +void reserve_more(std::vector& v, std::size_t k) +{ + const std::size_t needed = v.size() + k; + if (needed > v.capacity()) + { + v.reserve((std::max)(needed, 2 * v.capacity())); + } +} + /// one side of a comparison: a view template class view_side { public: using string_view_t = typename View::string_view_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using nav = navigation; + + /// the position in the elements of an array + struct cursor + { + const node* pos = nullptr; + }; + + view_side() noexcept + : m_view() + {} explicit view_side(const View& v) noexcept : m_view(v) {} + /// the document and the node of the view (nullptr: a discarded view) + const document_data& document() const noexcept + { + return *m_view.m_doc; + } + + const node* value_node() const noexcept + { + return m_view.m_node; + } + + static constexpr bool is_editable = View::is_editable; + value_t type() const noexcept { return m_view.type(); @@ -2332,6 +2994,32 @@ class view_side return m_view.get_string(); } + bool boolean() const noexcept + { + return (m_view.m_node->flags & node_flags::is_true) != 0; + } + + number_integer_t integer() const noexcept + { + return static_cast(static_cast(integer_bits(*m_view.m_node))); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return static_cast(integer_bits(*m_view.m_node)); + } + + number_float_t floating() const + { + return float_value(*m_view.m_doc, *m_view.m_node); + } + + /// the token of a float + string_view_t float_token() const noexcept + { + return string_view_t(m_view.m_doc->str(*m_view.m_node), m_view.m_node->len); + } + /// a number, boolean, or null as a basic_json value (no allocation) BasicJsonType scalar() const { @@ -2356,60 +3044,47 @@ class view_side } } - void elements(std::vector& out) const + /// start at the first element of this array + void open(cursor& c) const noexcept { - out.reserve(m_view.size()); - for (const View e : m_view) + c.pos = nav::first(*m_view.m_doc, m_view.m_node); + } + + /// the element at c (and advance) + view_side take(cursor& c) const noexcept + { + const node* const v = nav::value(c.pos); + c.pos = document_data::after(c.pos); + return view_side(View(m_view.m_doc, v)); + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = 0; i < n; ++i) { - out.emplace_back(e); + out[i] = string_view_t(m_view.m_doc->str(*pos), pos->len); + pos = document_data::after(pos + 1); } } - /// the members as parse() keeps them: one per key, the last value at the - /// position of the first occurrence; in that order, or sorted by key - void members(std::vector>& out, bool ordered) const + /// append the members to out, in document order; whether their keys ascend + /// (then they are unique, and in the order of the comparison of a std::map) + bool members(std::vector>& out, bool /*ordered*/) const { - struct member + const std::size_t base = out.size(); + reserve_more(out, m_view.m_node->len); + bool sorted = true; + const node* pos = nav::first(*m_view.m_doc, m_view.m_node); + for (std::size_t i = m_view.m_node->len; i != 0; --i) { - string_view_t key; - View value; - std::size_t position; - }; - std::vector all; - all.reserve(m_view.size()); - std::size_t position = 0; - for (auto it = m_view.begin(); it != m_view.end(); ++it) - { - all.push_back(member{it.key(), it.value(), position++}); - } - std::stable_sort(all.begin(), all.end(), [](const member & a, const member & b) - { - return a.key < b.key; - }); - std::vector unique; - unique.reserve(all.size()); - for (std::size_t i = 0; i < all.size();) - { - std::size_t last = i; - while (last + 1 < all.size() && all[last + 1].key == all[i].key) - { - ++last; - } - unique.push_back(member{all[i].key, all[last].value, all[i].position}); - i = last + 1; - } - if (ordered) - { - std::sort(unique.begin(), unique.end(), [](const member & a, const member & b) - { - return a.position < b.position; - }); - } - out.reserve(unique.size()); - for (const member& m : unique) - { - out.emplace_back(m.key, view_side(m.value)); + const string_view_t key(m_view.m_doc->str(*pos), pos->len); + sorted = sorted && (out.size() == base || key_less(out.back().first, key)); + out.emplace_back(key, view_side(View(m_view.m_doc, nav::value(pos + 1)))); + pos = document_data::after(pos + 1); } + return sorted; } private: @@ -2422,6 +3097,17 @@ class json_side { public: using string_view_t = StringView; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + + /// the position in the elements of an array + struct cursor + { + typename BasicJsonType::const_iterator it{}; + }; + + json_side() noexcept = default; explicit json_side(const BasicJsonType& j) noexcept : m_json(&j) @@ -2443,40 +3129,394 @@ class json_side return string_view_t(s.data(), s.size()); } + bool boolean() const noexcept + { + return *m_json->template get_ptr(); + } + + number_integer_t integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_unsigned_t unsigned_integer() const noexcept + { + return *m_json->template get_ptr(); + } + + number_float_t floating() const noexcept + { + return *m_json->template get_ptr(); + } + BasicJsonType scalar() const { return *m_json; } - void elements(std::vector& out) const + void open(cursor& c) const { - out.reserve(m_json->size()); - for (const auto& e : *m_json) + c.it = m_json->cbegin(); + } + + json_side take(cursor& c) const + { + const json_side r(*c.it); + ++c.it; + return r; + } + + /// the keys of the first n members of this object (n <= its size), in document order + void first_keys(string_view_t* out, std::size_t n) const noexcept + { + auto it = m_json->cbegin(); + for (std::size_t i = 0; i < n; ++i, ++it) { - out.emplace_back(e); + out[i] = string_view_t(it.key().data(), it.key().size()); } } - void members(std::vector>& out, bool ordered) const + /// append the members to out, in the order of the comparison (the keys of a + /// basic_json are unique); true + bool members(std::vector>& out, bool ordered) const { - out.reserve(m_json->size()); + const std::size_t base = out.size(); + reserve_more(out, m_json->size()); for (auto it = m_json->cbegin(); it != m_json->cend(); ++it) { out.emplace_back(string_view_t(it.key().data(), it.key().size()), json_side(it.value())); } if (!ordered) { - std::sort(out.begin(), out.end(), [](const std::pair& a, const std::pair& b) + const auto less = [](const std::pair& a, const std::pair& b) { - return a.first < b.first; - }); + return key_less(a.first, b.first); + }; + const auto first = out.begin() + static_cast(base); + if (!std::is_sorted(first, out.end(), less)) + { + std::sort(first, out.end(), less); + } + } + return true; + } + + private: + const BasicJsonType* m_json = nullptr; +}; + +/*! +@brief the values at two nodes, found equal by walking both documents in the +same order + +Two values are equal if they have the same type, the same number token or +string, the same elements, and the same members with the same keys in the same +order: a value that matches this way is equal under the rules above whatever +the object type and duplicate keys (the last value of a key then wins on +both sides, and their order of first occurrence is the same). The walk does +not find every equal pair (numbers spelled differently, members in another +order, duplicate keys with other values), only the ones that are the same +text; it stops at the first difference and then says nothing. Containers +nested deeper than the budget are not walked. + +The walk goes through the document in its order, the comparison that follows +goes through the members of an object by key. So that the walk never reaches +a difference later than the comparison would, and never costs more than it, +it enters an object only if the two orders agree (the keys ascend, or the +object type keeps the order) or if the walk is cheaper than preparing the +comparison of the object (all values are scalars). +*/ +/// the containers that lockstep found different, from the outermost to the +/// innermost: the comparison that goes on from there does not walk them again +struct lockstep_path +{ + /// the nesting depth up to which lockstep walks (the recursion is bounded) + static constexpr unsigned budget() noexcept + { + return 48; + } + + /// a pair of containers that differ; for arrays, the elements where the walk stopped + /// (the ones before are equal) + struct entry + { + const node* a; + const node* b; + const node* element_a; + const node* element_b; + std::uint32_t index; + }; + + /// the number of entries that a path can hold (a container deeper than the + /// budget of the walk is not entered) + static constexpr std::size_t levels() noexcept + { + return 50; + } + + explicit lockstep_path(entry* storage) noexcept + : nodes(storage) + {} + + entry* nodes; ///< levels() entries, written before they are read + unsigned length = 0; ///< pairs recorded by the last walk + unsigned next = 0; ///< the pair that the comparison enters next + const entry* resume = nullptr; ///< the entry of the pair that was just looked at, if it is in the path +}; + +template +class lockstep +{ + public: + lockstep(const document_data& a, const document_data& b, lockstep_path& path, bool ordered) noexcept + : m_a(a), m_b(b), m_path(path), m_ordered(ordered) + {} + + /// whether the values at a and b are the same (the nodes of the values, + /// not entries of a sequence) + bool same(const node* a, const node* b, unsigned budget) noexcept + { + if (a->kind != b->kind) + { + return false; + } + switch (static_cast(a->kind)) + { + case value_t::null: + return true; + case value_t::boolean: + return ((a->flags ^ b->flags) & node_flags::is_true) == 0; + case value_t::string: + return a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_float: // (the token, and the digit layout that the value is read with: they only differ in a damaged image) + return a->extra == b->extra && a->len == b->len && bytes_equal(m_a.str(*a), m_b.str(*b), a->len); + case value_t::number_integer: + case value_t::number_unsigned: + return integer_bits(*a) == integer_bits(*b); + case value_t::array: + if (budget == 0 || a->len != b->len) + { + return false; + } + { + lockstep_path::entry stop{a, b, nullptr, nullptr, 0}; + if (elements(a, b, budget - 1, stop)) + { + return true; + } + record(stop, budget); + return false; + } + case value_t::object: + if (budget == 0 || a->len != b->len) + { + return false; + } + if (!visits_in_order(a) || !members(a, b, budget - 1)) + { + record(lockstep_path::entry{a, b, nullptr, nullptr, 0}, budget); + return false; + } + return true; + case value_t::binary: // LCOV_EXCL_LINE (not in a document) + case value_t::discarded: // LCOV_EXCL_LINE + default: // LCOV_EXCL_LINE + return false; // LCOV_EXCL_LINE } } private: - const BasicJsonType* m_json; + using nav_a = navigation; + using nav_b = navigation; + + /// note the containers a and b (at the nesting level that budget stands + /// for) as different + NLOHMANN_VIEW_NOINLINE void record(const lockstep_path::entry& stop, unsigned budget) const noexcept + { + const unsigned level = lockstep_path::budget() - budget; + m_path.nodes[level] = stop; + if (m_path.length == 0) + { + m_path.length = level + 1; // (the innermost container is the first to be left) + } + } + + /// Whether the comparison that follows visits the members of the object a in the + /// order of the document: if the object type keeps the order, if the keys + /// ascend (it sorts by key), or if all values are scalars (then the walk costs + /// less than collecting and sorting the members, whichever member differs). + /// Otherwise a difference could be found by the walk later than by the + /// comparison, which is not worth the walk. + bool visits_in_order(const node* a) const noexcept + { + if (m_ordered || a->len < 2) + { + return true; + } + if (!EA && a->next == 1 + (2 * static_cast(a->len))) + { + return true; // (scalar values only) + } + const node* k = nav_a::first(m_a, a); + string_ref previous(m_a.str(*k), k->len); + for (std::uint32_t i = a->len - 1; i != 0; --i) + { + k = document_data::after(k + 1); + const string_ref key(m_a.str(*k), k->len); + if (!key_less(previous, key)) + { + return false; + } + previous = key; + } + return true; + } + + bool elements(const node* a, const node* b, unsigned budget, lockstep_path::entry& stop) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (!same(nav_a::value(pa), nav_b::value(pb), budget)) + { + stop.element_a = pa; + stop.element_b = pb; + stop.index = a->len - i; + return false; + } + pa = document_data::after(pa); + pb = document_data::after(pb); + } + return true; + } + + bool members(const node* a, const node* b, unsigned budget) noexcept + { + const node* pa = nav_a::first(m_a, a); + const node* pb = nav_b::first(m_b, b); + for (std::uint32_t i = a->len; i != 0; --i) + { + if (pa->len != pb->len || !bytes_equal(m_a.str(*pa), m_b.str(*pb), pa->len)) + { + return false; + } + if (!same(nav_a::value(pa + 1), nav_b::value(pb + 1), budget)) + { + return false; + } + pa = document_data::after(pa + 1); + pb = document_data::after(pb + 1); + } + return true; + } + + const document_data& m_a; + const document_data& m_b; + lockstep_path& m_path; + bool m_ordered; ///< the object type keeps the order of the members }; +/// whether two views certainly have equal values (see lockstep); if not, the +/// path says where they differ +template +bool certainly_equal(const view_side& a, const view_side& b, lockstep_path& path, bool ordered) noexcept +{ + const node* const na = a.value_node(); + const node* const nb = b.value_node(); + if (na == nullptr || nb == nullptr) + { + return false; + } + if (na == nb) + { + return true; // (the same node of the same document: documents have distinct nodes) + } + path.resume = nullptr; + if (path.next < path.length && path.nodes[path.next].a == na && path.nodes[path.next].b == nb) + { + path.resume = &path.nodes[path.next]; + ++path.next; // (the walk found these different already) + return false; + } + path.length = 0; + const bool equal = lockstep::is_editable, view_side::is_editable>(a.document(), b.document(), path, ordered).same(na, nb, lockstep_path::budget()); + path.next = path.length != 0 ? 1 : 0; // (this pair is the first in the path) + path.resume = path.length != 0 ? &path.nodes[0] : nullptr; + return equal; +} + +/// (a view and a basic_json value: no walk) +template +bool certainly_equal(const A& /*a*/, const B& /*b*/, lockstep_path& /*path*/, bool /*ordered*/) noexcept +{ + return false; +} + +/// (arrays of two views) the elements where the walk stopped are the next pair: the ones before are equal +template +void resume_array(const lockstep_path& path, const view_side& a, const view_side& b, + typename view_side::cursor& cursor_a, typename view_side::cursor& cursor_b, std::size_t& next) noexcept +{ + if (path.resume != nullptr && path.resume->element_a != nullptr && path.resume->a == a.value_node() && path.resume->b == b.value_node()) + { + cursor_a.pos = path.resume->element_a; + cursor_b.pos = path.resume->element_b; + next = path.resume->index; + } +} + +template +void resume_array(const lockstep_path& /*path*/, const A& /*a*/, const B& /*b*/, CursorA& /*cursor_a*/, CursorB& /*cursor_b*/, std::size_t& /*next*/) noexcept +{} + +/// whether two floats are equal (the tokens of two views: equal text and digit layout is an equal value; they +/// only differ in a damaged image) +template +bool floats_equal(const view_side& a, const view_side& b) +{ + return (a.value_node()->extra == b.value_node()->extra && a.float_token() == b.float_token()) || std::equal_to()(a.floating(), b.floating()); +} + +template +bool floats_equal(const A& a, const B& b) +{ + return std::equal_to()(a.floating(), b.floating()); +} + +/// Whether two objects of a type that keeps the order are different because of the keys of +/// their first members, found without the preparation of the comparison: the comparison goes +/// through the members in the order of the first occurrence of their keys, so if the first +/// members have the same distinct keys and then one pair of keys differs, where neither is +/// one of the keys before it, the objects are different. (The keys that are the same as an +/// earlier key are the duplicates that the comparison drops, so those are not decided here.) +template +bool keys_differ_early(const SideA& a, const SideB& b) +{ + using string_view_t = typename SideA::string_view_t; + constexpr std::size_t limit = 4; + const std::size_t n = (std::min)((std::min)(a.size(), b.size()), limit); + string_view_t ka[limit]; + string_view_t kb[limit]; + a.first_keys(ka, n); + b.first_keys(kb, n); + for (std::size_t i = 0; i < n; ++i) + { + if (ka[i] != kb[i]) + { + for (std::size_t k = 0; k < i; ++k) + { + if (ka[i] == ka[k] || kb[i] == ka[k]) + { + return false; // (a duplicate: not decided) + } + } + return true; + } + } + return false; +} + inline bool is_number(value_t t) noexcept { return t == value_t::number_integer || t == value_t::number_unsigned || t == value_t::number_float; @@ -2490,22 +3530,54 @@ bool equal(const A& a0, const B& b0) using string_view_t = typename A::string_view_t; const bool ordered = is_ordered_map::value; + constexpr std::size_t npos = static_cast(-1); struct frame { - std::vector elements_a{}; - std::vector elements_b{}; - std::vector> members_a{}; - std::vector> members_b{}; + A array_a{}; + B array_b{}; + typename A::cursor cursor_a{}; + typename B::cursor cursor_b{}; + std::size_t base_a = 0; ///< the members of an object: from this position in members_a + std::size_t base_b = 0; + std::size_t index_a = npos; ///< their order in the comparison: from this position in indices (npos: as they are) + std::size_t index_b = npos; + std::size_t index_mark = 0; ///< the size of indices before this frame bool object = false; std::size_t next = 0; + std::size_t count = 0; }; std::vector stack; + std::vector indices; // the order of the members of the open objects, where it is not their order in the document + // the members of the open objects (the ones of an object follow those of the objects around it) + std::vector> members_a; + std::vector> members_b; + std::array path_storage; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init): written before read + lockstep_path path(path_storage.data()); + { + // (values that differ in type or in the size of an array need no walk) + const value_t first = a0.type(); + const value_t second = b0.type(); + if (first != second && !(is_number(first) && is_number(second)) && first != value_t::discarded && second != value_t::discarded) + { + return false; + } + if (first == value_t::array && second == value_t::array && a0.size() != b0.size()) + { + return false; + } + } A a = a0; B b = b0; + // (pairs of views that are certainly equal need no further look) + if (certainly_equal(a, b, path, ordered)) + { + return true; + } for (;;) { const value_t ta = a.type(); const value_t tb = b.type(); + const bool numbers = is_number(ta) && is_number(tb); if (ta == value_t::discarded || tb == value_t::discarded) { // basic_json decides (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON) @@ -2516,7 +3588,7 @@ bool equal(const A& a0, const B& b0) } else { - if (ta != tb && !(is_number(ta) && is_number(tb))) // numbers of different types can be equal + if (ta != tb && !numbers) // numbers of different types can be equal { return false; } @@ -2533,31 +3605,87 @@ bool equal(const A& a0, const B& b0) { return false; } - frame f; + if (ordered && ta == value_t::object && keys_differ_early(a, b)) + { + return false; + } + stack.emplace_back(); + frame& f = stack.back(); f.object = ta == value_t::object; if (f.object) { - a.members(f.members_a, ordered); - b.members(f.members_b, ordered); - if (f.members_a.size() != f.members_b.size()) + f.base_a = members_a.size(); + f.base_b = members_b.size(); + f.index_mark = indices.size(); + const bool sorted_a = a.members(members_a, ordered); + const bool sorted_b = b.members(members_b, ordered); + const std::size_t size_a = members_a.size() - f.base_a; + const std::size_t size_b = members_b.size() - f.base_b; + std::size_t count_a = size_a; + std::size_t count_b = size_b; + if (!sorted_a && !sorted_b && size_a == size_b && same_keys(members_a.data() + f.base_a, members_b.data() + f.base_b, size_a)) + { + // the same keys in the same order: one order of the comparison for both + f.index_a = indices.size(); + f.index_b = f.index_a; + resolve_indices(members_a.data() + f.base_a, size_a, ordered, indices); + count_a = indices.size() - f.index_a; + count_b = count_a; + } + else + { + if (!sorted_a) + { + resolve_members(members_a, f.base_a, ordered); + count_a = members_a.size() - f.base_a; + } + if (!sorted_b) + { + resolve_members(members_b, f.base_b, ordered); + count_b = members_b.size() - f.base_b; + } + } + f.count = count_a; + if (count_a != count_b) { return false; } } else { - a.elements(f.elements_a); - b.elements(f.elements_b); + f.array_a = a; + f.array_b = b; + a.open(f.cursor_a); + b.open(f.cursor_b); + f.count = a.size(); + resume_array(path, a, b, f.cursor_a, f.cursor_b, f.next); } - stack.push_back(std::move(f)); } - else if (a.scalar() != b.scalar()) // numbers (also of different types), null, boolean + else if (ta == value_t::boolean) + { + if (a.boolean() != b.boolean()) + { + return false; + } + } + else if (numbers && ta == tb) + { + // numbers of one type: as basic_json compares them + if (ta == value_t::number_integer ? a.integer() != b.integer() + : ta == value_t::number_unsigned ? a.unsigned_integer() != b.unsigned_integer() + : !floats_equal(a, b)) + { + return false; + } + } + else if (numbers && !(a.scalar() == b.scalar())) // numbers of different types { return false; } + // (null equals null) } - // the next pair of values + // the next pair of values that is not certainly equal for (;;) { if (stack.empty()) @@ -2565,28 +3693,38 @@ bool equal(const A& a0, const B& b0) return true; } frame& f = stack.back(); - const std::size_t count = f.object ? f.members_a.size() : f.elements_a.size(); - if (f.next == count) + if (f.next == f.count) { + if (f.object) + { + members_a.erase(members_a.begin() + static_cast(f.base_a), members_a.end()); + members_b.erase(members_b.begin() + static_cast(f.base_b), members_b.end()); + indices.resize(f.index_mark); + } stack.pop_back(); // (f dangles now, and is bound again after continue) continue; } if (f.object) { - if (!(f.members_a[f.next].first == f.members_b[f.next].first)) + const auto& member_a = members_a[f.base_a + (f.index_a == npos ? f.next : indices[f.index_a + f.next])]; + const auto& member_b = members_b[f.base_b + (f.index_b == npos ? f.next : indices[f.index_b + f.next])]; + if (!(member_a.first == member_b.first)) { return false; } - a = f.members_a[f.next].second; - b = f.members_b[f.next].second; + a = member_a.second; + b = member_b.second; } else { - a = f.elements_a[f.next]; - b = f.elements_b[f.next]; + a = f.array_a.take(f.cursor_a); + b = f.array_b.take(f.cursor_b); } ++f.next; - break; + if (!certainly_equal(a, b, path, ordered)) + { + break; + } } } } @@ -4339,232 +5477,6 @@ NLOHMANN_JSON_NAMESPACE_END // #include // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // size_t -#include // int64_t, uint64_t -#include // numeric_limits -#include // string -#include // integral_constant - -// #include -// #include - -// #include - -// #include - -// #include - - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -/*! -@brief locate the decimal point and the end of the mantissa of a float token - -Also checks that the token is a JSON number. Tokens of the parser and of edits -always are; an image loaded with image_check::bounds can hold any bytes, which -must not reach the conversion (it expects a well-formed token). -*/ -inline bool float_token_layout(const char* first, const char* last, std::size_t& dot, std::size_t& mantissa_end) noexcept -{ - const auto digit = [last](const char* q) - { - return q != last && is_digit(static_cast(*q)); - }; - const char* p = first; - p += (p != last && *p == '-') ? 1 : 0; - if (!digit(p) || (*p == '0' && digit(p + 1))) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - dot = std::string::npos; - if (p != last && *p == '.') - { - dot = static_cast(p - first); - if (!digit(++p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - mantissa_end = static_cast(p - first); - if (p != last && (*p == 'e' || *p == 'E')) - { - ++p; - p += (p != last && (*p == '+' || *p == '-')) ? 1 : 0; - if (!digit(p)) - { - return false; - } - do - { - ++p; - } - while (digit(p)); - } - return p == last; -} - -/*! -@brief the value of the float token of a node, as parse() converts it - -Uses the lexer's conversion (detail::convert_float), so that the values are -bit-identical to parse(): float and double are converted without allocation -and independent of the locale. A token that is not a JSON number (only in a -damaged image loaded with image_check::bounds) yields 0. -*/ -template -NLOHMANN_VIEW_NOINLINE FloatType float_value(const char* first, const node& n) -{ - const char* const last = first + n.len; - std::size_t dot = 0; - std::size_t mantissa_end = 0; - if (NLOHMANN_VIEW_UNLIKELY(!float_token_layout(first, last, dot, mantissa_end))) - { - return FloatType{}; - } - return convert_float(first, last, dot, mantissa_end); -} - -/*! -@brief the digits of a float token with at most 19 digits, from its layout - -The digit layout recorded while parsing says where the integer digits, the -fraction digits, and the exponent are, so the digits are read eight at a -time without scanning. - -@param[in] p first character of the token -@param[in] e end of the token -@param[in] limit end of the readable memory (the source text) -*/ -NLOHMANN_VIEW_ALWAYS_INLINE float_significand layout_decimal(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - const bool negative = *p == '-'; - p += negative ? 1 : 0; - std::uint64_t w = parse_upto19(p, int_digits, limit); - p += int_digits; - std::int64_t q = 0; - if (frac_digits != 0) - { - w = (w * int_pow10(frac_digits)) + parse_upto19(p + 1, frac_digits, limit); - p += 1 + frac_digits; - q = -static_cast(frac_digits); - } - if (p != e) - { - // [eE][+-]digits; huge exponents saturate (the parser rejected - // overflow). The token is not read beyond e, and the digits are taken - // as unsigned, so that a token that is not well-formed (a damaged - // image loaded with image_check::bounds) yields a wrong value, but no - // overflow. - ++p; - const bool exp_negative = p != e && *p == '-'; - p += (p != e && (*p == '-' || *p == '+')) ? 1 : 0; - std::int64_t exp_value = 0; - for (; p != e; ++p) - { - if (exp_value < 0x10000000) - { - exp_value = (exp_value * 10) + static_cast(*p - '0'); - } - } - q += exp_negative ? -exp_value : exp_value; - } - - float_significand d; - d.w = w; - d.exponent = q; - d.negative = negative; - return d; -} - -/*! -@brief the value of a float token with at most 19 digits, from its layout - -The result is correctly rounded by the lexer's conversion -(detail::decimal_to_float(): Clinger's fast path where both operands are -exact, else the Eisel-Lemire algorithm, which needs no fallback for up to 19 -digits), so it is the value parse() produces. -*/ -template -NLOHMANN_VIEW_ALWAYS_INLINE FloatType layout_float(const unsigned char* p, const unsigned char* e, unsigned int_digits, unsigned frac_digits, const unsigned char* limit) noexcept -{ - return decimal_to_float(layout_decimal(p, e, int_digits, frac_digits, limit)); -} - -/// the value of a float set by an edit: its token (the shortest round-trip -/// text, or "nan", "inf", "-inf") in the edit arena -template -NLOHMANN_VIEW_NOINLINE FloatType edited_float(const char* token, const node& n) -{ - if (token[0] == 'n') - { - return std::numeric_limits::quiet_NaN(); - } - if (token[0] == 'i' || (token[0] == '-' && token[1] == 'i')) - { - return token[0] == 'i' ? std::numeric_limits::infinity() : -std::numeric_limits::infinity(); - } - return float_value(token, n); -} - -/// the value of the float token of a node, as parse() converts it; floats and -/// doubles with at most 19 digits are converted from the digit layout -template -FloatType float_value(const document_data& d, const node& n) -{ - if (NLOHMANN_VIEW_UNLIKELY((n.flags & node_flags::storage) == node_flags::edited)) - { - return edited_float(d.str(n), n); - } - return float_value(d, n, std::integral_constant::value> {}); -} - -template -FloatType float_value(const document_data& d, const node& n, std::true_type /*binary32 or binary64*/) -{ - const unsigned int_digits = n.extra & 0xFFu; - const unsigned frac_digits = n.extra >> 8u; - if (NLOHMANN_VIEW_LIKELY(int_digits + frac_digits <= 19)) // (255 marks "many") - { - // (a float token not written by an edit is in the text) - const auto* const first = reinterpret_cast(d.src + n.off); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - return layout_float(first, first + n.len, int_digits, frac_digits, reinterpret_cast(d.src + d.size)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast) - } - return float_value(d.str(n), n); -} - -template -FloatType float_value(const document_data& d, const node& n, std::false_type /*other*/) -{ - return float_value(d.str(n), n); -} - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include @@ -7112,120 +8024,6 @@ class view_serializer NLOHMANN_JSON_NAMESPACE_END // #include -// __ _____ _____ _____ -// __| | __| | | | JSON for Modern C++ -// | | |__ | | | | | | version 3.12.0 -// |_____|_____|_____|_|___| https://github.com/nlohmann/json -// -// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann -// SPDX-License-Identifier: MIT - - - -#include // min -#include // size_t -#include // memcmp, strlen -#include // basic_string - -// #include -// #include - - -#if NLOHMANN_VIEW_HAS_CPP_17 - #include // string_view -#endif -#ifndef JSON_NO_IO - #include // ostream -#endif - -NLOHMANN_JSON_NAMESPACE_BEGIN -namespace detail -{ -namespace view -{ - -#if NLOHMANN_VIEW_HAS_CPP_17 -using string_ref = std::string_view; -#else -/// minimal C++11 stand-in for std::string_view -class string_ref -{ - public: - using size_type = std::size_t; - using const_iterator = const char*; - - string_ref() noexcept = default; - // s must be null-terminated, as for std::string_view(const char*) - // flawfinder: ignore - string_ref(const char* s) : m_data(s), m_size(std::strlen(s)) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - string_ref(const char* s, std::size_t n) noexcept : m_data(s), m_size(n) {} - template - string_ref(const std::basic_string& s) noexcept : m_data(s.data()), m_size(s.size()) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions) - - const char* data() const noexcept - { - return m_data; - } - std::size_t size() const noexcept - { - return m_size; - } - std::size_t length() const noexcept - { - return m_size; - } - bool empty() const noexcept - { - return m_size == 0; - } - const char* begin() const noexcept - { - return m_data; - } - const char* end() const noexcept - { - return m_data + m_size; - } - char operator[](std::size_t i) const noexcept - { - return m_data[i]; - } - - template - explicit operator std::basic_string() const - { - return std::basic_string(m_data, m_size); - } - - friend bool operator==(string_ref a, string_ref b) noexcept - { - return a.m_size == b.m_size && (a.m_size == 0 || std::memcmp(a.m_data, b.m_data, a.m_size) == 0); - } - friend bool operator!=(string_ref a, string_ref b) noexcept - { - return !(a == b); - } - friend bool operator<(string_ref a, string_ref b) noexcept - { - const int c = std::memcmp(a.m_data, b.m_data, (std::min)(a.m_size, b.m_size)); - return c != 0 ? c < 0 : a.m_size < b.m_size; - } -#ifndef JSON_NO_IO - friend std::ostream& operator<<(std::ostream& o, string_ref s) - { - return o.write(s.m_data, static_cast(s.m_size)); - } -#endif - - private: - const char* m_data = ""; - std::size_t m_size = 0; -}; -#endif - -} // namespace view -} // namespace detail -NLOHMANN_JSON_NAMESPACE_END // #include // __ _____ _____ _____ @@ -8018,8 +8816,12 @@ class basic_json_view template friend class basic_json_document; template friend class basic_json_view; template friend class detail::view::editor; + template friend class detail::view::view_side; friend iterator; + /// whether the view is one of an editable document + static constexpr bool is_editable = Editable; + basic_json_view(const document_data* d, const node* n) noexcept : m_doc(d), m_node(n) {} diff --git a/tests/src/unit-json_view_dump.cpp b/tests/src/unit-json_view_dump.cpp index 3b9da6b89..13d2ac14f 100644 --- a/tests/src/unit-json_view_dump.cpp +++ b/tests/src/unit-json_view_dump.cpp @@ -345,6 +345,33 @@ TEST_CASE("json_view comparison") } } + SECTION("objects whose first keys differ or repeat") + { + const char* const texts[] = + { + R"({"a":1,"b":2})", R"({"a":1,"c":2})", R"({"b":1,"a":2})", R"({"a":1,"a":2,"b":3})", R"({"a":2,"b":3})", + R"({"a":1,"b":1,"a":2})", R"({"a":2,"b":1})", R"({"a":1,"a":2})", R"({"b":1})", R"({"a":1,"b":2,"c":3,"d":4,"e":5})", + R"({"a":1,"b":2,"c":3,"d":4,"f":5})", R"({"a":1,"b":2,"c":3,"a":4,"e":5})", R"({"a":4,"b":2,"c":3,"e":5})", + R"({"\u0061":1,"b":2})", R"({})", R"([{"a":1,"b":2}])", R"([{"a":1,"c":2}])" + }; + for (const char* const x : texts) + { + for (const char* const y : texts) + { + CAPTURE(x) + CAPTURE(y) + const ordered_json_document dx = ordered_json_document::parse(x); + const ordered_json_document dy = ordered_json_document::parse(y); + const ordered_json_editable_document ex = ordered_json_editable_document::parse(x); + const ordered_json jx = ordered_json::parse(x); + const ordered_json jy = ordered_json::parse(y); + CHECK((dx.root() == dy.root()) == (jx == jy)); + CHECK((dx.root() == jy) == (jx == jy)); + CHECK((ex.root() == dy.root()) == (jx == jy)); + } + } + } + SECTION("numbers, duplicate keys, member order") { const auto same = [](const char* x, const char* y) @@ -385,6 +412,129 @@ TEST_CASE("json_view comparison") CHECK(!(null_document.root() == discarded)); } + SECTION("values that are the same text, or the same value written differently, at any depth") + { + // nested arrays and objects around depths where the walk over equal text gives up + for (const int depth : + { + 1, 5, 47, 48, 49, 50, 60, 300 + }) + { + CAPTURE(depth) + std::string open; + std::string close; + for (int i = 0; i < depth; ++i) + { + open += i % 2 == 0 ? R"({"k":[1,"s",{"x":true},)" : R"([null,2.5,{"y":)"; + close = (i % 2 == 0 ? "]}" : "}]") + close; + } + // the same text, one number changed, and one number written differently + const std::string text = open + "7" + close; + const std::string changed = open + "8" + close; + const std::string spelled = open + "7.0" + close; + const json_document a = json_document::parse(text); + const json_document b = json_document::parse(text); + const json_document c = json_document::parse(changed); + const json_document d = json_document::parse(spelled); + CHECK(a.root() == b.root()); + CHECK(a.root() != c.root()); + CHECK(c.root() != a.root()); + CHECK(a.root() == d.root()); + CHECK(d.root() == a.root()); + CHECK(a.root() == json::parse(text)); + CHECK(a.root() != json::parse(changed)); + CHECK(json::parse(spelled) == a.root()); + const nlohmann::json_editable_document e = nlohmann::json_editable_document::parse(text); + CHECK(a.root() == e.root()); + CHECK(e.root() == a.root()); + CHECK(c.root() != e.root()); + CHECK(e.root() == e.root()); + const ordered_json_document oa = ordered_json_document::parse(text); + const ordered_json_document oc = ordered_json_document::parse(changed); + CHECK(oa.root() != oc.root()); + CHECK(oa.root() == ordered_json::parse(text)); + CHECK(oc.root() != ordered_json::parse(text)); + } + + // not the same text, but the same value + const auto same = [](const char* x, const char* y) + { + const json_document dx = json_document::parse(x); + const json_document dy = json_document::parse(y); + const bool equal = dx.root() == dy.root(); + CHECK(equal == (json::parse(x) == json::parse(y))); + CHECK(equal == (dx.root() == json::parse(y))); + return equal; + }; + CHECK(same(R"({"a":1,"a":2})", R"({"a":3,"a":2})")); + CHECK(same(R"({"a":1,"b":[1,{"c":2,"c":3}]})", R"({"b":[1,{"c":9,"c":3}],"a":1})")); + CHECK(!same(R"({"a":1,"a":2})", R"({"a":2,"a":3})")); + CHECK(same("[1.0,-0,0,1e2]", "[1.00,0,-0,100.0]")); + CHECK(same(R"(["\u00e9","\n"])", "[\"\xc3\xa9\",\"\\n\"]")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":false}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"c":null}]})")); + CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":null,"c":1}]})")); + + // edited documents: moved sequences, links, and values of new nodes + nlohmann::json_editable_document edited = nlohmann::json_editable_document::parse(R"({"a":[1,2,3],"b":{"c":"d"},"e":0.5})"); + const json_document plain = json_document::parse(R"({"a":[1,2,3,4],"b":{"c":"x","y":null},"e":0.5,"z":[]})"); + edited.push_back(edited.root()["a"], 4); + edited.set(edited.root()["b"], "c", "x"); + edited.set(edited.root()["b"], "y", nullptr); + edited.set(edited.root(), "z", json::array()); + CHECK(edited.root() == plain.root()); + CHECK(plain.root() == edited.root()); + edited.erase(edited.root()["a"], 0); + CHECK(edited.root() != plain.root()); + CHECK(edited.root()["a"] == json::parse("[2,3,4]")); + const json_document tail = json_document::parse("[2,3,4]"); + CHECK(edited.root()["a"] == tail.root()); + } + + SECTION("large documents, with a difference at any place") + { + const auto make = [](std::size_t count, std::size_t changed, bool swap_members) + { + std::string text = R"({"b":1,"list":[)"; + for (std::size_t i = 0; i < count; ++i) + { + text += (i != 0 ? "," : ""); + const std::string id = std::to_string(i == changed ? i + 1 : i); + text += swap_members && i % 3 == 0 ? R"({"v":[1.5,"s",{"k":)" + id + R"(}],"id":)" + id + "}" : R"({"id":)" + id + R"(,"v":[1.5,"s",{"k":)" + id + "}]}"; + } + return text + R"(],"a":{"y":2,"x":3}})"; + }; + const std::size_t count = 3000; + for (const std::size_t changed : + { + count, std::size_t{0}, std::size_t{1}, count / 2, count - 1 + }) + { + for (const bool swap : + { + false, true + }) + { + CAPTURE(changed) + CAPTURE(swap) + const std::string text_a = make(count, count, false); + const std::string text_b = make(count, changed, swap); + const json_document a = json_document::parse(text_a); + const json_document b = json_document::parse(text_b); + const bool equal = changed == count; + CHECK((a.root() == b.root()) == equal); + CHECK((b.root() == a.root()) == equal); + CHECK((a.root() == json::parse(text_b)) == equal); + CHECK((json::parse(text_b) == a.root()) == equal); + CHECK((a.root()["list"] == b.root()["list"]) == equal); + const ordered_json_document oa = ordered_json_document::parse(text_a); + const ordered_json_document ob = ordered_json_document::parse(text_b); + CHECK((oa.root() == ob.root()) == (equal && !swap)); + CHECK((oa.root() == ordered_json::parse(text_b)) == (equal && !swap)); + } + } + } + SECTION("deep nesting") { const std::string deep = std::string(100000, '[') + std::string(100000, ']');