Rename names that Flawfinder takes for the file function open()

The comparison cursors' open() is now enter(), and a test's open and
close strings are head and tail.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/image.hpp

@@ -255,6 +255,82 @@ struct number_token
     bool is_float; ///< a fraction or an exponent
 };
 
+/// advance over ASCII digits, eight bytes at a time
+inline const unsigned char* digit_run(const unsigned char* p, const unsigned char* e) noexcept
+{
+    constexpr std::uint64_t ones = 0x0101010101010101ull;
+    while (e - p >= 8)
+    {
+        // (the digits become 0..9; a byte above 9, or with bit 7 set, is no digit. A carry
+        // out of a byte with bit 7 set only affects the bytes above, after the first one
+        // that count_trailing_zeros finds.)
+        const std::uint64_t x = read_eight_bytes(p) ^ (ones * 0x30u);
+        const std::uint64_t not_digit = ((x + (ones * 0x76u)) | x) & (ones * 0x80u);
+        if (not_digit != 0)
+        {
+            return p + (static_cast<std::size_t>(count_trailing_zeros(not_digit)) / 8);
+        }
+        p += 8;
+    }
+    while (p != e && is_digit(*p))
+    {
+        ++p;
+    }
+    return p;
+}
+
+/// whether the n bytes at p are ASCII digits (n > 0)
+inline bool all_digits(const unsigned char* p, std::size_t n) noexcept
+{
+    constexpr std::uint64_t ones = 0x0101010101010101ull;
+    const auto digits = [](const unsigned char* q)
+    {
+        const std::uint64_t x = read_eight_bytes(q) ^ (ones * 0x30u);
+        return (((x + (ones * 0x76u)) | x) & (ones * 0x80u)) == 0;
+    };
+    if (n >= 8)
+    {
+        std::size_t i = 0;
+        for (; i + 8 < n; i += 8)
+        {
+            if (!digits(p + i))
+            {
+                return false;
+            }
+        }
+        return digits(p + n - 8); // (overlapping the bytes before: they are digits)
+    }
+    for (std::size_t i = 0; i < n; ++i)
+    {
+        if (!is_digit(p[i]))
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+/// Whether a float token has the shape that the parser records for most of
+/// them: [-]digits.digits, with the numbers of digits of the node's layout
+/// (extra). Then the token is a number of that layout, and the general scan
+/// that follows is not needed.
+inline bool plain_float_token(const unsigned char* s, std::size_t len, std::uint16_t extra) noexcept
+{
+    const std::size_t int_digits = extra & 0xFFu;
+    const std::size_t frac_digits = extra >> 8u;
+    if (int_digits == 0 || int_digits >= 255 || frac_digits == 0 || frac_digits >= 255 || len == 0)
+    {
+        return false;
+    }
+    const std::size_t negative = *s == '-' ? 1 : 0;
+    if (len != negative + int_digits + 1 + frac_digits)
+    {
+        return false;
+    }
+    const unsigned char* const p = s + negative;
+    return !(int_digits > 1 && *p == '0') && p[int_digits] == '.' && all_digits(p, int_digits) && all_digits(p + int_digits + 1, frac_digits);
+}
+
 /// whether [s, s + len) is a JSON number (the grammar the parser accepts)
 inline bool scan_number_token(const unsigned char* s, std::size_t len, number_token& t)
 {
@@ -273,11 +349,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to
     }
     else if (*p >= '1' && *p <= '9')
     {
-        do
+        p = digit_run(p + 1, e);
-        {
-            ++p;
-        }
-        while (p != e && is_digit(*p));
     }
     else
     {
@@ -289,10 +361,7 @@ inline bool scan_number_token(const unsigned char* s, std::size_t len, number_to
     if (p != e && *p == '.')
     {
         const unsigned char* const f0 = ++p;
-        while (p != e && is_digit(*p))
+        p = digit_run(p, e);
-        {
-            ++p;
-        }
         if (p == f0)
         {
             return false;
@@ -403,10 +472,15 @@ struct float_range
 template<typename Check>
 bool check_distinct_ranges(std::vector<byte_range>& ranges, Check check)
 {
-    std::sort(ranges.begin(), ranges.end(), [](const byte_range & a, const byte_range & b)
+    // (save() writes the ranges in the order of the nodes, which is sorted already)
+    const auto before = [](const byte_range & a, const byte_range & b)
     {
         return a.off != b.off ? a.off < b.off : a.len < b.len;
-    });
+    };
+    if (!std::is_sorted(ranges.begin(), ranges.end(), before))
+    {
+        std::sort(ranges.begin(), ranges.end(), before);
+    }
     std::size_t end = 0;
     for (std::size_t i = 0; i < ranges.size(); ++i)
     {
@@ -428,7 +502,8 @@ bool check_distinct_ranges(std::vector<byte_range>& ranges, Check check)
 /// record layouts that match it), tokens must not overlap.
 inline bool check_float_ranges(std::vector<float_range>& ranges, const unsigned char* text)
 {
-    std::sort(ranges.begin(), ranges.end(), [](const float_range & a, const float_range & b)
+    // (save() writes the tokens in the order of the nodes, which is sorted already)
+    const auto before = [](const float_range & a, const float_range & b)
     {
         if (a.off != b.off)
         {
@@ -439,7 +514,11 @@ inline bool check_float_ranges(std::vector<float_range>& ranges, const unsigned
             return a.len < b.len;
         }
         return a.extra < b.extra;
-    });
+    };
+    if (!std::is_sorted(ranges.begin(), ranges.end(), before))
+    {
+        std::sort(ranges.begin(), ranges.end(), before);
+    }
     std::size_t end = 0;
     std::size_t i = 0;
     while (i < ranges.size())
@@ -449,14 +528,18 @@ inline bool check_float_ranges(std::vector<float_range>& ranges, const unsigned
         {
             return false;
         }
-        number_token t{};
         const unsigned char* const s = text + r.off;
-        if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t))
+        std::uint16_t layout = r.extra;
+        if (!plain_float_token(s, r.len, r.extra))
         {
-            return false;
+            number_token t{};
+            if (!scan_number_token(s, r.len, t) || !float_token_finite(s, r.len, t))
+            {
+                return false;
+            }
+            // the digit layout the parser records (or "many", as compaction writes it)
+            layout = float_layout(t);
         }
-        // the digit layout the parser records (or "many", as compaction writes it)
-        const std::uint16_t layout = float_layout(t);
         for (; i < ranges.size() && ranges[i].off == r.off && ranges[i].len == r.len; ++i)
         {
             if (ranges[i].extra != layout && ranges[i].extra != 0xFFFFu)

include/nlohmann/detail/view/node.hpp

@@ -120,6 +120,74 @@ NLOHMANN_VIEW_ALWAYS_INLINE std::uint32_t number_length(const node& n) noexcept
            : (n.extra & 0xFFu) + (n.kind == static_cast<std::uint8_t>(value_t::number_integer) ? 1u : 0u);
 }
 
+/// whether n bytes are equal; the short texts of JSON (keys, numbers, most
+/// strings) are compared by two overlapping loads instead of a library call
+NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_equal(const char* x, const char* y, std::size_t n) noexcept
+{
+    const auto load8 = [](const char* p)
+    {
+        std::uint64_t v = 0;
+        std::memcpy(&v, p, sizeof(v));
+        return v;
+    };
+    const auto load4 = [](const char* p)
+    {
+        std::uint32_t v = 0;
+        std::memcpy(&v, p, sizeof(v));
+        return v;
+    };
+    if (n >= 8)
+    {
+        std::size_t i = 0;
+        for (; i + 8 < n; i += 8)
+        {
+            if (load8(x + i) != load8(y + i))
+            {
+                return false;
+            }
+        }
+        return load8(x + n - 8) == load8(y + n - 8);
+    }
+    if (n >= 4)
+    {
+        return load4(x) == load4(y) && load4(x + n - 4) == load4(y + n - 4);
+    }
+    for (std::size_t i = 0; i < n; ++i)
+    {
+        if (x[i] != y[i])
+        {
+            return false;
+        }
+    }
+    return true;
+}
+
+/// whether the n1 bytes at x are before the n2 bytes at y (memcmp() order, then the shorter first);
+/// the first difference is found eight bytes at a time
+NLOHMANN_VIEW_ALWAYS_INLINE bool bytes_less(const char* x, std::size_t n1, const char* y, std::size_t n2) noexcept
+{
+    const std::size_t n = n1 < n2 ? n1 : n2;
+    std::size_t i = 0;
+    for (; i + 8 <= n; i += 8)
+    {
+        const std::uint64_t u = read_eight_bytes(x + i);
+        const std::uint64_t v = read_eight_bytes(y + i);
+        if (u != v)
+        {
+            const auto shift = static_cast<unsigned>(count_trailing_zeros(u ^ v)) & ~7u; // (the first byte that differs)
+            return ((u >> shift) & 0xFFu) < ((v >> shift) & 0xFFu);
+        }
+    }
+    for (; i < n; ++i)
+    {
+        if (x[i] != y[i])
+        {
+            return static_cast<unsigned char>(x[i]) < static_cast<unsigned char>(y[i]);
+        }
+    }
+    return n1 < n2;
+}
+
 /// estimated number of nodes for an input of `size` bytes (one node per ~12
 /// bytes covers typical documents without regrowth)
 inline std::size_t estimate_nodes(std::size_t size) noexcept

include/nlohmann/json_view.hpp

@@ -723,8 +723,12 @@ class basic_json_view
     template<typename, bool> friend class basic_json_document;
     template<typename, bool> friend class basic_json_view;
     template<typename, typename> friend class detail::view::editor;
+    template<typename, typename> friend class detail::view::view_side;
     friend iterator;
 
+    /// whether the view is one of an editable document
+    static constexpr bool is_editable = Editable;
+
     basic_json_view(const document_data* d, const node* n) noexcept
         : m_doc(d), m_node(n)
     {}

tests/src/unit-json_view_dump.cpp

@@ -345,6 +345,33 @@ TEST_CASE("json_view comparison")
         }
     }
 
+    SECTION("objects whose first keys differ or repeat")
+    {
+        const char* const texts[] =
+        {
+            R"({"a":1,"b":2})", R"({"a":1,"c":2})", R"({"b":1,"a":2})", R"({"a":1,"a":2,"b":3})", R"({"a":2,"b":3})",
+            R"({"a":1,"b":1,"a":2})", R"({"a":2,"b":1})", R"({"a":1,"a":2})", R"({"b":1})", R"({"a":1,"b":2,"c":3,"d":4,"e":5})",
+            R"({"a":1,"b":2,"c":3,"d":4,"f":5})", R"({"a":1,"b":2,"c":3,"a":4,"e":5})", R"({"a":4,"b":2,"c":3,"e":5})",
+            R"({"\u0061":1,"b":2})", R"({})", R"([{"a":1,"b":2}])", R"([{"a":1,"c":2}])"
+        };
+        for (const char* const x : texts)
+        {
+            for (const char* const y : texts)
+            {
+                CAPTURE(x)
+                CAPTURE(y)
+                const ordered_json_document dx = ordered_json_document::parse(x);
+                const ordered_json_document dy = ordered_json_document::parse(y);
+                const ordered_json_editable_document ex = ordered_json_editable_document::parse(x);
+                const ordered_json jx = ordered_json::parse(x);
+                const ordered_json jy = ordered_json::parse(y);
+                CHECK((dx.root() == dy.root()) == (jx == jy));
+                CHECK((dx.root() == jy) == (jx == jy));
+                CHECK((ex.root() == dy.root()) == (jx == jy));
+            }
+        }
+    }
+
     SECTION("numbers, duplicate keys, member order")
     {
         const auto same = [](const char* x, const char* y)
@@ -385,6 +412,129 @@ TEST_CASE("json_view comparison")
         CHECK(!(null_document.root() == discarded));
     }
 
+    SECTION("values that are the same text, or the same value written differently, at any depth")
+    {
+        // nested arrays and objects around depths where the walk over equal text gives up
+        for (const int depth :
+                {
+                    1, 5, 47, 48, 49, 50, 60, 300
+                })
+        {
+            CAPTURE(depth)
+            std::string head;
+            std::string tail;
+            for (int i = 0; i < depth; ++i)
+            {
+                head += i % 2 == 0 ? R"({"k":[1,"s",{"x":true},)" : R"([null,2.5,{"y":)";
+                tail = (i % 2 == 0 ? "]}" : "}]") + tail;
+            }
+            // the same text, one number changed, and one number written differently
+            const std::string text = head + "7" + tail;
+            const std::string changed = head + "8" + tail;
+            const std::string spelled = head + "7.0" + tail;
+            const json_document a = json_document::parse(text);
+            const json_document b = json_document::parse(text);
+            const json_document c = json_document::parse(changed);
+            const json_document d = json_document::parse(spelled);
+            CHECK(a.root() == b.root());
+            CHECK(a.root() != c.root());
+            CHECK(c.root() != a.root());
+            CHECK(a.root() == d.root());
+            CHECK(d.root() == a.root());
+            CHECK(a.root() == json::parse(text));
+            CHECK(a.root() != json::parse(changed));
+            CHECK(json::parse(spelled) == a.root());
+            const nlohmann::json_editable_document e = nlohmann::json_editable_document::parse(text);
+            CHECK(a.root() == e.root());
+            CHECK(e.root() == a.root());
+            CHECK(c.root() != e.root());
+            CHECK(e.root() == e.root());
+            const ordered_json_document oa = ordered_json_document::parse(text);
+            const ordered_json_document oc = ordered_json_document::parse(changed);
+            CHECK(oa.root() != oc.root());
+            CHECK(oa.root() == ordered_json::parse(text));
+            CHECK(oc.root() != ordered_json::parse(text));
+        }
+
+        // not the same text, but the same value
+        const auto same = [](const char* x, const char* y)
+        {
+            const json_document dx = json_document::parse(x);
+            const json_document dy = json_document::parse(y);
+            const bool equal = dx.root() == dy.root();
+            CHECK(equal == (json::parse(x) == json::parse(y)));
+            CHECK(equal == (dx.root() == json::parse(y)));
+            return equal;
+        };
+        CHECK(same(R"({"a":1,"a":2})", R"({"a":3,"a":2})"));
+        CHECK(same(R"({"a":1,"b":[1,{"c":2,"c":3}]})", R"({"b":[1,{"c":9,"c":3}],"a":1})"));
+        CHECK(!same(R"({"a":1,"a":2})", R"({"a":2,"a":3})"));
+        CHECK(same("[1.0,-0,0,1e2]", "[1.00,0,-0,100.0]"));
+        CHECK(same(R"(["\u00e9","\n"])", "[\"\xc3\xa9\",\"\\n\"]"));
+        CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":false}]})"));
+        CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"c":null}]})"));
+        CHECK(!same(R"({"a":[1,2,{"b":null}]})", R"({"a":[1,2,{"b":null,"c":1}]})"));
+
+        // edited documents: moved sequences, links, and values of new nodes
+        nlohmann::json_editable_document edited = nlohmann::json_editable_document::parse(R"({"a":[1,2,3],"b":{"c":"d"},"e":0.5})");
+        const json_document plain = json_document::parse(R"({"a":[1,2,3,4],"b":{"c":"x","y":null},"e":0.5,"z":[]})");
+        edited.push_back(edited.root()["a"], 4);
+        edited.set(edited.root()["b"], "c", "x");
+        edited.set(edited.root()["b"], "y", nullptr);
+        edited.set(edited.root(), "z", json::array());
+        CHECK(edited.root() == plain.root());
+        CHECK(plain.root() == edited.root());
+        edited.erase(edited.root()["a"], 0);
+        CHECK(edited.root() != plain.root());
+        CHECK(edited.root()["a"] == json::parse("[2,3,4]"));
+        const json_document tail = json_document::parse("[2,3,4]");
+        CHECK(edited.root()["a"] == tail.root());
+    }
+
+    SECTION("large documents, with a difference at any place")
+    {
+        const auto make = [](std::size_t count, std::size_t changed, bool swap_members)
+        {
+            std::string text = R"({"b":1,"list":[)";
+            for (std::size_t i = 0; i < count; ++i)
+            {
+                text += (i != 0 ? "," : "");
+                const std::string id = std::to_string(i == changed ? i + 1 : i);
+                text += swap_members && i % 3 == 0 ? R"({"v":[1.5,"s",{"k":)" + id + R"(}],"id":)" + id + "}" : R"({"id":)" + id + R"(,"v":[1.5,"s",{"k":)" + id + "}]}";
+            }
+            return text + R"(],"a":{"y":2,"x":3}})";
+        };
+        const std::size_t count = 3000;
+        for (const std::size_t changed :
+                {
+                    count, std::size_t{0}, std::size_t{1}, count / 2, count - 1
+                })
+        {
+            for (const bool swap :
+                    {
+                        false, true
+                    })
+            {
+                CAPTURE(changed)
+                CAPTURE(swap)
+                const std::string text_a = make(count, count, false);
+                const std::string text_b = make(count, changed, swap);
+                const json_document a = json_document::parse(text_a);
+                const json_document b = json_document::parse(text_b);
+                const bool equal = changed == count;
+                CHECK((a.root() == b.root()) == equal);
+                CHECK((b.root() == a.root()) == equal);
+                CHECK((a.root() == json::parse(text_b)) == equal);
+                CHECK((json::parse(text_b) == a.root()) == equal);
+                CHECK((a.root()["list"] == b.root()["list"]) == equal);
+                const ordered_json_document oa = ordered_json_document::parse(text_a);
+                const ordered_json_document ob = ordered_json_document::parse(text_b);
+                CHECK((oa.root() == ob.root()) == (equal && !swap));
+                CHECK((oa.root() == ordered_json::parse(text_b)) == (equal && !swap));
+            }
+        }
+    }
+
     SECTION("deep nesting")
     {
         const std::string deep = std::string(100000, '[') + std::string(100000, ']');

tests/src/unit-json_view_image.cpp

@@ -682,6 +682,66 @@ TEST_CASE("json_view images: ownership")
 #if !defined(JSON_NOEXCEPTION)
 TEST_CASE("json_view images: errors")
 {
+#if !defined(JSON_NOEXCEPTION)
+    SECTION("float tokens: a changed byte is accepted if the text is still a number with the digits the node records")
+    {
+        for (const char* const token :
+                {
+                    "-12.375", "0.5", "123456.0", "-0.0", "9999999999999999999.5", "0.000000000000000000000001", "1.25e10", "5E-3"
+                })
+        {
+            CAPTURE(token)
+            const std::string text = std::string("[") + token + "]";
+            const json_document d = json_document::parse(text);
+            const std::vector<std::uint8_t> image = d.save();
+            const std::size_t at = text_at(image) + 1; // the first byte of the token
+            const std::uint16_t recorded = node_at(image, 1).extra;
+            CHECK(loaded_dump(image) == json::parse(text).dump());
+            for (std::size_t pos = 0; token[pos] != '\0'; ++pos)
+            {
+                for (unsigned int byte = 0; byte < 256; ++byte)
+                {
+                    std::vector<std::uint8_t> changed = image;
+                    changed[at + pos] = static_cast<std::uint8_t>(byte);
+                    std::string t = token;
+                    t[pos] = static_cast<char>(byte);
+                    // the digits of the integer part and of the fraction of the changed token
+                    std::size_t integer = 0;
+                    std::size_t fraction = 0;
+                    std::size_t k = (t[0] == '-') ? 1 : 0;
+                    for (; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k)
+                    {
+                        ++integer;
+                    }
+                    if (k < t.size() && t[k] == '.')
+                    {
+                        for (++k; k < t.size() && t[k] >= '0' && t[k] <= '9'; ++k)
+                        {
+                            ++fraction;
+                        }
+                    }
+                    // (as a JSON text, the token may have white space around it)
+                    const bool float_token = json::accept(t) && t.find_first_of(std::string(" \t\r\n\0", 5)) == std::string::npos && (t.find('.') != std::string::npos || t.find_first_of("eE") != std::string::npos)
+                                             && ((integer | (fraction << 8U)) == recorded);
+                    CAPTURE(pos)
+                    CAPTURE(byte)
+                    std::string result;
+                    try
+                    {
+                        result = loaded_dump(changed);
+                    }
+                    catch (const json::parse_error& e)
+                    {
+                        CHECK(std::string(e.what()) == check_failed);
+                        result = "rejected";
+                    }
+                    CHECK((result != "rejected") == float_token);
+                }
+            }
+        }
+    }
+#endif
+
     SECTION("a literal as the root: dump() after loading")
     {
         for (const char* text :