Merge branch 'develop' into json-view/5869-7-check-utf8

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

docs/mkdocs/docs/api/basic_json_document/shrink_to_fit.md

@@ -23,6 +23,10 @@ Linear in [`node_count()`](node_count.md) plus the length of the decoded strings
 
 ## Notes
 
+The node index of a small document (an estimate of up to 1024 nodes, which is up to about 4 KiB of minified text) is
+allocated together with the document's header, so that parsing needs one allocation. That part cannot be released
+separately: `shrink_to_fit()` only trims a node index that has its own allocation, and the decoded strings.
+
 !!! warning "Invalidates views"
 
     Unlike moving the document, `shrink_to_fit()` **invalidates every view taken from this document before the

include/nlohmann/detail/view/builder.hpp

@@ -543,7 +543,15 @@ run_found:
     NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n)
     {
         const std::size_t used = arena_used();
-        m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, used + n + 256));
+        std::size_t wanted = used + n + 256;
+        if (m_doc.arena.empty())
+        {
+            // the first growth: a decoded string is never longer than its source,
+            // so a fraction of the input (capped, as the buffer is zero-filled)
+            // covers the escapes of a typical document without regrowing it
+            wanted = (std::max)(wanted, (std::min)((static_cast<std::size_t>(m_end - m_begin) / 4) + 16, std::size_t{4096}));
+        }
+        m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, wanted));
         m_arena_write = m_doc.arena.data() + used;
         m_arena_end = m_doc.arena.data() + m_doc.arena.size();
     }

include/nlohmann/detail/view/document_data.hpp

@@ -82,14 +82,17 @@ struct document_data
     };
     std::unique_ptr<edit_state> edits; ///< created by the first edit
 
+    /// the most nodes that are allocated together with the header (16 KiB)
+    static constexpr std::size_t max_inline_nodes = 1024;
+
     /// One allocation for the header and room for `count` nodes, so that a
     /// small document needs one allocation instead of two. Large documents get
     /// a separate node array instead (so it can be trimmed). When a document
-    /// outgrows the inline array, that array stays unused: at most 256 nodes
+    /// outgrows the inline array, that array stays unused: at most 1024 nodes
-    /// (4 KiB), and only for documents whose estimate was too small.
+    /// (16 KiB), and only for documents whose estimate was too small.
     static document_data* create(std::size_t count)
     {
-        count = count <= 256 ? count : 0;
+        count = count <= max_inline_nodes ? count : 0;
         void* mem = ::operator new (sizeof(document_data) + (count * sizeof(node)));
         auto* d = new (mem) document_data(); // NOLINT(cppcoreguidelines-owning-memory): owned by the returned pointer, freed by deleter
         // (aligned: sizeof is a multiple of the alignment; through void*, as GCC's -Wcast-align wants)

include/nlohmann/json_view.hpp

@@ -54,9 +54,11 @@
 // IWYU pragma: no_include <version>
 // IWYU pragma: no_include "nlohmann/detail/abi_config.hpp"
 // IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp"
+// IWYU pragma: no_include "nlohmann/detail/conversions/from_json.hpp"
 // IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp"
 // IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp"
 // IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp"
+// IWYU pragma: no_include "nlohmann/detail/meta/type_traits.hpp"
 // IWYU pragma: no_include "nlohmann/detail/string_concat.hpp"
 // IWYU pragma: no_include "nlohmann/detail/value_t.hpp"
 // IWYU pragma: no_include "nlohmann/json.hpp"
@@ -1313,7 +1315,7 @@ class basic_json_document
     /// member is set (added if missing), an array element assigned, and "-"
     /// or the size of the array appends. A null parent becomes what
     /// basic_json's operator[](json_pointer) makes of it: an array for "-"
-    /// and for digits (padded with nulls up to the index), an object otherwise.
+    /// and for array indices (padded with nulls up to the index), an object otherwise.
     template<typename V>
     view_type set(const json_pointer& ptr, V&& value)
     {
@@ -1329,7 +1331,11 @@ class basic_json_document
             {
                 return push_back(parent, std::forward<V>(value));
             }
-            const bool digits = std::all_of(token.begin(), token.end(), [](const char c)
+            // (a token that cannot be an array index, such as "01" or "", names an
+            // object member, as in basic_json)
+            const bool digits = !token.empty()
+                                && (token.size() == 1 || token[0] != '0')
+                                && std::all_of(token.begin(), token.end(), [](const char c)
             {
                 return c >= '0' && c <= '9';
             });

single_include/nlohmann/json_view.hpp

@@ -54,9 +54,11 @@
 // IWYU pragma: no_include <version>
 // IWYU pragma: no_include "nlohmann/detail/abi_config.hpp"
 // IWYU pragma: no_include "nlohmann/detail/abi_macros.hpp"
+// IWYU pragma: no_include "nlohmann/detail/conversions/from_json.hpp"
 // IWYU pragma: no_include "nlohmann/detail/input/input_adapters.hpp"
 // IWYU pragma: no_include "nlohmann/detail/json_pointer.hpp"
 // IWYU pragma: no_include "nlohmann/detail/meta/cpp_future.hpp"
+// IWYU pragma: no_include "nlohmann/detail/meta/type_traits.hpp"
 // IWYU pragma: no_include "nlohmann/detail/string_concat.hpp"
 // IWYU pragma: no_include "nlohmann/detail/value_t.hpp"
 // IWYU pragma: no_include "nlohmann/json.hpp"
@@ -410,14 +412,17 @@ struct document_data
     };
     std::unique_ptr<edit_state> edits; ///< created by the first edit
 
+    /// the most nodes that are allocated together with the header (16 KiB)
+    static constexpr std::size_t max_inline_nodes = 1024;
+
     /// One allocation for the header and room for `count` nodes, so that a
     /// small document needs one allocation instead of two. Large documents get
     /// a separate node array instead (so it can be trimmed). When a document
-    /// outgrows the inline array, that array stays unused: at most 256 nodes
+    /// outgrows the inline array, that array stays unused: at most 1024 nodes
-    /// (4 KiB), and only for documents whose estimate was too small.
+    /// (16 KiB), and only for documents whose estimate was too small.
     static document_data* create(std::size_t count)
     {
-        count = count <= 256 ? count : 0;
+        count = count <= max_inline_nodes ? count : 0;
         void* mem = ::operator new (sizeof(document_data) + (count * sizeof(node)));
         auto* d = new (mem) document_data(); // NOLINT(cppcoreguidelines-owning-memory): owned by the returned pointer, freed by deleter
         // (aligned: sizeof is a multiple of the alignment; through void*, as GCC's -Wcast-align wants)
@@ -1747,7 +1752,15 @@ run_found:
     NLOHMANN_VIEW_NOINLINE void arena_grow(std::size_t n)
     {
         const std::size_t used = arena_used();
-        m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, used + n + 256));
+        std::size_t wanted = used + n + 256;
+        if (m_doc.arena.empty())
+        {
+            // the first growth: a decoded string is never longer than its source,
+            // so a fraction of the input (capped, as the buffer is zero-filled)
+            // covers the escapes of a typical document without regrowing it
+            wanted = (std::max)(wanted, (std::min)((static_cast<std::size_t>(m_end - m_begin) / 4) + 16, std::size_t{4096}));
+        }
+        m_doc.arena.resize((std::max)(m_doc.arena.size() * 2, wanted));
         m_arena_write = m_doc.arena.data() + used;
         m_arena_end = m_doc.arena.data() + m_doc.arena.size();
     }
@@ -9105,7 +9118,7 @@ class basic_json_document
     /// member is set (added if missing), an array element assigned, and "-"
     /// or the size of the array appends. A null parent becomes what
     /// basic_json's operator[](json_pointer) makes of it: an array for "-"
-    /// and for digits (padded with nulls up to the index), an object otherwise.
+    /// and for array indices (padded with nulls up to the index), an object otherwise.
     template<typename V>
     view_type set(const json_pointer& ptr, V&& value)
     {
@@ -9121,7 +9134,11 @@ class basic_json_document
             {
                 return push_back(parent, std::forward<V>(value));
             }
-            const bool digits = std::all_of(token.begin(), token.end(), [](const char c)
+            // (a token that cannot be an array index, such as "01" or "", names an
+            // object member, as in basic_json)
+            const bool digits = !token.empty()
+                                && (token.size() == 1 || token[0] != '0')
+                                && std::all_of(token.begin(), token.end(), [](const char c)
             {
                 return c >= '0' && c <= '9';
             });

tests/src/unit-bson.cpp

@@ -1311,7 +1311,7 @@ TEST_CASE("BSON declared document size bounds the content")
     // records the events that a SAX parser receives
     struct recorder : nlohmann::json_sax<json>
     {
-        std::string events;
+        std::string events{};
 
         bool null() override
         {

tests/src/unit-json_pointer.cpp

@@ -1122,17 +1122,17 @@ TEST_CASE("flatten of structured values")
 
         // the same for other keys that are not valid array indices
         CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/1a", 2}}).unflatten(),
-                             "[json.exception.out_of_range.404] unresolved reference token '1a'", json::out_of_range&);
+        "[json.exception.out_of_range.404] unresolved reference token '1a'", json::out_of_range&);
         CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/-", 2}}).unflatten(),
-                             "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&);
+        "[json.exception.out_of_range.404] unresolved reference token '-'", json::out_of_range&);
     }
 
     SECTION("unflatten with the empty token among array indices")
     {
         CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/", 2}}).unflatten(),
-                             "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&);
+        "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&);
         CHECK_THROWS_WITH_AS(json({{"/a/0", 1}, {"/a/1", 2}, {"/a/", 3}}).unflatten(),
-                             "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&);
+        "[json.exception.out_of_range.404] unresolved reference token ''", json::out_of_range&);
 
         // without an index "0", the empty token is an ordinary object key
         CHECK(json({{"/a/", 2}}).unflatten() == json({{"a", {{"", 2}}}}));

tests/src/unit-json_view.cpp

@@ -666,6 +666,61 @@ TEST_CASE("json_view")
         CHECK(small.root().materialize() == json::parse("[1,[2,3],{\"a\":\"b\\n\"}]"));
     }
 
+    SECTION("block sizes")
+    {
+        // documents whose node estimate lies around the size that is allocated
+        // with the header, and whose decoded strings need the first growth of
+        // the buffer, in all sizes around the points where the policies change
+        for (const int count :
+                {
+                    1, 2, 10, 50, 100, 255, 256, 257, 500, 1000, 1023, 1024, 1025, 2000, 5000
+                })
+        {
+            // strings with escapes (decoded text, minified)
+            std::string escaped = "[";
+            for (int i = 0; i < count; ++i)
+            {
+                escaped += (i != 0 ? ",\"a\\n" : "\"a\\n") + std::to_string(i) + "\\u00e4\"";
+            }
+            escaped += ']';
+            json_document e = json_document::parse(escaped);
+            CHECK(e.root().materialize() == json::parse(escaped));
+            e.shrink_to_fit();
+            CHECK(e.root().materialize() == json::parse(escaped));
+
+            // dense numbers (more nodes per byte than the estimate expects)
+            std::string dense = "[";
+            for (int i = 0; i < count; ++i)
+            {
+                dense += (i != 0 ? ",1" : "1");
+            }
+            dense += ']';
+            json_document n = json_document::parse(dense);
+            CHECK(n.node_count() == static_cast<std::size_t>(count) + 1);
+            CHECK(n.root().materialize() == json::parse(dense));
+            n.shrink_to_fit();
+            CHECK(n.root().size() == static_cast<std::size_t>(count));
+
+            // the same documents, pretty-printed
+            const std::string pretty = json::parse(escaped).dump(2);
+            json_document p = json_document::parse(pretty);
+            CHECK(p.root().materialize() == json::parse(escaped));
+        }
+
+        // a large document with a single escape does not need a buffer of the
+        // size of its input
+        std::string large = R"(["a\n")";
+        for (int i = 0; i < 100000; ++i)
+        {
+            large += ",1";
+        }
+        large += ']';
+        const json_document l = json_document::parse(large);
+        CHECK(l.root().size() == 100001);
+        CHECK(l.root()[0].get_string() == "a\n");
+        CHECK(l.root()[100000].get<int>() == 1);
+    }
+
     SECTION("source offsets")
     {
         const std::string text = R"(  {"key": "value", "escaped": "a\nb", "n": 42})";

tests/src/unit-json_view_from_json.cpp

@@ -41,6 +41,13 @@ using nlohmann::ordered_json_document;
     #include <optional>
 #endif
 
+// NLOHMANN_JSON_SERIALIZE_ENUM uses a static std::pair; same() compares floating-point values on purpose
+DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
+DOCTEST_CLANG_SUPPRESS_WARNING("-Wfloat-equal")
+DOCTEST_GCC_SUPPRESS_WARNING_PUSH
+DOCTEST_GCC_SUPPRESS_WARNING("-Wfloat-equal")
+
 // The from_json() functions of the NLOHMANN_DEFINE_TYPE_* and
 // NLOHMANN_JSON_SERIALIZE_ENUM macros read views directly; other from_json()
 // functions get the value of materialize().
@@ -61,10 +68,11 @@ struct point
 
 struct shape
 {
-    std::string name;
+    // initialized for GCC -Weffc++
-    std::vector<point> points;
+    std::string name{};
-    point center;
+    std::vector<point> points{};
-    std::map<std::string, point> named;
+    point center{};
+    std::map<std::string, point> named{};
     NLOHMANN_DEFINE_TYPE_INTRUSIVE(shape, name, points, center, named)
 };
 
@@ -79,7 +87,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(with_defaults, a, b, c)
 struct with_names
 {
     int value = 0;
-    std::string text;
+    std::string text{};  // initialized for GCC -Weffc++
 };
 NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_NAMES(with_names, "Value", value, "Text", text)
 
@@ -447,3 +455,6 @@ TEST_CASE("json_view from_json")
     }
 #endif
 }
+
+DOCTEST_CLANG_SUPPRESS_WARNING_POP
+DOCTEST_GCC_SUPPRESS_WARNING_POP

tests/src/unit-pointer_access.cpp

@@ -141,6 +141,7 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         const test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();
@@ -170,6 +171,7 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();
@@ -199,6 +201,7 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         const test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();
@@ -232,6 +235,7 @@ TEST_CASE("pointer access")
 
         const test_type* p2 = value.get_ptr<const test_type*>();
         CHECK(p2 == value.get_ptr<const test_type*>());
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p2 == value.get<test_type>());
 
         const test_type* const p3 = value.get_ptr<const test_type* const>();
@@ -257,6 +261,7 @@ TEST_CASE("pointer access")
         // check if pointers are returned correctly
         const test_type* p1 = value.get_ptr<test_type*>();
         CHECK(p1 == value.get_ptr<test_type*>());
+        // @infer-ignore NULLPTR_DEREFERENCE
         CHECK(*p1 == value.get<test_type>());
 
         const test_type* p2 = value.get_ptr<const test_type*>();