Address the review of the view builder and document

Changes promised in the replies to the review of #5620:

- read() reuses the owned copy: a copied input (an rvalue byte container,
  a stream, FILE*, wide characters, an iterator range) is appended to the
  cleared owned buffer instead of a fresh std::string being move-assigned
  over it. A moved-in std::string still replaces it.
- The error replay runs the library parser with a SAX handler that stores
  nothing and throws the reported exception, instead of
  BasicJsonType::parse(), which built a DOM up to the error.
- Pointers to and arrays of wide characters, and FILE*, take the input
  adapter path (converted to UTF-8 into an owned buffer) instead of
  failing a static_assert. Wide pointers are NUL-terminated.
- Streams are read in blocks through rdbuf() into the buffer, instead of
  character by character through the input adapter; eofbit is set at the
  end, as by the library's stream adapter.
- tape/tape_size/tape_cap are now nodes/node_count/node_capacity.
- The builder's members get descriptive, m_-prefixed names.
- The shallow frame stack is a std::array (in a wrapper with a
  user-provided constructor: -Weffc++ wants the member initialized, and
  value-initializing the 64 frames costs up to 30 % on small documents).
- The value dispatch macro has no NEXT parameter; the switch is left with
  break, and the macro is undefined after its last use.
- literal() and digits_exceed() deduce the length from the string literal;
  each literal has one helper; the two `if (!is_float)` blocks are merged.
- arena_ensure() sits right before the writes; the decode arena is a
  std::vector<char>.
- The builder records the deepest nesting, and materialize() sizes its
  stack from it once (growing it only if a subtree is deeper).
- Comment fixes (hex4, slow_string, the NUL rule, put_utf8, 0 - m, the
  arena resize, grow()) and formatting (macro backslashes, digit cases,
  one power of ten per line, value_t assertion next to is_container()).
- build_with's template parameters follow the builder's order.

Greg's idea of keeping the source offset in `off` and the arena offset in
`next` for escaped strings was measured and not adopted: string access got
7-18 % slower with a select, and 3.5-18 % slower with a branch-free field
pick (Apple M-series, clang -O3), parsing was unchanged.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/src/unit-json_view.cpp

@@ -17,6 +17,9 @@ using nlohmann::ordered_json_document;
 
 #include <cstddef>
 #include <cstdint>
+#include <cstdio>
+#include <ios>
+#include <istream>
 #include <list>
 #include <map>
 #include <random>
@@ -396,6 +399,112 @@ TEST_CASE("json_view")
         CHECK(materialized<json_document>(wide) == json::parse(wide));
         CHECK(json_document::parse(static_cast<const char*>(nullptr), false).is_discarded());
         CHECK(json_document::parse("", false).is_discarded());
+
+        // wide-character arrays and pointers are converted to UTF-8, as by
+        // parse(); the pointers are NUL-terminated
+        CHECK(materialized<json_document>(L"[\"ä€\", 1]") == json::parse(L"[\"ä€\", 1]"));
+        CHECK(materialized<json_document>(u"[\"ä€\", 1]") == json::parse(u"[\"ä€\", 1]"));
+        CHECK(materialized<json_document>(U"[\"ä€\", 1]") == json::parse(U"[\"ä€\", 1]"));
+        const wchar_t* const wide_pointer = wide.c_str();
+        const json_document from_wide_pointer = json_document::parse(wide_pointer);
+        CHECK(from_wide_pointer.owns_source());
+        CHECK(from_wide_pointer.root().materialize() == json::parse(wide));
+        const std::u16string wide16 = u"{\"\U0001F600\": [true]}";
+        CHECK(materialized<json_document>(wide16.c_str()) == json::parse(wide16));
+        const std::u32string wide32 = U"{\"\U0001F600\": [true]}";
+        CHECK(materialized<json_document>(wide32.c_str()) == json::parse(wide32));
+        CHECK(json_document::parse(static_cast<const wchar_t*>(nullptr), false).is_discarded());
+
+        // FILE*: read to its end
+        std::FILE* file = std::tmpfile(); // NOLINT(cppcoreguidelines-owning-memory)
+        if (file != nullptr)
+        {
+            CHECK(std::fputs(text.c_str(), file) >= 0);
+            CHECK(std::fseek(file, 0, SEEK_SET) == 0);
+            const json_document from_file = json_document::parse(file);
+            CHECK(from_file.owns_source());
+            CHECK(from_file.root().materialize() == expected);
+            CHECK(std::fclose(file) == 0); // NOLINT(cppcoreguidelines-owning-memory)
+        }
+    }
+
+    SECTION("streams")
+    {
+        // longer than one block of the bulk read
+        std::string text = "[";
+        for (int i = 0; i < 5000; ++i)
+        {
+            text += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\"";
+        }
+        text += ']';
+        std::istringstream stream(text);
+        const json_document from_stream = json_document::parse(stream);
+        CHECK(from_stream.source() == text);
+        CHECK(from_stream.root().materialize() == json::parse(text));
+        // the stream is read to its end, and eofbit set, as by parse()
+        CHECK(stream.eof());
+        CHECK(!stream.bad());
+
+        std::istringstream empty_stream;
+        CHECK(json_document::parse(empty_stream, false).is_discarded());
+        CHECK(empty_stream.eof());
+        std::istringstream rvalue_source("[1]");
+        CHECK(json_document::parse_copy(std::move(rvalue_source)).owns_source());
+
+        // a stream without a buffer is an empty input
+        std::istream no_buffer(nullptr);
+        CHECK(json_document::parse(no_buffer, false).is_discarded());
+
+#if !defined(JSON_NOEXCEPTION)
+        // an exception while reading the input leaves a discarded document
+        json_document d = json_document::parse(std::vector<char>(text.begin(), text.end()));
+        std::istringstream throwing("[1]");
+        throwing.exceptions(std::ios::eofbit);
+        CHECK_THROWS_AS(d.read(throwing), std::ios_base::failure&);
+        CHECK(d.is_discarded());
+        CHECK(d.root().is_discarded());
+        d.read("[2]");
+        CHECK(d.root().materialize() == json::parse("[2]"));
+#endif
+    }
+
+    SECTION("read() reuses the owned copy")
+    {
+        std::string big = "[";
+        for (int i = 0; i < 1000; ++i)
+        {
+            big += (i != 0 ? ",\"" : "\"") + std::to_string(i) + "\"";
+        }
+        big += ']';
+        const std::string small = "{\"a\": [1, 2]}";
+
+        // copied inputs are written into the existing buffer
+        json_document d = json_document::parse(std::vector<char>(big.begin(), big.end()));
+        REQUIRE(d.owns_source());
+        const char* const buffer = d.source().data();
+        d.read(std::vector<char>(small.begin(), small.end()));
+        CHECK(d.owns_source());
+        CHECK(d.source().data() == buffer);
+        CHECK(d.root().materialize() == json::parse(small));
+        std::istringstream stream(small);
+        d.read(stream);
+        CHECK(d.source().data() == buffer);
+        CHECK(d.root().materialize() == json::parse(small));
+        d.read(std::wstring(L"[3]"));
+        CHECK(d.source().data() == buffer);
+        CHECK(d.root().materialize() == json::parse("[3]"));
+
+        // a moved-in string replaces it (long enough to be on the heap)
+        std::string moved = big;
+        const char* const moved_buffer = moved.data();
+        d.read(std::move(moved));
+        CHECK(d.source().data() == moved_buffer);
+        CHECK(d.root().materialize() == json::parse(big));
+
+        // a borrowed input leaves nothing owned
+        d.read(big);
+        CHECK(!d.owns_source());
+        CHECK(d.source().data() == big.data());
     }
 
     SECTION("integer arguments do not compile")

tests/src/unit-json_view_builder.cpp

@@ -113,9 +113,9 @@ json value_of(const document_data& d, const node*& n)
 
 json value_of(const built& b)
 {
-    const node* n = b.data->tape;
+    const node* n = b.data->nodes;
     json v = value_of(*b.data, n);
-    CHECK(n == b.data->tape + b.data->tape_size);
+    CHECK(n == b.data->nodes + b.data->node_count);
     return v;
 }
 
@@ -340,6 +340,17 @@ TEST_CASE("json_view builder")
 
     SECTION("deep nesting")
     {
+        // the deepest nesting is recorded (materialize() sizes its stack from it)
+        CHECK(build("1", false, false, false).data->max_depth == 0);
+        CHECK(build("[]", false, false, false).data->max_depth == 1);
+        CHECK(build(R"([[1],{"a":[[]]},[]])", false, false, false).data->max_depth == 4);
+        // a stack that turns out too small still grows
+        const std::string nested = R"([[1],{"a":[[{"b":[]}]]},[]])";
+        const built shallow = build(nested, false, false, false);
+        REQUIRE(shallow.ok);
+        shallow.data->max_depth = 1;
+        CHECK(nlohmann::detail::view::materialize<json>(*shallow.data, shallow.data->nodes) == json::parse(nested));
+
         // the open containers beyond 64 levels live on the heap
         for (const std::size_t depth :
                 {
@@ -349,8 +360,9 @@ TEST_CASE("json_view builder")
             const std::string arrays = std::string(depth, '[') + std::string(depth, ']');
             const built b = build(arrays, false, false, false);
             REQUIRE(b.ok);
-            CHECK(b.data->tape_size == depth);
-            CHECK(b.data->tape[0].next == depth);
+            CHECK(b.data->node_count == depth);
+            CHECK(b.data->nodes[0].next == depth);
+            CHECK(b.data->max_depth == depth);
             std::string objects;
             for (std::size_t i = 0; i < depth; ++i)
             {
@@ -359,7 +371,8 @@ TEST_CASE("json_view builder")
             objects += '1' + std::string(depth, '}');
             const built o = build(objects, false, false, false);
             REQUIRE(o.ok);
-            CHECK(o.data->tape_size == (2 * depth) + 1);
+            CHECK(o.data->node_count == (2 * depth) + 1);
+            CHECK(o.data->max_depth == depth);
             CHECK(!build(std::string(depth, '[') + std::string(depth - 1, ']'), false, false, false).ok);
         }
     }
@@ -454,7 +467,7 @@ TEST_CASE("json_view node integer bits")
             CAPTURE(c.text)
             const built b = build(c.text, false, false, true);
             REQUIRE(b.ok);
-            const node& n = b.data->tape[1];
+            const node& n = b.data->nodes[1];
             CHECK(integer_bits(n) == c.bits);
             CHECK(n.len == static_cast<std::uint32_t>(c.bits));
             CHECK(n.next == static_cast<std::uint32_t>(c.bits >> 32));
@@ -469,23 +482,23 @@ TEST_CASE("json_view node array size limit")
     // for absurd counts everywhere)
     std::unique_ptr<document_data, document_data::deleter> d(document_data::create(0));
     d->reserve(8);
-    REQUIRE(d->tape_cap >= 8);
-    d->tape_size = 2;
-    const std::size_t cap = d->tape_cap;
-    node* const tape = d->tape;
+    REQUIRE(d->node_capacity >= 8);
+    d->node_count = 2;
+    const std::size_t cap = d->node_capacity;
+    node* const nodes = d->nodes;
 
 #if !defined(JSON_NOEXCEPTION)
     const std::size_t too_many = document_data::max_nodes() + 1;
     CHECK_THROWS_AS(d->reserve(too_many), std::bad_alloc&);
     CHECK_THROWS_AS(d->reserve((std::numeric_limits<std::size_t>::max)()), std::bad_alloc&);
     // the array is unchanged
-    CHECK(d->tape == tape);
-    CHECK(d->tape_cap == cap);
-    CHECK(d->tape_size == 2);
+    CHECK(d->nodes == nodes);
+    CHECK(d->node_capacity == cap);
+    CHECK(d->node_count == 2);
 #endif
 
     // the largest count that fits is not refused by the check (nothing is
     // allocated for a count that is already there)
     d->reserve(cap);
-    CHECK(d->tape == tape);
+    CHECK(d->nodes == nodes);
 }