json_view: check_utf8 skips well-formed runs with the parser's string scan

Strings of 32 bytes or more that are written into an editable document
are validated with scan_string_run(), which skips well-formed UTF-8 16
bytes at a time; the byte-wise check runs only from where the scan found
ill-formed UTF-8, to report the same type_error.316. Shorter strings
keep the byte-wise check (inlined with a constant start, so that this
path is the code it was), where the vector scan costs more than it
saves.

set()/push_back() of strings of 32 bytes or more take 0.29-0.76 of the
time (x86-64 GCC, averaged over code layouts) and 0.25-0.80 (AArch64
clang); shorter strings take 1.5-4 % fewer instructions; the json_view
rows of bench_edit execute at most 0.04 % more instructions.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/view/edit.hpp

@@ -26,6 +26,7 @@
 #include <nlohmann/detail/view/lookup.hpp>
 #include <nlohmann/detail/view/macro_scope.hpp>
 #include <nlohmann/detail/view/node.hpp>
+#include <nlohmann/detail/view/scan.hpp>
 #include <nlohmann/detail/view/view_mode.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
@@ -46,10 +47,9 @@ struct first_true<false, Conditions...> : std::integral_constant < int, 1 + firs
 /// (type_error.316 with the same message), so that an editable document
 /// only holds valid UTF-8: the error is at the first byte that no
 /// well-formed sequence can continue with (Unicode, Table 3-7).
-inline void check_utf8(const char* s, std::size_t n)
+NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i)
 {
-    const auto* const p = reinterpret_cast<const unsigned char*>(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+    for (; i < n;)
-    for (std::size_t i = 0; i < n;)
     {
         const unsigned char c = p[i];
         if (c < 0x80)
@@ -79,6 +79,43 @@ inline void check_utf8(const char* s, std::size_t n)
     }
 }
 
+/// The long-string path of check_utf8(): skips well-formed runs with the
+/// parser's string scan (which stops at a quote, a backslash, or a control
+/// character, all legal here, and at ill-formed UTF-8); the byte-wise check
+/// runs only from where the scan found ill-formed UTF-8, to report it.
+NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n)
+{
+    const unsigned char* const e = p + n;
+    const unsigned char* q = p;
+    for (;;)
+    {
+        q = scan_string_run<true>(q, e);
+        if (q == e)
+        {
+            return;
+        }
+        if (*q >= 0x80)
+        {
+            check_utf8_bytewise(p, n, static_cast<std::size_t>(q - p));
+            return; // not reached: the scan stopped at ill-formed UTF-8
+        }
+        ++q; // a quote, backslash, or control character
+    }
+}
+
+inline void check_utf8(const char* s, std::size_t n)
+{
+    const auto* const p = reinterpret_cast<const unsigned char*>(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+    if (NLOHMANN_VIEW_UNLIKELY(n >= 32))
+    {
+        check_utf8_scan(p, n);
+        return;
+    }
+    // short strings: the vector scan costs more than it saves; (the loop is
+    // inlined with a constant start, so this path is the plain byte-wise check)
+    check_utf8_bytewise(p, n, 0);
+}
+
 /*!
 @brief the edits of an editable basic_json_document
 

single_include/nlohmann/json_view.hpp

@@ -3727,6 +3727,8 @@ NLOHMANN_JSON_NAMESPACE_END
 
 // #include <nlohmann/detail/view/node.hpp>
 
+// #include <nlohmann/detail/view/scan.hpp>
+
 // #include <nlohmann/detail/view/view_mode.hpp>
 //     __ _____ _____ _____
 //  __|  |   __|     |   | |  JSON for Modern C++
@@ -3800,10 +3802,9 @@ struct first_true<false, Conditions...> : std::integral_constant < int, 1 + firs
 /// (type_error.316 with the same message), so that an editable document
 /// only holds valid UTF-8: the error is at the first byte that no
 /// well-formed sequence can continue with (Unicode, Table 3-7).
-inline void check_utf8(const char* s, std::size_t n)
+NLOHMANN_VIEW_ALWAYS_INLINE void check_utf8_bytewise(const unsigned char* p, std::size_t n, std::size_t i)
 {
-    const auto* const p = reinterpret_cast<const unsigned char*>(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+    for (; i < n;)
-    for (std::size_t i = 0; i < n;)
     {
         const unsigned char c = p[i];
         if (c < 0x80)
@@ -3833,6 +3834,43 @@ inline void check_utf8(const char* s, std::size_t n)
     }
 }
 
+/// The long-string path of check_utf8(): skips well-formed runs with the
+/// parser's string scan (which stops at a quote, a backslash, or a control
+/// character, all legal here, and at ill-formed UTF-8); the byte-wise check
+/// runs only from where the scan found ill-formed UTF-8, to report it.
+NLOHMANN_VIEW_NOINLINE inline void check_utf8_scan(const unsigned char* p, std::size_t n)
+{
+    const unsigned char* const e = p + n;
+    const unsigned char* q = p;
+    for (;;)
+    {
+        q = scan_string_run<true>(q, e);
+        if (q == e)
+        {
+            return;
+        }
+        if (*q >= 0x80)
+        {
+            check_utf8_bytewise(p, n, static_cast<std::size_t>(q - p));
+            return; // not reached: the scan stopped at ill-formed UTF-8
+        }
+        ++q; // a quote, backslash, or control character
+    }
+}
+
+inline void check_utf8(const char* s, std::size_t n)
+{
+    const auto* const p = reinterpret_cast<const unsigned char*>(s); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
+    if (NLOHMANN_VIEW_UNLIKELY(n >= 32))
+    {
+        check_utf8_scan(p, n);
+        return;
+    }
+    // short strings: the vector scan costs more than it saves; (the loop is
+    // inlined with a constant start, so this path is the plain byte-wise check)
+    check_utf8_bytewise(p, n, 0);
+}
+
 /*!
 @brief the edits of an editable basic_json_document
 

tests/benchmarks/json_view/README.md

@@ -21,10 +21,6 @@ python3 tests/benchmarks/json_view/compare.py --data <json_test_data directory>
 - The other libraries come from the system: pkg-config, or Homebrew (`brew install yyjson simdjson boost`). With
   `--download`, pinned releases are downloaded instead and checked against their SHA-256. Without Boost headers (or
   with `--no-boost`), the Boost.JSON columns are skipped, and the results say so.
-- `--aligned` additionally builds and runs everything with all functions aligned to 64 bytes (`-mllvm
-  -align-all-functions=6` for Clang, `-falign-functions=64 -falign-loops=64` for GCC), and reports both result
-  tables as "default build" and "aligned build". Code layout alone can move a result by a few percent; a difference
-  that appears in only one of the two tables is a layout artifact. Off by default, which doubles the run time.
 - `--corpus file...` adds files to the corpus benchmark, e.g. those of
   [simdjson-data](https://github.com/simdjson/simdjson-data) or the
   [yyjson benchmark](https://github.com/ibireme/yyjson_benchmark).
@@ -43,7 +39,7 @@ where `json_view` stands on another architecture; they are not meant for publica
 
 ## What is measured
 
-`bench_view.cpp` runs these workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a
+`bench_view.cpp` runs four workloads on twitter, citm_catalog, canada, jeopardy, a single tweet (`status`), and a
 JSON-RPC request (`rpc`):
 
 | workload | what it does |
@@ -52,7 +48,6 @@ JSON-RPC request (`rpc`):
 | traverse | parse, then visit every value, convert every number, touch every string and key |
 | select | parse, then read a few fields per record (e.g. id, user name, and retweet count of each tweet) |
 | dump | serialize a parsed document (compact) |
-| parse+dump | build a fresh document, then serialize it (compact), in one timed call |
 
 `bench_corpus.cpp` runs parse, traverse, and dump on any list of files, so that no library is tuned to a handful of
 documents. Its dump also writes the numbers as they are in the input: `json_view` with `number_format::source`, and
@@ -77,32 +72,11 @@ call follows an untimed call of the same engine: otherwise the engine after `jso
 cleaning up the tens of thousands of nodes `json::parse` just freed (with glibc, this made `json_view` look 1.7 times
 slower on citm_catalog traverse).
 
-`json_view` is an index into the source text, not a tree of owning nodes. Its like-for-like peers are yyjson and
-simdjson DOM (immutable, indexed, no per-node allocation). Boost.JSON and `json::parse` build mutable, owning DOMs and
-are reference points for what that costs, not equals; simdjson On-Demand is lazy and forward-only, so it is the other
-way round: it has no random access to pay for. All engines validate UTF-8 and syntax by default.
-
-Number conversion differs: `json_view` converts integers while parsing but converts floating-point numbers lazily,
-when they are read or dumped; yyjson (default flags) and simdjson DOM convert every number while parsing. The cost of
-float conversion therefore appears in the traverse and dump rows for `json_view`, and in the parse row for the others.
-For float-heavy documents (canada), the fair reading is a sum of parse and use: the `traverse` workload already
-includes the parse, and `parse+dump` measures a fresh document and its dump in one timed call; the `dump` row alone
-leaves the conversion in `json_view` and out of the others. Eager conversion at parse time was prototyped as an opt-in
-and measured: canada traverse (with its parse) was 4-7 % faster, but parse+dump 11-30 % slower and parse 36-90 % slower. It was
-dropped, so lazy conversion is the only mode.
-
-`json_view` can also be edited: `json_editable_document` has the same read API, and the `(editable)` rows in
-`bench_view` measure it on the same workloads. They are the fair counterpart of Boost.JSON and `json::parse`, which are
-mutable too; the plain `json_view` rows are the counterpart of yyjson and simdjson DOM, which are not. `bench_edit` adds
-the read-modify-write workloads. Read the tables with that in mind: an immutable-only library is not compared with an
-editable `json_view` unless it says so.
-
 The engines do not all offer the same features, which the numbers should be read with:
 
 | engine | document | random access | editable | notes |
 |---|---|---|---|---|
 | `json_view` | immutable index into the text | yes | no | a fresh document per parse; "reused" parses into the same document |
-| `json_view` (editable) | index into the text plus edits | yes | yes, in place | `json_editable_document`; keeps member order and number spellings |
 | yyjson | immutable (`yyjson_read`) | yes | via a mutable copy | |
 | simdjson DOM | immutable, parser reused | yes | no | "fresh" uses a new parser per parse |
 | simdjson On-Demand | none: forward-only, lazy | no | no | only traverse and select |

tests/benchmarks/json_view/bench_view.cpp

@@ -8,9 +8,8 @@
 
 // Same-feature-set benchmark: read-only JSON documents with random access.
 //
-//   json_view      nlohmann/json_view.hpp (fresh document per parse / reused;
+//   json_view      nlohmann/json_view.hpp (fresh document per parse / reused)
-//                  "(editable)": json_editable_document, same read API)
+//   yyjson         yyjson_read(): immutable document, random access
-//   yyjson        yyjson_read(): immutable document, random access
 //   simdjson DOM   dom::parser (reused, as recommended; "fresh": a new parser
 //                  per parse): immutable, random access
 // references (different feature sets):
@@ -20,11 +19,7 @@
 //
 // Workloads: parse (build + free), traverse (visit everything, convert every
 // number, touch every string and key), select (a few fields per document),
-// dump (compact serialization of the parsed document), and parse+dump (a fresh
+// dump (compact serialization of the parsed document).
-// document, then its dump, in one timed call). json_view converts floating-point
-// numbers lazily (when read or dumped), yyjson and simdjson DOM while parsing:
-// for float-heavy documents, traverse (which includes the parse) and parse+dump
-// are the fair reading of the parse and dump rows.
 // All engines run interleaved in every round, each timed call after an untimed
 // one of the same engine; the best round is reported.
 #include <nlohmann/json_view.hpp>
@@ -48,8 +43,6 @@
 
 using nlohmann::json;
 using nlohmann::json_document;
-using nlohmann::json_editable_document;
-using nlohmann::json_editable_view;
 using nlohmann::json_view;
 
 static volatile double g_sink;
@@ -62,8 +55,7 @@ struct stats
 
 // ---------------- traversal ----------------
 
-template<class V>
+static void walk(json_view v, stats& st)
-static void walk_view(V v, stats& st)
 {
     ++st.nodes;
     switch (v.type())
@@ -72,42 +64,32 @@ static void walk_view(V v, stats& st)
             for (auto it = v.begin(); it != v.end(); ++it)
             {
                 st.str += it.key().size();
-                walk_view<V>(*it, st);
+                walk(*it, st);
             }
             break;
         case json::value_t::array:
-            for (const V e : v)
+            for (const json_view e : v)
             {
-                walk_view<V>(e, st);
+                walk(e, st);
             }
             break;
         case json::value_t::string:
             st.str += v.get_string().size();
             break;
         case json::value_t::number_integer:
-            st.num += static_cast<double>(v.template get<std::int64_t>());
+            st.num += static_cast<double>(v.get<std::int64_t>());
             break;
         case json::value_t::number_unsigned:
-            st.num += static_cast<double>(v.template get<std::uint64_t>());
+            st.num += static_cast<double>(v.get<std::uint64_t>());
             break;
         case json::value_t::number_float:
-            st.num += v.template get<double>();
+            st.num += v.get<double>();
             break;
         default:
             break;
     }
 }
 
-static void walk(json_view v, stats& st)
-{
-    walk_view<json_view>(v, st);
-}
-
-static void walk(json_editable_view v, stats& st)
-{
-    walk_view<json_editable_view>(v, st);
-}
-
 static void walk(const json& j, stats& st)
 {
     ++st.nodes;
@@ -318,21 +300,20 @@ static void walk(const boost::json::value& v, stats& st)
 // status (one tweet): id, user.screen_name, retweet_count
 // rpc: method, params.minuend, id
 
-template<class V>
+static double pick(const std::string& name, json_view r)
-static double pick_view(const std::string& name, V r)
 {
     double acc = 0;
     if (name == "twitter" || name == "status")
     {
-        auto one = [&](V s)
+        auto one = [&](json_view s)
         {
-            acc += static_cast<double>(s["id"].template get<std::uint64_t>());
+            acc += static_cast<double>(s["id"].get<std::uint64_t>());
             acc += static_cast<double>(s["user"]["screen_name"].get_string().size());
-            acc += static_cast<double>(s["retweet_count"].template get<std::int64_t>());
+            acc += static_cast<double>(s["retweet_count"].get<std::int64_t>());
         };
         if (name == "twitter")
         {
-            for (const V s : r["statuses"])
+            for (const json_view s : r["statuses"])
             {
                 one(s);
             }
@@ -344,20 +325,20 @@ static double pick_view(const std::string& name, V r)
     }
     else if (name == "citm_catalog")
     {
-        for (const V p : r["performances"])
+        for (const json_view p : r["performances"])
         {
-            acc += static_cast<double>(p["id"].template get<std::uint64_t>() + p["eventId"].template get<std::uint64_t>() + p["seatCategories"].size());
+            acc += static_cast<double>(p["id"].get<std::uint64_t>() + p["eventId"].get<std::uint64_t>() + p["seatCategories"].size());
         }
         acc += static_cast<double>(r["events"].size());
     }
     else if (name == "canada")
     {
-        const V g = r["features"][0]["geometry"];
+        const json_view g = r["features"][0]["geometry"];
         acc += static_cast<double>(r["type"].get_string().size() + g["type"].get_string().size() + g["coordinates"].size());
     }
     else if (name == "jeopardy")
     {
-        for (const V q : r)
+        for (const json_view q : r)
         {
             acc += q["round"].get_string() == "Final Jeopardy!" ? 1 : 0;
             acc += static_cast<double>(q["category"].get_string().size());
@@ -366,21 +347,11 @@ static double pick_view(const std::string& name, V r)
     else if (name == "rpc")
     {
         acc += static_cast<double>(r["method"].get_string().size());
-        acc += static_cast<double>(r["params"]["minuend"].template get<std::int64_t>() + r["id"].template get<std::int64_t>());
+        acc += static_cast<double>(r["params"]["minuend"].get<std::int64_t>() + r["id"].get<std::int64_t>());
     }
     return acc;
 }
 
-static double pick(const std::string& name, json_view r)
-{
-    return pick_view<json_view>(name, r);
-}
-
-static double pick(const std::string& name, json_editable_view r)
-{
-    return pick_view<json_editable_view>(name, r);
-}
-
 static double pick(const std::string& name, const json& r)
 {
     double acc = 0;
@@ -663,35 +634,15 @@ int main(int argc, char** argv)
         walk(json::parse(dc.text), a);
         auto d = json_document::parse(dc.text);
         walk(d.root(), b);
-        auto ed = json_editable_document::parse(dc.text);
-        stats e;
-        walk(ed.root(), e);
         yyjson_doc* y = yyjson_read(dc.text.data(), dc.text.size(), 0);
         walk(yyjson_doc_get_root(y), c);
         simdjson::dom::parser p;
         walk(p.parse(dc.text).value(), dd);
-        const bool ok = a.nodes == b.nodes && a.nodes == e.nodes && a.str == e.str && std::fabs(a.num - e.num) <= 1e-9 * std::fabs(a.num) && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str
+        const bool ok = a.nodes == b.nodes && a.nodes == c.nodes && a.nodes == dd.nodes && a.str == b.str && a.str == c.str && a.str == dd.str
                         && std::fabs(a.num - b.num) <= 1e-9 * std::fabs(a.num) && std::fabs(a.num - c.num) <= 1e-9 * std::fabs(a.num);
-        const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pe = pick(dc.name, ed.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value());
+        const double pa = pick(dc.name, json::parse(dc.text)), pb = pick(dc.name, d.root()), pc = pick(dc.name, yyjson_doc_get_root(y)), pd = pick(dc.name, p.parse(dc.text).value());
-        std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pe && pa == pc && pa == pd) ? "OK" : "MISMATCH");
+        std::printf("check %-13s traverse %s select %s\n", dc.name.c_str(), ok ? "OK" : "MISMATCH", (pa == pb && pa == pc && pa == pd) ? "OK" : "MISMATCH");
         yyjson_doc_free(y);
-
-        // the dumps of the engines may format floats differently, so only: every dump is non-empty, and the
-        // dump of json_view (also editable) and of json::parse parse back to the same number of nodes
-        const std::string dj = json::parse(dc.text).dump();
-        const std::string dv = d.root().dump();
-        const std::string de = ed.root().dump();
-        std::size_t yn = 0;
-        yyjson_doc* y2 = yyjson_read(dc.text.data(), dc.text.size(), 0);
-        char* yo = yyjson_write(y2, 0, &yn);
-        const bool dumps_ok = !dj.empty() && !dv.empty() && !de.empty() && yo != nullptr && yn > 0 && !simdjson::to_string(p.parse(dc.text).value()).empty();
-        std::free(yo);
-        yyjson_doc_free(y2);
-        stats sj_nodes, sv_nodes, se_nodes;
-        walk(json::parse(dj), sj_nodes);
-        walk(json::parse(dv), sv_nodes);
-        walk(json::parse(de), se_nodes);
-        std::printf("check %-13s dump %s\n", dc.name.c_str(), (dumps_ok && sj_nodes.nodes == a.nodes && sv_nodes.nodes == a.nodes && se_nodes.nodes == a.nodes) ? "OK" : "MISMATCH");
     }
 
     std::FILE* csv = std::fopen("bench_view.csv", "w");
@@ -718,7 +669,6 @@ int main(int argc, char** argv)
         workloads.push_back({"parse", {
                 {"json_view", [&] { auto d = json_document::parse(s); g_sink = static_cast<double>(d.node_count()); }},
                 {"json_view (reused)", [&] { reused.read(s); g_sink = static_cast<double>(reused.node_count()); }},
-                {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = static_cast<double>(d.root().size()); }},
                 {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = static_cast<double>(yyjson_doc_get_val_count(d)); yyjson_doc_free(d); }},
                 {"simdjson DOM", [&] { auto e = sj.parse(ps).value_unsafe(); g_sink = e.is_object(); }},
                 {"simdjson DOM (fresh)", [&] { simdjson::dom::parser p; auto e = p.parse(ps).value_unsafe(); g_sink = e.is_object(); }},
@@ -730,7 +680,6 @@ int main(int argc, char** argv)
         workloads.push_back({"traverse", {
                 {"json_view", [&] { auto d = json_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }},
                 {"json_view (reused)", [&] { reused.read(s); stats st; walk(reused.root(), st); g_sink = st.num; }},
-                {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); stats st; walk(d.root(), st); g_sink = st.num; }},
                 {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); stats st; walk(yyjson_doc_get_root(d), st); g_sink = st.num; yyjson_doc_free(d); }},
                 {"simdjson DOM", [&] { stats st; walk(sj.parse(ps).value_unsafe(), st); g_sink = st.num; }},
                 {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); stats st; walk_od(d.get_value().value_unsafe(), st); g_sink = st.num; }},
@@ -742,31 +691,15 @@ int main(int argc, char** argv)
         workloads.push_back({"select", {
                 {"json_view", [&] { auto d = json_document::parse(s); g_sink = pick(name, d.root()); }},
                 {"json_view (reused)", [&] { reused.read(s); g_sink = pick(name, reused.root()); }},
-                {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); g_sink = pick(name, d.root()); }},
                 {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); g_sink = pick(name, yyjson_doc_get_root(d)); yyjson_doc_free(d); }},
                 {"simdjson DOM", [&] { g_sink = pick(name, sj.parse(ps).value_unsafe()); }},
                 {"simdjson OD", [&] { auto d = od.iterate(ps).value_unsafe(); g_sink = pick_od(name, d); }},
                 {"json::parse", [&] { json j = json::parse(s); g_sink = pick(name, j); }},
             }});
-        // pipeline: a fresh document, then a dump of it, in one timed call
-        // (simdjson DOM reuses its parser, as in the parse row); the traverse
-        // workload above is already parse+traverse
-        workloads.push_back({"parse+dump", {
-                {"json_view", [&] { auto d = json_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast<double>(o.size()); }},
-                {"json_view (editable)", [&] { auto d = json_editable_document::parse(s); std::string o = d.root().dump(); g_sink = static_cast<double>(o.size()); }},
-                {"yyjson", [&] { yyjson_doc* d = yyjson_read(s.data(), s.size(), 0); std::size_t n = 0; char* o = yyjson_write(d, 0, &n); g_sink = static_cast<double>(n); std::free(o); yyjson_doc_free(d); }},
-                {"simdjson DOM", [&] { std::string o = simdjson::to_string(sj.parse(ps).value_unsafe()); g_sink = static_cast<double>(o.size()); }},
-#if JSON_VIEW_BENCH_BOOST
-                {"Boost.JSON", [&] { boost::json::monotonic_resource mr; auto v = boost::json::parse(s, &mr); std::string o = boost::json::serialize(v); g_sink = static_cast<double>(o.size()); }},
-#endif
-                {"json::parse", [&] { json j = json::parse(s); std::string o = j.dump(); g_sink = static_cast<double>(o.size()); }},
-            }});
         {
             // serialization of an already parsed document
             static json_document vd;
             vd.read(s);
-            static json_editable_document ed = json_editable_document::parse(s);
-            ed = json_editable_document::parse(s);
             static yyjson_doc* yd = nullptr;
             if (yd)
             {
@@ -780,7 +713,6 @@ int main(int argc, char** argv)
             jd = json::parse(s);
             workloads.push_back({"dump", {
                     {"json_view", [&] { std::string o = vd.root().dump(); g_sink = static_cast<double>(o.size()); }},
-                    {"json_view (editable)", [&] { std::string o = ed.root().dump(); g_sink = static_cast<double>(o.size()); }},
                     {"yyjson", [&] { std::size_t n = 0; char* o = yyjson_write(yd, 0, &n); g_sink = static_cast<double>(n); std::free(o); }},
                     {"simdjson DOM", [&] { std::string o = simdjson::to_string(se); g_sink = static_cast<double>(o.size()); }},
                     {"json::parse", [&] { std::string o = jd.dump(); g_sink = static_cast<double>(o.size()); }},
@@ -808,7 +740,7 @@ int main(int argc, char** argv)
                 }
             }
             const double ref = best[0];
-            std::printf("%-13s %-14s", dc.name.c_str(), wl.first.c_str());
+            std::printf("%-13s %-9s", dc.name.c_str(), wl.first.c_str());
             for (std::size_t k = 0; k < wl.second.size(); ++k)
             {
                 const double us = best[k] / 1e3;

tests/benchmarks/json_view/compare.py

@@ -239,8 +239,6 @@ def main():
     ap.add_argument('--download', action='store_true', help='use pinned downloads instead of system libraries')
     ap.add_argument('--no-boost', action='store_true', help='skip Boost.JSON')
     ap.add_argument('--native', action='store_true', help='compile for this CPU (-march=native / -mcpu=native)')
-    ap.add_argument('--aligned', action='store_true',
-                    help='also build and run with all functions (and loops) aligned, to make code layout artifacts visible')
     ap.add_argument('--rounds', type=int, default=30, help='rounds of bench_view (default: 30)')
     ap.add_argument('--corpus', nargs='*', default=[], help='more files for bench_corpus')
     ap.add_argument('--build-dir', default=os.path.join(HERE, 'build'), help='where to build (default: build/ next to this script)')
@@ -284,46 +282,32 @@ def main():
                 + ['-I' + d for d in lib.include])
             objects.append(obj)
 
-    cxx_version = output(cxx + ['--version'])
+    binaries = {}
-    is_clang = 'clang' in cxx_version.lower()
+    for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']:
-    # alignment flags: functions and (gcc) loops on 64-byte boundaries
+        exe = os.path.join(args.build_dir, bench)
-    align_flags = ['-mllvm', '-align-all-functions=6'] if is_clang else ['-falign-functions=64', '-falign-loops=64']
+        run(cxx + flags + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe])
-
+        binaries[bench] = exe
-    # the builds: the default one and, with --aligned, one with aligned code
-    variants = [('default build', args.build_dir, [])]
-    if args.aligned:
-        variants.append(('aligned build', os.path.join(args.build_dir, 'aligned'), align_flags))
 
+    # run: bench_view on its documents, bench_corpus on those and the given files
     corpus = [os.path.join(args.data, f) for f in DEFAULT_CORPUS] + args.corpus
-    # label -> {benchmark: output}
+    outputs = {}
-    all_outputs = {}
+    outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=args.build_dir,
-    for label, build_dir, extra in variants:
+                                capture_output=True, text=True).stdout
-        os.makedirs(build_dir, exist_ok=True)
+    outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=args.build_dir,
-        binaries = {}
+                                  capture_output=True, text=True).stdout
-        for bench in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']:
+    outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir,
-            exe = os.path.join(build_dir, bench)
+                                capture_output=True, text=True).stdout
-            run(cxx + flags + extra + include + [os.path.join(HERE, bench + '.cpp')] + objects + link + ['-o', exe])
+    outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))], cwd=args.build_dir,
-            binaries[bench] = exe
+                                  capture_output=True, text=True).stdout
-
+    for name, text in outputs.items():
-        # run: bench_view on its documents, bench_corpus on those and the given files
+        print(text)
-        outputs = {}
-        outputs['bench_view'] = run([binaries['bench_view'], args.data, str(args.rounds)], cwd=build_dir,
-                                    capture_output=True, text=True).stdout
-        outputs['bench_corpus'] = run([binaries['bench_corpus']] + corpus, cwd=build_dir,
-                                      capture_output=True, text=True).stdout
-        outputs['bench_edit'] = run([binaries['bench_edit'], args.data, str(max(1, args.rounds // 2))], cwd=build_dir,
-                                    capture_output=True, text=True).stdout
-        outputs['bench_decode'] = run([binaries['bench_decode'], args.data, str(max(1, args.rounds // 2))],
-                                      cwd=build_dir, capture_output=True, text=True).stdout
-        for name, text in outputs.items():
-            print(text)
-        all_outputs[label] = outputs
 
     # results with their metadata
     now = datetime.datetime.now()
     host = re.sub(r'[^A-Za-z0-9-]+', '-', platform.node().split('.')[0]) or 'host'
     stem = os.path.join(HERE, 'results', f'{now:%Y-%m-%d}-{host}')
     os.makedirs(os.path.dirname(stem), exist_ok=True)
+    cxx_version = output(cxx + ['--version'])
     compiler_version = cxx_version.splitlines()[0] if cxx_version else ' '.join(cxx)
     meta = [
         ('date', f'{now:%Y-%m-%d %H:%M}'),
@@ -332,7 +316,6 @@ def main():
         ('OS', f'{platform.system()} {platform.release()} ({platform.machine()})'),
         ('compiler', compiler_version),
         ('flags', ' '.join(flags)),
-        ('aligned results', ('included (flags: ' + ' '.join(align_flags) + ')') if args.aligned else 'not included'),
         ('yyjson', libs['yyjson'].version),
         ('simdjson', libs['simdjson'].version),
         ('Boost.JSON', libs['boost'].version if with_boost else 'skipped (not found)'),
@@ -345,20 +328,15 @@ def main():
         f.write('| | |\n|---|---|\n')
         for key, value in meta:
             f.write(f'| {key} | {value} |\n')
-        for label, outputs in all_outputs.items():
+        for name, text in outputs.items():
-            # (without --aligned there is only the default build, and no extra heading level)
+            f.write(f'\n## {name}\n\n```\n{text.rstrip()}\n```\n')
-            prefix = f'{label}: ' if args.aligned else ''
-            for name, text in outputs.items():
-                f.write(f'\n## {prefix}{name}\n\n```\n{text.rstrip()}\n```\n')
     with open(stem + '.csv', 'w', encoding='utf-8') as out:
         out.write(''.join(f'# {key}: {value}\n' for key, value in meta))
-        for label, build_dir, _ in variants:
+        for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']:
-            for name in ['bench_view', 'bench_corpus', 'bench_edit', 'bench_decode']:
+            path = os.path.join(args.build_dir, name + '.csv')
-                path = os.path.join(build_dir, name + '.csv')
+            if os.path.isfile(path):
-                if os.path.isfile(path):
+                with open(path, encoding='utf-8') as f:
-                    with open(path, encoding='utf-8') as f:
+                    out.write(f'# {name}\n' + f.read())
-                        # (the default build keeps its plain section names)
-                        out.write(f'# {name}' + (f' ({label})' if args.aligned else '') + '\n' + f.read())
     print(f'results: {stem}.md, {stem}.csv')
 
 

tests/src/unit-json_view_edit.cpp

@@ -941,3 +941,123 @@ TEST_CASE("json_view edits: duplicate keys")
         CHECK(d.root()["o"]["d"].get<int>() == 0);
     }
 }
+
+#if !defined(JSON_NOEXCEPTION)
+TEST_CASE("json_view edits: check_utf8 agrees with the byte-wise check")
+{
+    namespace dv = nlohmann::detail::view;
+
+    // "ok" or the exception message
+    const auto run = [](bool fast, const std::string & s) -> std::string
+    {
+        try
+        {
+            if (fast)
+            {
+                dv::check_utf8(s.data(), s.size());
+            }
+            else
+            {
+                dv::check_utf8_bytewise(reinterpret_cast<const unsigned char*>(s.data()), s.size(), 0);
+            }
+        }
+        catch (const json::type_error& e)
+        {
+            return {e.what()};
+        }
+        return {"ok"};
+    };
+    std::size_t errors = 0;
+    const auto same = [&](const std::string & s)
+    {
+        const std::string a = run(true, s);
+        const std::string b = run(false, s);
+        if (a != b)
+        {
+            INFO("fast: " << a << ", byte-wise: " << b << ", length " << s.size());
+            CHECK(a == b);
+        }
+        errors += b != "ok" ? 1 : 0;
+    };
+
+    const std::vector<std::string> pieces =
+    {
+        "a", "Z", " ", "\"", "\\", "\n", "\x01", "\x7f", "\xc3\xa4", "\xe4\xb8\xad", "\xf0\x9f\x98\x80",
+        "\x80", "\xbf", "\xc0\x80", "\xc1\xbf", "\xc2", "\xe0\x80\x80", "\xe0\x9f\xbf", "\xed\xa0\x80", "\xed\xbf\xbf",
+        "\xf0\x80\x80\x80", "\xf0\x8f\xbf\xbf", "\xf4\x90\x80\x80", "\xf4\x8f\xbf\xbf", "\xf5", "\xf8", "\xff", "\xfe",
+        "\xe4\xb8", "\xf0\x9f\x98", "\xf0\x9f", "\xf0"
+    };
+
+    // every piece behind prefixes of every length, and followed by more bytes
+    for (std::size_t prefix = 0; prefix <= 64; ++prefix)
+    {
+        for (const auto& piece : pieces)
+        {
+            same(std::string(prefix, 'x') + piece);
+            same(std::string(prefix, 'x') + piece + "yz");
+            same(std::string(prefix, '"') + piece);
+            same(std::string(prefix, '\xc3') + piece);
+            same(std::string(prefix / 2, '\xc3').append(prefix / 2, '\xa4') + piece);
+            same(std::string(prefix, 'x') + "\xe4\xb8\xad" + piece);
+        }
+    }
+    // an invalid byte at every position of valid strings of every length
+    for (std::size_t len = 0; len <= 64; ++len)
+    {
+        const std::string ascii(len, 'a');
+        std::string mixed;
+        while (mixed.size() < len)
+        {
+            if (mixed.size() % 7 == 0)
+            {
+                mixed += "\xc3\xa4";
+            }
+            else if (mixed.size() % 11 == 0)
+            {
+                mixed += "\xf0\x9f\x98\x80";
+            }
+            else
+            {
+                mixed += "a";
+            }
+        }
+        same(ascii);
+        same(mixed);
+        for (std::size_t pos = 0; pos < len; ++pos)
+        {
+            for (const char bad :
+                    {'\x80', '\xc0', '\xc2', '\xe0', '\xf4', '\xff', '"', '\\', '\x1f'
+                    })
+            {
+                std::string s = ascii;
+                s[pos] = bad;
+                same(s);
+                s = mixed;
+                s[pos] = bad;
+                same(s);
+            }
+        }
+    }
+    // random bytes, biased to the interesting ones
+    std::mt19937 gen(5869); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed): reproducible
+    static const std::array<unsigned char, 23> interesting = {{0x00, 0x1f, 0x22, 0x5c, 0x7f, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc0, 0xc2, 0xdf, 0xe0, 0xe1, 0xed, 0xef, 0xf0, 0xf1, 0xf4, 0xf5, 0xff}};
+    for (int i = 0; i < 100000; ++i)
+    {
+        std::string s(gen() % 80, 'a');
+        for (auto& c : s)
+        {
+            const unsigned r = gen() % 8;
+            if (r == 0)
+            {
+                c = static_cast<char>(gen() & 0xff);
+            }
+            else if (r == 1)
+            {
+                c = static_cast<char>(interesting[gen() % interesting.size()]);
+            }
+        }
+        same(s);
+    }
+    CHECK(errors > 1000);
+}
+#endif