Keep thousands_sep/decimal_point const via a small locale_chars struct

const members can't be assigned in a constructor body, so seeding them
from std::localeconv() meant either dropping const or holding onto the
lconv* for longer than needed. A sub-object computes both from the
pointer in its own constructor and is itself initialized in serializer's
mem-initializer-list, so the two chars stay const, std::localeconv() is
still called exactly once, and nothing outlives the constructor.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/string_scan.hpp

@@ -93,6 +93,52 @@ inline std::size_t find_string_special(const unsigned char* data, std::size_t n)
     return n;
 }
 
+// classify a byte as one the serializer must NOT copy verbatim when
+// ensure_ascii is requested: the closing quote, an escape, a control character
+// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
+// printable ASCII except '"' and '\\' - is emitted unchanged. Note this differs
+// from is_string_special() only in that 0x7F is also a stop (it is escaped as
+// \u007f under ensure_ascii).
+inline bool is_ascii_copyable(unsigned char c) noexcept
+{
+    return c >= 0x20u && c < 0x7Fu && c != '"' && c != '\\';
+}
+
+// return the index of the first byte in [data, data+n) that is NOT
+// is_ascii_copyable(), or n if every byte can be copied verbatim; scans 8 bytes
+// at a time. Used by the serializer's ensure_ascii fast path.
+inline std::size_t find_ascii_copyable_run(const unsigned char* data, std::size_t n) noexcept
+{
+    constexpr std::uint64_t ones = 0x0101010101010101ull;
+    constexpr std::uint64_t high = 0x8080808080808080ull;
+    std::size_t i = 0;
+    for (; i + 8 <= n; i += 8)
+    {
+        std::uint64_t v = 0;
+        std::memcpy(&v, data + i, sizeof(v));
+        const std::uint64_t q = v ^ 0x2222222222222222ull; // '"'  (0x22)
+        const std::uint64_t b = v ^ 0x5C5C5C5C5C5C5C5Cull; // '\\' (0x5C)
+        const std::uint64_t d = v ^ 0x7F7F7F7F7F7F7F7Full; // DEL  (0x7F)
+        const std::uint64_t stop = ((q - ones) & ~q & high)   // == '"'
+                                   | ((b - ones) & ~b & high)  // == '\\'
+                                   | ((d - ones) & ~d & high)  // == 0x7F
+                                   | ((v - 0x2020202020202020ull) & ~v & high) // < 0x20
+                                   | (v & high);               // >= 0x80
+        if (stop != 0)
+        {
+            break;
+        }
+    }
+    for (; i < n; ++i)
+    {
+        if (!is_ascii_copyable(data[i]))
+        {
+            return i;
+        }
+    }
+    return n;
+}
+
 // Validate one UTF-8 sequence at the front of [data, data+avail). Returns its
 // length (2..4) only when the bytes form a *well-formed* sequence using exactly
 // the same ranges as scan_string()'s per-byte switch, so the bulk path accepts

include/nlohmann/detail/output/binary_writer.hpp

@@ -1681,16 +1681,6 @@ class binary_writer
         }
         CharType dtype = it->second;
 
-        // the 'B' (byte) marker is only defined by BJData Draft 3; emitting it
-        // under the default Draft 2 mode would produce a stream that Draft 2
-        // readers reject, so such an object falls back to a plain object
-        // encoding instead (see the "Binary values" section of the BJData
-        // documentation)
-        if (dtype == 'B' && bjdata_version != bjdata_version_t::draft3)
-        {
-            return true;
-        }
-
         key = "_ArraySize_";
         // the dimensions are written verbatim as the header length below, so a
         // value that is not an array cannot produce a valid one: null emits 'Z'

include/nlohmann/json.hpp

@@ -1343,15 +1343,18 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
                   const error_handler_t error_handler = error_handler_t::strict) const
     {
         string_t result;
-        serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler);
 
         if (indent >= 0)
         {
-            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
+            serializer s(detail::output_adapter<char, string_t>(result), indent_char,
+                         true, ensure_ascii, static_cast<std::size_t>(indent), error_handler);
+            s.dump(*this);
         }
         else
         {
-            s.dump(*this, false, ensure_ascii, 0);
+            serializer s(detail::output_adapter<char, string_t>(result), indent_char,
+                         false, ensure_ascii, 0, error_handler);
+            s.dump(*this);
         }
 
         return result;
@@ -4080,8 +4083,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         o.width(0);
 
         // do the actual serialization
-        serializer s(detail::output_adapter<char>(o), o.fill());
-        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
+        serializer s(detail::output_adapter<char>(o), o.fill(),
+                     pretty_print, false, static_cast<std::size_t>(indentation));
+        s.dump(j);
         return o;
     }
 

tests/src/unit-bjdata.cpp

@@ -2586,12 +2586,7 @@ TEST_CASE("BJData")
                 CHECK(json::to_bjdata(json::from_bjdata(v_d), true, true) == v_d);
                 CHECK(json::to_bjdata(json::from_bjdata(v_D), true, true) == v_D);
                 CHECK(json::to_bjdata(json::from_bjdata(v_C), true, true) == v_C);
-                // v_B uses the Draft-3-only 'B' marker, so it round-trips only when
-                // Draft 3 is explicitly selected (see GitHub issue #5404); the
-                // default Draft 2 falls back to a plain object instead, covered by
-                // the "ndarray with _ArrayType_ "byte" is gated by the BJData draft
-                // version" section below
-                CHECK(json::to_bjdata(json::from_bjdata(v_B), true, true, json::bjdata_version_t::draft3) == v_B);
+                CHECK(json::to_bjdata(json::from_bjdata(v_B), true, true) == v_B);
             }
 
             SECTION("ndarray with data not matching _ArrayType_ is written as an object")
@@ -2634,10 +2629,8 @@ TEST_CASE("BJData")
                 // the C++ API stores an int literal as number_integer, so _ArrayType_
                 // names the wire type rather than the storage. Both storages have to
                 // produce the same typed array for every type.
-                // "byte" is checked separately below since it additionally requires
-                // BJData Draft 3 to be selected explicitly (see GitHub issue #5404).
                 for (const char* type :
-                        {"uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64", "char"
+                        {"uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64", "char", "byte"
                         })
                 {
                     CAPTURE(type);
@@ -2648,14 +2641,6 @@ TEST_CASE("BJData")
                     CHECK(from_text == json::to_bjdata(json({{"_ArrayType_", type}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}})));
                 }
 
-                {
-                    const std::string text = R"({"_ArrayType_":"byte","_ArraySize_":[2,3],"_ArrayData_":[1,2,3,4,5,6]})";
-                    const auto from_text = json::to_bjdata(json::parse(text), true, true, json::bjdata_version_t::draft3);
-                    CHECK(from_text.at(0) == '[');
-                    CHECK(from_text == json::to_bjdata(json({{"_ArrayType_", "byte"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}}),
-                    true, true, json::bjdata_version_t::draft3));
-                }
-
                 // negative values under a signed type behave the same way
                 const auto from_neg = json::to_bjdata(json::parse(R"({"_ArrayType_":"int32","_ArraySize_":[2],"_ArrayData_":[-5,7]})"));
                 CHECK(from_neg.at(0) == '[');
@@ -2838,36 +2823,6 @@ TEST_CASE("BJData")
                 CHECK(out_single_ok.at(0) == '[');
                 CHECK(json::from_bjdata(out_single_ok) == json({1.5f}));
             }
-
-            SECTION("ndarray with _ArrayType_ \"byte\" is gated by the BJData draft version")
-            {
-                // the 'B' (byte) marker used by _ArrayType_ "byte" is only defined
-                // by BJData Draft 3; Draft 2 (the default) has no such marker, so
-                // emitting it unconditionally produced a stream that a Draft 2
-                // reader could not parse as intended (see GitHub issue #5404).
-                // Two dimensions are used so that a successfully written ndarray
-                // round-trips back into the annotated object (a single dimension
-                // is, by the BJData ndarray convention, read back as a plain
-                // binary value rather than the annotated object, same as every
-                // other single-dimension ndarray of a non-"byte" type is read
-                // back as a plain array instead of the annotated object).
-                json const j_byte = json({{"_ArrayType_", "byte"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}});
-
-                // default (Draft 2): falls back to a plain object and round-trips
-                const auto out_draft2 = json::to_bjdata(j_byte);
-                CHECK(out_draft2.at(0) == '{');
-                CHECK(json::from_bjdata(out_draft2) == j_byte);
-
-                // explicit Draft 2: same as the default
-                const auto out_draft2_explicit = json::to_bjdata(j_byte, true, true, json::bjdata_version_t::draft2);
-                CHECK(out_draft2_explicit.at(0) == '{');
-                CHECK(json::from_bjdata(out_draft2_explicit) == j_byte);
-
-                // Draft 3 explicitly selected: still uses the compact 'B' ndarray encoding
-                const auto out_draft3 = json::to_bjdata(j_byte, true, true, json::bjdata_version_t::draft3);
-                CHECK(out_draft3 == std::vector<uint8_t>({'[', '$', 'B', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5, 6}));
-                CHECK(json::from_bjdata(out_draft3) == j_byte);
-            }
         }
     }
 

tests/src/unit-convenience.cpp

@@ -98,8 +98,9 @@ void check_escaped(const char* original, const char* escaped = "", bool ensure_a
 void check_escaped(const char* original, const char* escaped, const bool ensure_ascii)
 {
     std::stringstream ss;
-    json::serializer s(nlohmann::detail::output_adapter<char>(ss), ' ');
-    s.dump_escaped(original, ensure_ascii);
+    json::serializer s(nlohmann::detail::output_adapter<char>(ss), ' ', false, ensure_ascii);
+    s.dump_escaped(original);
+    s.flush(); // dump_escaped writes into the serializer's internal buffer
     CHECK(ss.str() == escaped);
 }
 } // namespace

tests/src/unit-serialization.cpp

@@ -387,3 +387,232 @@ TEST_CASE("dump for basic_json with long double number_float_t")
         check_same(100.0L,  100.0);
     }
 }
+
+TEST_CASE("serialization of strings (bulk fast path)")
+{
+    // These cases exercise the SWAR bulk-copy fast path in dump_escaped and the
+    // internal write buffer: long runs, escapes interrupting runs, 0x7F/DEL,
+    // multibyte UTF-8 under both ensure_ascii settings, and payloads larger than
+    // the write buffer.
+
+    SECTION("long unescaped ASCII exceeds the write buffer")
+    {
+        const std::string big(3000, 'a');
+        const json j = big;
+        CHECK(j.dump() == '"' + big + '"');
+        CHECK(j.dump(-1, ' ', true) == '"' + big + '"');
+        // round-trips
+        CHECK(json::parse(j.dump()) == j);
+    }
+
+    SECTION("runs interrupted by escapes")
+    {
+        const json j = std::string(500, 'x') + "\n\"\\" + std::string(500, 'y');
+        const std::string out = j.dump();
+        CHECK(out == '"' + std::string(500, 'x') + "\\n\\\"\\\\" + std::string(500, 'y') + '"');
+        CHECK(json::parse(out) == j);
+    }
+
+    SECTION("DEL (0x7F) depends on ensure_ascii")
+    {
+        const json j = std::string("a\x7f" "b");
+        CHECK(j.dump(-1, ' ', false) == "\"a\x7f" "b\"");   // copied verbatim
+        CHECK(j.dump(-1, ' ', true) == "\"a\\u007fb\"");    // escaped
+    }
+
+    SECTION("multibyte UTF-8 under both ensure_ascii settings")
+    {
+        const json j = std::string("A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z"); // A é 你 😀 Z
+        // not escaping non-ASCII: bytes are copied through the bulk validator
+        CHECK(j.dump(-1, ' ', false) == "\"A\xc3\xa9\xe4\xbd\xa0\xf0\x9f\x98\x80Z\"");
+        // ensure_ascii: escaped (with a surrogate pair for the emoji)
+        CHECK(j.dump(-1, ' ', true) == "\"A\\u00e9\\u4f60\\ud83d\\ude00Z\"");
+        CHECK(json::parse(j.dump(-1, ' ', true)) == j);
+    }
+
+    SECTION("many small structural writes exceed the write buffer")
+    {
+        json arr = json::array();
+        for (int i = 0; i < 2000; ++i)
+        {
+            arr.push_back(i);
+        }
+        const std::string out = arr.dump();
+        CHECK(out.front() == '[');
+        CHECK(out.back() == ']');
+        CHECK(json::parse(out) == arr);
+
+        json obj = json::object();
+        for (int i = 0; i < 500; ++i)
+        {
+            obj["key" + std::to_string(i)] = i;
+        }
+        CHECK(json::parse(obj.dump()) == obj);
+        CHECK(json::parse(obj.dump(2)) == obj);
+
+        // an array of many empty strings emits a long run of single-character
+        // writes ('"', '"', ',') at shallow nesting depth, so the write buffer
+        // fills and flushes mid-run without the deep recursion that would
+        // overflow the stack on some debug builds
+        json many_empty = json::array();
+        for (int i = 0; i < 500; ++i)
+        {
+            many_empty.push_back("");
+        }
+        const std::string out2 = many_empty.dump();
+        CHECK(out2.size() > 1024); // spans multiple write-buffer flushes
+        CHECK(out2.front() == '[');
+        CHECK(out2.back() == ']');
+        CHECK(json::parse(out2) == many_empty);
+    }
+
+    SECTION("invalid UTF-8 handling is unaffected by the fast path")
+    {
+        const json j = std::string("valid\xff" "more");
+        CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&);
+        CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\"");
+        CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\"");
+        CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\"");
+    }
+}
+
+TEST_CASE("indentation is written straight into the write buffer")
+{
+    // put_indent() memsets the indentation into the write buffer instead of
+    // copying it out of a pre-grown indentation string. These cases cover an
+    // indentation wider than the buffer, a non-space indentation character, and
+    // nesting deep enough that the accumulated indentation spans several
+    // buffer-fulls - the situations the old grow-a-string approach got wrong.
+
+    SECTION("indent_step wider than the write buffer")
+    {
+        const json j = {{"a", 1}};
+        // 2000 > the 1024-byte write buffer, and > the 512 the indentation
+        // string used to start at
+        CHECK(j.dump(2000) == "{\n" + std::string(2000, ' ') + "\"a\": 1\n}");
+        // several whole buffer-fulls, so the buffer is refilled once and then
+        // flushed repeatedly
+        CHECK(j.dump(5000) == "{\n" + std::string(5000, ' ') + "\"a\": 1\n}");
+        CHECK(j.dump(5000, '\t') == "{\n" + std::string(5000, '\t') + "\"a\": 1\n}");
+        // an exact multiple of the buffer size
+        CHECK(j.dump(4096) == "{\n" + std::string(4096, ' ') + "\"a\": 1\n}");
+    }
+
+    SECTION("a non-space indentation character is used throughout")
+    {
+        const json j = {{"a", 1}};
+        // 600 is past the point where the indentation used to be grown, which
+        // is where a hard-coded space would have shown up
+        CHECK(j.dump(600, '\t') == "{\n" + std::string(600, '\t') + "\"a\": 1\n}");
+        CHECK(j.dump(3, '.') == "{\n...\"a\": 1\n}");
+    }
+
+    SECTION("accumulated indentation spans several buffer-fulls")
+    {
+        // five levels deep at 400 per level: the innermost value is indented by
+        // 2000 characters, reached in steps that each straddle the buffer end
+        json j = json::array({1});
+        for (int i = 0; i < 4; ++i)
+        {
+            j = json::array({j});
+        }
+
+        const std::string out = j.dump(400);
+        CHECK(out.find(std::string("\n") + std::string(2000, ' ') + "1\n") != std::string::npos);
+        CHECK(json::parse(out) == j);
+    }
+
+    SECTION("indentation is unchanged for ordinary widths")
+    {
+        const json j = {{"a", {1, 2}}, {"b", nullptr}};
+        CHECK(j.dump(2) == "{\n  \"a\": [\n    1,\n    2\n  ],\n  \"b\": null\n}");
+        CHECK(j.dump(0) == "{\n\"a\": [\n1,\n2\n],\n\"b\": null\n}");
+    }
+}
+
+TEST_CASE("serialization of deeply nested values")
+{
+    // dump() descends into a bounded number of levels and writes out whatever
+    // is nested deeper than that without the call stack; see
+    // https://github.com/nlohmann/json/issues/5387
+
+    SECTION("nested deeper than the call stack could follow")
+    {
+        // parsing is iterative, so building these costs little
+        const std::size_t depth = 100000;
+
+        const std::string array_text = std::string(depth, '[') + '0' + std::string(depth, ']');
+        CHECK(json::parse(array_text).dump() == array_text);
+
+        std::string object_text;
+        object_text.reserve((6 * depth) + 1);
+        for (std::size_t i = 0; i < depth; ++i)
+        {
+            object_text += "{\"a\":";
+        }
+        object_text += '1';
+        object_text.append(depth, '}');
+        CHECK(json::parse(object_text).dump() == object_text);
+    }
+
+    SECTION("depths around the bound of the descent")
+    {
+        // Cover every depth around the bound, so that the two ways of writing a
+        // value are known to meet cleanly - wherever the bound is set.
+        for (std::size_t d = 1; d <= 300; ++d)
+        {
+            CAPTURE(d);
+
+            const std::string array_text = std::string(d, '[') + '7' + std::string(d, ']');
+            CHECK(json::parse(array_text).dump() == array_text);
+
+            std::string object_text;
+            for (std::size_t i = 0; i < d; ++i)
+            {
+                object_text += "{\"k\":";
+            }
+            object_text += '7';
+            object_text.append(d, '}');
+            CHECK(json::parse(object_text).dump() == object_text);
+        }
+    }
+
+    SECTION("pretty-printing across the bound")
+    {
+        for (std::size_t d = 120; d <= 140; ++d)
+        {
+            CAPTURE(d);
+
+            const json j = json::parse(std::string(d, '[') + '7' + std::string(d, ']'));
+
+            std::string expected;
+            for (std::size_t i = 0; i < d; ++i)
+            {
+                expected += std::string(2 * i, ' ') + "[\n";
+            }
+            expected += std::string(2 * d, ' ') + '7';
+            for (std::size_t i = d; i > 0; --i)
+            {
+                expected += '\n' + std::string(2 * (i - 1), ' ') + ']';
+            }
+
+            CHECK(j.dump(2) == expected);
+        }
+    }
+
+    SECTION("an empty container below the bound")
+    {
+        // an empty container is written out in full and never descended into,
+        // so it must not gain a newline when it is reached iteratively
+        for (std::size_t d = 125; d <= 135; ++d)
+        {
+            CAPTURE(d);
+
+            const std::string compact = std::string(d, '[') + "[]" + std::string(d, ']');
+            CHECK(json::parse(compact).dump() == compact);
+
+            const std::string with_object = std::string(d, '[') + "{}" + std::string(d, ']');
+            CHECK(json::parse(with_object).dump() == with_object);
+        }
+    }
+}