Merge branch 'json-view/02b-float-parser' into json-view/23-zmij

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

tests/src/unit-bjdata.cpp

@@ -19,6 +19,7 @@
 
 #include <nlohmann/json.hpp>
 using nlohmann::json;
+using ordered_json = nlohmann::ordered_json;
 
 #include <algorithm>
 #include <climits>
@@ -2258,29 +2259,33 @@ TEST_CASE("BJData")
 
         SECTION("start_array() in ndarray _ArraySize_")
         {
+            // _ArrayType_ (2 events: key + string) is now emitted before
+            // _ArraySize_ (see GitHub issue #5661), which shifts the events
+            // below later by the same 2 events
             std::vector<uint8_t> const v = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 2, 1, 1, 2};
-            SaxCountdown scp(2);
+            SaxCountdown scp(4);
             CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata));
         }
 
         SECTION("number_integer() in ndarray _ArraySize_")
         {
             std::vector<uint8_t> const v = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 1, 1, 2};
-            SaxCountdown scp(3);
+            SaxCountdown scp(5);
             CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata));
         }
 
         SECTION("key() in ndarray _ArrayType_")
         {
+            // _ArrayType_ is emitted right after start_object(), before _ArraySize_
             std::vector<uint8_t> const v = {'[', '$', 'U', '#', '[', '$', 'U', '#', 'i', 2, 2, 2, 1, 2, 3, 4};
-            SaxCountdown scp(6);
+            SaxCountdown scp(1);
             CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata));
         }
 
         SECTION("string() in ndarray _ArrayType_")
         {
             std::vector<uint8_t> const v = {'[', '$', 'U', '#', '[', '$', 'U', '#', 'i', 2, 2, 2, 1, 2, 3, 4};
-            SaxCountdown scp(7);
+            SaxCountdown scp(2);
             CHECK_FALSE(json::sax_parse(v, &scp, json::input_format_t::bjdata));
         }
 
@@ -2883,6 +2888,22 @@ TEST_CASE("BJData")
                 CHECK(out_single.at(0) == '{');
                 CHECK(json::from_bjdata(out_single) == j_single);
 
+                // a double element that is finite and within the range of "single"
+                // but is not exactly representable as a float, so narrowing it would
+                // silently round it (0.1 is read back as 0.10000000149011612); this,
+                // like the overflow case above, falls back to a plain object (see
+                // GitHub issue #5661)
+                json const j_single_rounded = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5, 0.1}}});
+                const auto out_single_rounded = json::to_bjdata(j_single_rounded);
+                CHECK(out_single_rounded.at(0) == '{');
+                CHECK(json::from_bjdata(out_single_rounded) == j_single_rounded);
+
+                // a double element that underflows to 0 when narrowed to "single"
+                json const j_single_underflow = json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5, 1e-300}}});
+                const auto out_single_underflow = json::to_bjdata(j_single_underflow);
+                CHECK(out_single_underflow.at(0) == '{');
+                CHECK(json::from_bjdata(out_single_underflow) == j_single_underflow);
+
                 // in-range boundary values still use the compact ndarray encoding
                 json const j_uint8_ok = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {0, 255}}});
                 CHECK(json::to_bjdata(j_uint8_ok) == std::vector<uint8_t>({'[', '$', 'U', '#', '[', 'i', 2, 'i', 1, ']', 0, 255}));
@@ -2896,6 +2917,23 @@ TEST_CASE("BJData")
                 CHECK(json::from_bjdata(out_single_ok) == json({{"_ArrayType_", "single"}, {"_ArraySize_", {2, 1}}, {"_ArrayData_", {1.5f, -1.5f}}}));
             }
 
+            SECTION("ndarray annotation keys are read back in the documented order")
+            {
+                // from_bjdata() must emit the annotation object's keys in the order
+                // used throughout the documentation, _ArrayType_, _ArraySize_,
+                // _ArrayData_: the type marker precedes the dimension vector on the
+                // wire (see get_ubjson_size_type()), so it is known, and emitted,
+                // before _ArraySize_. For a plain json this key order is invisible
+                // (its comparison ignores it), but for an ordered_json it is not (see
+                // GitHub issue #5661).
+                const ordered_json o = ordered_json::parse(R"({"_ArrayType_":"uint8","_ArraySize_":[2,2],"_ArrayData_":[1,2,3,4]})");
+                const auto packed = ordered_json::to_bjdata(o);
+                CHECK(packed.at(0) == '[');
+                const ordered_json o_back = ordered_json::from_bjdata(packed);
+                CHECK(o_back == o);
+                CHECK(o_back.dump() == o.dump());
+            }
+
             SECTION("ndarray that would not be read back as an annotated object stays as object")
             {
                 // the reader only restores an annotated object from an ND-array

tests/src/unit-class_lexer.cpp

@@ -801,7 +801,7 @@ TEST_CASE("lexer escape fast path")
         // digits and non-hex bytes, at varying distances from the start of
         // the string, to compare the two scanners on many more shapes than
         // are practical to enumerate by hand.
-        std::mt19937 gen(7654321); // NOLINT(cert-msc32-c,cert-msc51-cpp)
+        std::mt19937 gen(7654321); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed)
         const std::string hex_alphabet = "0123456789AaBbCcDdEeFf";
         std::uniform_int_distribution<std::size_t> pick_hex(0, hex_alphabet.size() - 1);
         std::uniform_int_distribution<int> pick_byte(1, 255); // never NUL
@@ -1599,7 +1599,7 @@ template<typename Json, typename Bits>
 void check_parse(const std::string& token, Bits expected, Bits infinity)
 {
     std::stringstream stream(token);
-    if ((expected & ~(Bits{1} << (8 * sizeof(Bits) - 1))) == infinity)
+    if ((expected & ~(Bits{1} << ((8 * sizeof(Bits)) - 1))) == infinity)
     {
         Json _;
         CHECK_THROWS_WITH_AS(_ = Json::parse(token), ("[json.exception.out_of_range.406] number overflow parsing '" + token + "'").c_str(), typename Json::out_of_range&);

tests/src/unit-hash.cpp

@@ -12,8 +12,10 @@
 using json = nlohmann::json;
 using ordered_json = nlohmann::ordered_json;
 
+#include <limits>
 #include <set>
 #include <string>
+#include <unordered_set>
 
 namespace
 {
@@ -91,6 +93,9 @@ TEST_CASE("hash<nlohmann::json>")
     // Collect hashes for different JSON values and make sure that they are distinct
     // We cannot compare against fixed values, because the implementation of
     // std::hash may differ between compilers.
+    //
+    // numbers that compare equal under operator== (0 == 0U == 0.0) must hash
+    // equally, so they are only inserted once below and checked separately.
 
     std::set<std::size_t> hashes;
 
@@ -107,10 +112,7 @@ TEST_CASE("hash<nlohmann::json>")
 
     // number
     hashes.insert(std::hash<json> {}(json(0)));
-    hashes.insert(std::hash<json> {}(json(static_cast<unsigned>(0))));
-
     hashes.insert(std::hash<json> {}(json(-1)));
-    hashes.insert(std::hash<json> {}(json(0.0)));
     hashes.insert(std::hash<json> {}(json(42.23)));
 
     // array
@@ -132,7 +134,36 @@ TEST_CASE("hash<nlohmann::json>")
     // discarded
     hashes.insert(std::hash<json> {}(json(json::value_t::discarded)));
 
-    CHECK(hashes.size() == 21);
+    CHECK(hashes.size() == 19);
+
+    // numbers that compare equal under operator== must hash equally,
+    // regardless of which of number_integer, number_unsigned, or
+    // number_float actually holds the value
+    CHECK(json(0) == json(static_cast<unsigned>(0)));
+    CHECK(json(0) == json(0.0));
+    CHECK(std::hash<json> {}(json(0)) == std::hash<json> {}(json(static_cast<unsigned>(0))));
+    CHECK(std::hash<json> {}(json(0)) == std::hash<json> {}(json(0.0)));
+    CHECK(std::hash<json> {}(json(-1)) == std::hash<json> {}(json(-1.0)));
+
+    // a std::unordered_set relies on this same consistency between == and hash
+    const std::unordered_set<json> numbers {json(0), json(static_cast<unsigned>(0)), json(0.0)};
+    CHECK(numbers.size() == 1);
+
+    // -0.0 compares equal to 0 and 0.0
+    CHECK(json(-0.0) == json(0));
+    CHECK(std::hash<json> {}(json(-0.0)) == std::hash<json> {}(json(0)));
+    CHECK(std::hash<json> {}(json(-0.0)) == std::hash<json> {}(json(0.0)));
+
+    // the ends of the integer ranges, which equal floats exactly
+    const auto int_min = (std::numeric_limits<json::number_integer_t>::min)();
+    const auto int_max = (std::numeric_limits<json::number_integer_t>::max)();
+    const auto two_63 = json::number_unsigned_t(1) << 63U;
+    CHECK(json(int_min) == json(-9223372036854775808.0));
+    CHECK(std::hash<json> {}(json(int_min)) == std::hash<json> {}(json(-9223372036854775808.0)));
+    CHECK(json(two_63) == json(9223372036854775808.0));
+    CHECK(std::hash<json> {}(json(two_63)) == std::hash<json> {}(json(9223372036854775808.0)));
+    CHECK(json(json::number_unsigned_t(int_max)) == json(int_max));
+    CHECK(std::hash<json> {}(json(json::number_unsigned_t(int_max))) == std::hash<json> {}(json(int_max)));
 }
 
 TEST_CASE("hash<nlohmann::ordered_json>")
@@ -156,10 +187,7 @@ TEST_CASE("hash<nlohmann::ordered_json>")
 
     // number
     hashes.insert(std::hash<ordered_json> {}(ordered_json(0)));
-    hashes.insert(std::hash<ordered_json> {}(ordered_json(static_cast<unsigned>(0))));
-
     hashes.insert(std::hash<ordered_json> {}(ordered_json(-1)));
-    hashes.insert(std::hash<ordered_json> {}(ordered_json(0.0)));
     hashes.insert(std::hash<ordered_json> {}(ordered_json(42.23)));
 
     // array
@@ -181,7 +209,10 @@ TEST_CASE("hash<nlohmann::ordered_json>")
     // discarded
     hashes.insert(std::hash<ordered_json> {}(ordered_json(ordered_json::value_t::discarded)));
 
-    CHECK(hashes.size() == 21);
+    CHECK(hashes.size() == 19);
+
+    CHECK(std::hash<ordered_json> {}(ordered_json(0)) == std::hash<ordered_json> {}(ordered_json(static_cast<unsigned>(0))));
+    CHECK(std::hash<ordered_json> {}(ordered_json(0)) == std::hash<ordered_json> {}(ordered_json(0.0)));
 }
 
 TEST_CASE("hash of deeply nested values")

tests/src/unit-large_json.cpp

@@ -13,6 +13,7 @@ using nlohmann::json;
 
 #include <algorithm>
 #include <string>
+#include <utility>
 #include <vector>
 
 TEST_CASE("tests on very large JSONs")
@@ -354,3 +355,199 @@ TEST_CASE("tests on deeply nested JSONs")
     }
 }
 
+namespace
+{
+json nested_array(const std::size_t depth, json leaf)
+{
+    json j = std::move(leaf);
+    for (std::size_t i = 0; i < depth; ++i)
+    {
+        json a = json::array();
+        a.push_back(std::move(j));
+        j = std::move(a);
+    }
+    return j;
+}
+
+json nested_object(const std::size_t depth, json leaf)
+{
+    json j = std::move(leaf);
+    for (std::size_t i = 0; i < depth; ++i)
+    {
+        json o = json::object();
+        o["k"] = std::move(j);
+        j = std::move(o);
+    }
+    return j;
+}
+} // namespace
+
+TEST_CASE("issue #5392 - binary writers on deeply nested values")
+{
+    // 200 is past the point where the writers stop recursing, and still
+    // shallow enough that from_* and operator== (which still recurse) are fine.
+    const json deep_array = nested_array(200, json(0));
+    const json deep_object = nested_object(200, json("x"));
+    const json empty_array = nested_array(200, json::array());
+    const json empty_object = nested_object(200, json::object());
+    const json mixed = nested_object(80, nested_array(80, json(true)));
+
+    SECTION("roundtrip past the recursion bound")
+    {
+        CHECK(json::from_cbor(json::to_cbor(deep_array)) == deep_array);
+        CHECK(json::from_msgpack(json::to_msgpack(deep_array)) == deep_array);
+        CHECK(json::from_ubjson(json::to_ubjson(deep_array)) == deep_array);
+        CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, false)) == deep_array);
+        CHECK(json::from_ubjson(json::to_ubjson(deep_array, true, true)) == deep_array);
+        CHECK(json::from_bjdata(json::to_bjdata(deep_array)) == deep_array);
+
+        CHECK(json::from_cbor(json::to_cbor(deep_object)) == deep_object);
+        CHECK(json::from_msgpack(json::to_msgpack(deep_object)) == deep_object);
+        CHECK(json::from_ubjson(json::to_ubjson(deep_object)) == deep_object);
+        CHECK(json::from_ubjson(json::to_ubjson(deep_object, true, true)) == deep_object);
+        CHECK(json::from_bjdata(json::to_bjdata(deep_object)) == deep_object);
+
+        CHECK(json::from_cbor(json::to_cbor(empty_array)) == empty_array);
+        CHECK(json::from_msgpack(json::to_msgpack(empty_array)) == empty_array);
+        CHECK(json::from_ubjson(json::to_ubjson(empty_array)) == empty_array);
+        CHECK(json::from_ubjson(json::to_ubjson(empty_array, true, true)) == empty_array);
+
+        CHECK(json::from_cbor(json::to_cbor(empty_object)) == empty_object);
+        CHECK(json::from_msgpack(json::to_msgpack(empty_object)) == empty_object);
+        CHECK(json::from_ubjson(json::to_ubjson(empty_object)) == empty_object);
+
+        CHECK(json::from_cbor(json::to_cbor(mixed)) == mixed);
+        CHECK(json::from_msgpack(json::to_msgpack(mixed)) == mixed);
+        CHECK(json::from_ubjson(json::to_ubjson(mixed)) == mixed);
+        CHECK(json::from_bjdata(json::to_bjdata(mixed)) == mixed);
+    }
+
+    SECTION("the two ways of writing a value meet at the bound")
+    {
+        for (std::size_t depth = 120; depth <= 140; ++depth)
+        {
+            CAPTURE(depth);
+
+            const json array = nested_array(depth, json(7));
+            CHECK(json::from_cbor(json::to_cbor(array)) == array);
+            CHECK(json::from_msgpack(json::to_msgpack(array)) == array);
+            CHECK(json::from_ubjson(json::to_ubjson(array, true, true)) == array);
+
+            const json object = nested_object(depth, json(7));
+            CHECK(json::from_cbor(json::to_cbor(object)) == object);
+            CHECK(json::from_msgpack(json::to_msgpack(object)) == object);
+            CHECK(json::from_bjdata(json::to_bjdata(object)) == object);
+        }
+    }
+
+    SECTION("a BJData ndarray below the bound is still an ndarray")
+    {
+        const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}});
+        const json invalid = json({{"_ArrayType_", "nope"}, {"_ArraySize_", {1}}, {"_ArrayData_", {1}}});
+
+        const json deep_ndarray = nested_array(140, ndarray);
+        const json deep_invalid = nested_array(140, invalid);
+
+        CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray);
+        CHECK(json::from_bjdata(json::to_bjdata(deep_invalid)) == deep_invalid);
+        CHECK(json::from_bjdata(json::to_bjdata(ndarray)) == ndarray);
+    }
+
+    SECTION("byte-exact across the switch-over")
+    {
+        // nested one-element arrays around the recursion bound: the exact
+        // bytes a writer produces do not depend on whether it stayed on the
+        // call stack or moved to the heap one partway through
+        for (const std::size_t depth :
+                {
+                    nlohmann::detail::recursion_depth_limit() - 1, nlohmann::detail::recursion_depth_limit(),
+                    nlohmann::detail::recursion_depth_limit() + 1, nlohmann::detail::recursion_depth_limit() + 2
+                })
+        {
+            CAPTURE(depth);
+            const json array = nested_array(depth, json(0));
+
+            std::vector<std::uint8_t> expected_cbor(depth, 0x81);
+            expected_cbor.push_back(0x00);
+            CHECK(json::to_cbor(array) == expected_cbor);
+
+            std::vector<std::uint8_t> expected_msgpack(depth, 0x91);
+            expected_msgpack.push_back(0x00);
+            CHECK(json::to_msgpack(array) == expected_msgpack);
+
+            std::string expected_ubjson(depth, '[');
+            expected_ubjson += "i";
+            expected_ubjson += '\0';
+            expected_ubjson.append(depth, ']');
+            const auto packed_ubjson = json::to_ubjson(array);
+            CHECK(std::string(packed_ubjson.begin(), packed_ubjson.end()) == expected_ubjson);
+        }
+    }
+
+    SECTION("a deep object, and a BJData ndarray, past the recursion bound")
+    {
+        const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50;
+
+        const json object = nested_object(depth, json(42));
+        CHECK(json::from_cbor(json::to_cbor(object)) == object);
+        CHECK(json::from_msgpack(json::to_msgpack(object)) == object);
+        CHECK(json::from_ubjson(json::to_ubjson(object, true, true)) == object);
+        CHECK(json::from_bjdata(json::to_bjdata(object)) == object);
+
+        const json ndarray = json({{"_ArrayType_", "uint8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2, 3, 4, 5, 6}}});
+        const json deep_ndarray = nested_array(depth, ndarray);
+        CHECK(json::from_bjdata(json::to_bjdata(deep_ndarray)) == deep_ndarray);
+    }
+
+    SECTION("a discarded value past the recursion bound still throws type_error.321")
+    {
+        const std::size_t depth = nlohmann::detail::recursion_depth_limit() + 50;
+        const json discarded_leaf(json::value_t::discarded);
+        const json deep_discarded = nested_array(depth, discarded_leaf);
+
+        CHECK_THROWS_WITH_AS(json::to_cbor(deep_discarded), "[json.exception.type_error.321] cannot serialize discarded value to CBOR", json::type_error);
+        CHECK_THROWS_WITH_AS(json::to_msgpack(deep_discarded), "[json.exception.type_error.321] cannot serialize discarded value to MessagePack", json::type_error);
+        CHECK_THROWS_WITH_AS(json::to_ubjson(deep_discarded), "[json.exception.type_error.321] cannot serialize discarded value to UBJSON", json::type_error);
+        CHECK_THROWS_WITH_AS(json::to_bjdata(deep_discarded), "[json.exception.type_error.321] cannot serialize discarded value to BJData", json::type_error);
+    }
+
+    SECTION("does not overflow the C++ stack")
+    {
+        const std::size_t depth = 100000;
+        const json j = json::parse(std::string(depth, '[') + "0" + std::string(depth, ']'));
+
+        std::vector<std::uint8_t> packed;
+        CHECK_NOTHROW(packed = json::to_cbor(j));
+        CHECK(json::from_cbor(packed) == j);
+
+        CHECK_NOTHROW(packed = json::to_msgpack(j));
+        CHECK(json::from_msgpack(packed) == j);
+
+        CHECK_NOTHROW(packed = json::to_ubjson(j));
+        CHECK(json::from_ubjson(packed) == j);
+
+        CHECK_NOTHROW(packed = json::to_ubjson(j, true, false));
+        CHECK(json::from_ubjson(packed) == j);
+
+        CHECK_NOTHROW(packed = json::to_bjdata(j));
+        CHECK(json::from_bjdata(packed) == j);
+    }
+
+    SECTION("regression test for https://issues.oss-fuzz.com/issues/566583014")
+    {
+        // 200000 nested one-element CBOR arrays, the innermost holding null;
+        // round-tripping this used to recurse once per level on the way back
+        // out through to_cbor(), deep enough to overflow the stack
+        std::vector<std::uint8_t> v(200000, 0x81);
+        v.push_back(0xf6);
+        const json j = json::from_cbor(v);
+        CHECK(json::to_cbor(j) == v);
+
+        // the MessagePack analogue: fixarray of 1 nesting down to nil
+        std::vector<std::uint8_t> v_msgpack(200000, 0x91);
+        v_msgpack.push_back(0xc0);
+        const json j_msgpack = json::from_msgpack(v_msgpack);
+        CHECK(json::to_msgpack(j_msgpack) == v_msgpack);
+    }
+}
+

tests/src/unit-regression2.cpp

@@ -888,6 +888,15 @@ TEST_CASE("regression tests 2")
         CHECK(j == k);
     }
 
+    SECTION("issue #4552 - UTF-8 invalid characters are not always ignored when dumping with error_handler_t::ignore")
+    {
+        json node;
+        node["test"] = "test\334\005";
+        CHECK(node.dump(-1, ' ', false, json::error_handler_t::ignore) == "{\"test\":\"test\\u0005\"}");
+        CHECK(node.dump(-1, ' ', false, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
+        CHECK(node.dump(-1, ' ', true, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
+    }
+
 #ifdef JSON_HAS_CPP_17
     SECTION("issue #5066 - MSVC converts json to std::variant<json> via the conversion operator")
     {

tests/src/unit-serialization.cpp

@@ -102,6 +102,8 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"äü\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"ä\xEF\xBF\xBDü\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"ä\xA9ü\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\\u00e4\xA9\\u00fc\"");
         }
 
         SECTION("invalid character (regression guard for shared UTF-8 decoder, see #5529)")
@@ -124,6 +126,8 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xC2\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xC2\"");
         }
 
         SECTION("unexpected character")
@@ -136,6 +140,39 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123456\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\x34\x35\x36\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd456\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
+        }
+
+        SECTION("keep: valid characters are still escaped")
+        {
+            // an invalid byte followed by characters that must be escaped
+            const json j = "\xC2\"\\\n\xFF\x05";
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
+        }
+
+        SECTION("keep: truncated multibyte sequences")
+        {
+            CHECK(json("\xF0\x9F\x98").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
+            CHECK(json("\xF0\x9F\x98").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
+            CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
+            CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
+        }
+
+        SECTION("keep: long string with many invalid bytes")
+        {
+            // exceeds the internal string buffer several times
+            std::string input;
+            std::string expected = "\"";
+            for (int i = 0; i < 2000; ++i)
+            {
+                input += "\xFF\xE2\x82\n\xC3\xA4";
+                expected += "\xFF\xE2\x82\\n\xC3\xA4";
+            }
+            expected += "\"";
+            const json j = input;
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == expected);
         }
 
         SECTION("U+FFFD Substitution of Maximal Subparts")

tests/src/unit-unicode.cpp

@@ -11,6 +11,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <fstream>
 #include <string>
 #include "make_test_data_available.hpp"
@@ -67,8 +68,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     static std::string s_replaced2;
     static std::string s_replaced_ascii;
     static std::string s_replaced2_ascii;
+    static std::string s_kept;
+    static std::string s_kept2;
+    static std::string s_kept_ascii;
 
-    // dumping with ignore/replace must not throw in any case
+    // dumping with ignore/replace/keep must not throw in any case
     s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -77,6 +81,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
     s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
     s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
+    s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
 
     if (success_expected)
     {
@@ -86,6 +93,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
         // all dumps should agree on the string
         CHECK(s_strict == s_ignored);
         CHECK(s_strict == s_replaced);
+        CHECK(s_strict == s_kept);
     }
     else
     {
@@ -97,6 +105,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
 
         // check that replace string contains a replacement character
         CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
+
+        // ignore drops the invalid bytes, keep copies them
+        CHECK(s_ignored != s_kept);
+        CHECK(s_ignored_ascii != s_kept_ascii);
+
+        // unless a byte needs escaping, keep copies the input unchanged
+        const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
+        });
+        if (!needs_escaping)
+        {
+            CHECK(s_kept == "\"" + json_string + "\"");
+        }
     }
 
     // check that prefix and suffix are preserved
@@ -108,6 +130,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
     CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
     CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
+    CHECK(s_kept2.substr(1, 3) == "abc");
+    CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
 }
 
 void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);