Merge branch 'develop' into issue4552-ignore

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/ubuntu.yml

@@ -100,7 +100,7 @@ jobs:
     container: ubuntu:focal
     strategy:
       matrix:
-        target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_disabletuplereferenceconversion, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_no_thread_local]
+        target: [ci_cmake_flags, ci_test_diagnostics, ci_test_diagnostic_positions, ci_test_noexceptions, ci_test_noimplicitconversions, ci_test_legacycomparison, ci_test_noglobaludls, ci_test_disableenumserialization, ci_test_skiplibraryversioncheck, ci_test_simdutf, ci_test_strict_nul_handling, ci_test_no_thread_local]
     steps:
       - name: Install build-essential
         run: apt-get update ; apt-get install -y build-essential unzip wget git libssl-dev

CMakeLists.txt

@@ -55,7 +55,6 @@ option(JSON_Diagnostic_Positions           "Enable diagnostic positions." OFF)
 option(JSON_GlobalUDLs                     "Place user-defined string literals in the global namespace." ON)
 option(JSON_ImplicitConversions            "Enable implicit conversions." ON)
 option(JSON_DisableEnumSerialization       "Disable default integer enum serialization." OFF)
-option(JSON_DisableTupleReferenceConversion "Disable conversion from a one-element tuple of a JSON reference." OFF)
 option(JSON_LegacyDiscardedValueComparison "Enable legacy discarded value comparison." OFF)
 option(JSON_Install                        "Install CMake targets during install step." ${MAIN_PROJECT})
 option(JSON_MultipleHeaders                "Use non-amalgamated version of the library." ON)
@@ -102,10 +101,6 @@ if (JSON_DisableEnumSerialization)
     message(STATUS "Enum integer serialization is disabled (JSON_DISABLE_ENUM_SERIALIZATION=1)")
 endif()
 
-if (JSON_DisableTupleReferenceConversion)
-    message(STATUS "Tuple reference conversion is disabled (JSON_DISABLE_TUPLE_REFERENCE_CONVERSION=1)")
-endif()
-
 if (JSON_LegacyDiscardedValueComparison)
     message(STATUS "Legacy discarded value comparison enabled (JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1)")
 endif()
@@ -148,7 +143,6 @@ target_compile_definitions(
     $<$<NOT:$<BOOL:${JSON_GlobalUDLs}>>:JSON_USE_GLOBAL_UDLS=0>
     $<$<NOT:$<BOOL:${JSON_ImplicitConversions}>>:JSON_USE_IMPLICIT_CONVERSIONS=0>
     $<$<BOOL:${JSON_DisableEnumSerialization}>:JSON_DISABLE_ENUM_SERIALIZATION=1>
-    $<$<BOOL:${JSON_DisableTupleReferenceConversion}>:JSON_DISABLE_TUPLE_REFERENCE_CONVERSION=1>
     $<$<BOOL:${JSON_Diagnostics}>:JSON_DIAGNOSTICS=1>
     $<$<BOOL:${JSON_Diagnostic_Positions}>:JSON_DIAGNOSTIC_POSITIONS=1>
     $<$<BOOL:${JSON_LegacyDiscardedValueComparison}>:JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1>

README.md

@@ -363,7 +363,7 @@ std::cout << j_string << " == " << serialized_string << std::endl;
 
 [`.dump()`](https://json.nlohmann.me/api/basic_json/dump/) returns the originally stored string value.
 
-Note the library only supports UTF-8. When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace` or `json::error_handler_t::ignore` are used as error handlers.
+Note the library only supports UTF-8. When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace`, `json::error_handler_t::ignore`, or `json::error_handler_t::keep` are used as error handlers.
 
 #### To/from streams (e.g., files, string streams)
 
@@ -1915,7 +1915,7 @@ The library supports **Unicode input** as follows:
 - [Unicode noncharacters](https://www.unicode.org/faq/private_use.html#nonchar1) will not be replaced by the library.
 - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors.
 - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs.
-- When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace` or `json::error_handler_t::ignore` are used as error handlers.
+- When you store strings with different encodings in the library, calling [`dump()`](https://json.nlohmann.me/api/basic_json/dump/) may throw an exception unless `json::error_handler_t::replace`, `json::error_handler_t::ignore`, or `json::error_handler_t::keep` are used as error handlers.
 - To store wide strings (e.g., `std::wstring`), you need to convert them to a UTF-8 encoded `std::string` before, see [an example](https://json.nlohmann.me/home/faq/#wide-string-handling).
 
 ### Comments in JSON

cmake/ci.cmake

@@ -276,20 +276,6 @@ add_custom_target(ci_test_disableenumserialization
     COMMENT "Compile and test with enum serialization disabled"
 )
 
-###############################################################################
-# Disable conversion from a one-element tuple of a JSON reference.
-###############################################################################
-
-add_custom_target(ci_test_disabletuplereferenceconversion
-    COMMAND ${CMAKE_COMMAND}
-    -DCMAKE_BUILD_TYPE=Debug -GNinja
-    -DJSON_BuildTests=ON -DJSON_FastTests=ON -DJSON_DisableTupleReferenceConversion=ON
-    -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_disabletuplereferenceconversion
-    COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_disabletuplereferenceconversion
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_disabletuplereferenceconversion && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure
-    COMMENT "Compile and test with tuple reference conversion disabled"
-)
-
 ###############################################################################
 # Skip the multiple-inclusion library version check.
 ###############################################################################

docs/docset/docSet.sql

@@ -210,7 +210,6 @@ INSERT INTO searchIndex(name, type, path) VALUES ('JSON_CATCH_USER', 'Macro', 'a
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_DIAGNOSTICS', 'Macro', 'api/macros/json_diagnostics/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_DIAGNOSTIC_POSITIONS', 'Macro', 'api/macros/json_diagnostic_positions/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_DISABLE_ENUM_SERIALIZATION', 'Macro', 'api/macros/json_disable_enum_serialization/index.html');
-INSERT INTO searchIndex(name, type, path) VALUES ('JSON_DISABLE_TUPLE_REFERENCE_CONVERSION', 'Macro', 'api/macros/json_disable_tuple_reference_conversion/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_HAS_CPP_11', 'Macro', 'api/macros/json_has_cpp_11/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_HAS_CPP_14', 'Macro', 'api/macros/json_has_cpp_11/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('JSON_HAS_CPP_17', 'Macro', 'api/macros/json_has_cpp_11/index.html');

docs/mkdocs/docs/api/basic_json/basic_json.md

@@ -159,8 +159,6 @@ basic_json(basic_json&& other) noexcept;
     - `CompatibleType` is not `basic_json` (to avoid hijacking copy/move constructors),
     - `CompatibleType` is not a different `basic_json` type (i.e. with different template arguments)
     - `CompatibleType` is not a `basic_json` nested type (e.g., `json_pointer`, `iterator`, etc.)
-    - if [`JSON_DISABLE_TUPLE_REFERENCE_CONVERSION`](../macros/json_disable_tuple_reference_conversion.md) is defined
-      to `1`: `CompatibleType` is not a one-element `std::tuple` holding a reference to `basic_json`
     - `json_serializer<U>` (with `U = uncvref_t<CompatibleType>`) has a `to_json(basic_json_t&, CompatibleType&&)`
        method
 

docs/mkdocs/docs/api/basic_json/dump.md

@@ -25,10 +25,15 @@ and `ensure_ascii` parameters.
     result consists of ASCII characters only.
 
 `error_handler` (in)
-:   how to react on decoding errors; there are three possible values (see [`error_handler_t`](error_handler_t.md):
-    `strict` (throws an exception in case a decoding error occurs; default), `replace` (replace invalid UTF-8 sequences
-    with U+FFFD), and `ignore` (ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the
-    output unchanged, and invalid bytes are dropped)).
+:   how to react on decoding errors; there are four possible values (see [`error_handler_t`](error_handler_t.md)):
+
+    - `strict`: throw a [`type_error`](../../home/exceptions.md#type-errors) exception in case a decoding error occurs
+      (default),
+    - `replace`: replace invalid UTF-8 sequences with U+FFFD (� REPLACEMENT CHARACTER),
+    - `ignore`: ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the output unchanged,
+      and invalid bytes are dropped, and
+    - `keep`: keep invalid UTF-8 sequences during serialization; all bytes are copied to the output unchanged, so the
+      result is not valid UTF-8.
     
 ## Return value
 
@@ -94,3 +99,4 @@ Binary values are serialized as an object containing two keys:
 - Indentation character `indent_char`, option `ensure_ascii` and exceptions added in version 3.0.0.
 - Error handlers added in version 3.4.0.
 - Serialization of binary values added in version 3.8.0.
+- Error handler value `keep` added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/error_handler_t.md

@@ -4,12 +4,13 @@
 enum class error_handler_t {
     strict,
     replace,
-    ignore
+    ignore,
+    keep
 };
 ```
 
 This enumeration is used in the [`dump`](dump.md) function to choose how to treat decoding errors while serializing a
-`basic_json` value. Three values are differentiated:
+`basic_json` value. Four values are differentiated:
 
 strict
 :   throw a `type_error` exception in case of invalid UTF-8
@@ -20,6 +21,12 @@ replace
 ignore
 :   ignore invalid UTF-8 sequences; all valid bytes are copied to the output unchanged, and invalid bytes are dropped
 
+keep
+:   keep invalid UTF-8 sequences; all bytes are copied to the output unchanged. Valid characters are still escaped as
+    usual (e.g., `"`, `\\`, and control characters), so the result has valid JSON syntax, but it is not valid UTF-8.
+    In particular, [`parse`](parse.md) rejects it, and with `ensure_ascii` set to `true`, the invalid bytes are the
+    only non-ASCII bytes of the output.
+
 ## Examples
 
 ??? example
@@ -40,3 +47,4 @@ ignore
 ## Version history
 
 - Added in version 3.4.0.
+- Added value `keep` in version 3.13.0.

docs/mkdocs/docs/api/macros/index.md

@@ -52,7 +52,6 @@ header. See also the [macro overview page](../../features/macros.md).
 
 - [**JSON_BRACE_INIT_COPY_SEMANTICS**](json_brace_init_copy_semantics.md) - opt in to copy/move semantics for single-element brace initialization
 - [**JSON_DISABLE_ENUM_SERIALIZATION**](json_disable_enum_serialization.md) - switch off default serialization/deserialization functions for enums
-- [**JSON_DISABLE_TUPLE_REFERENCE_CONVERSION**](json_disable_tuple_reference_conversion.md) - switch off conversion from a one-element tuple of a JSON reference
 - [**JSON_USE_IMPLICIT_CONVERSIONS**](json_use_implicit_conversions.md) - control implicit conversions
 
 ## Comparison behavior

docs/mkdocs/docs/api/macros/json_disable_tuple_reference_conversion.md

@@ -1,114 +0,0 @@
-# JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-
-```cpp
-#define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION /* value */
-```
-
-When defined to `1`, a `basic_json` value can no longer be constructed from a one-element `std::tuple` whose element is
-a reference to that `basic_json` type, such as `std::tuple<json&>`, `std::tuple<const json&>`, or `std::tuple<json&&>`.
-These are the tuples created by `std::forward_as_tuple(j)`.
-
-## Default definition
-
-The default value is `0` (disabled — existing behavior is preserved).
-
-```cpp
-#define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0
-```
-
-## Notes
-
-!!! note "Background"
-
-    By default, `basic_json` can be constructed from any `std::tuple` whose elements can be converted to JSON; the result
-    is an array. This includes `std::tuple<json&>`, which becomes a one-element array.
-
-    `std::tuple` only converts another tuple element by element if its element type cannot be constructed from the whole
-    source tuple. Because `json` *can* be constructed from `std::tuple<json&>`, `std::tuple` instead converts the whole
-    tuple into a single `json` value. This has two surprising effects:
-
-    ```cpp
-    json j = true;
-
-    // rejected by some standard libraries (e.g., libc++); with others, the
-    // reference binds to a temporary that is destroyed right away
-    std::tuple<const json&> t1(std::forward_as_tuple(j));
-
-    // compiles, but std::get<0>(t2) is [true], not true
-    std::tuple<json> t2(std::forward_as_tuple(j));
-    ```
-
-    Enabling this macro removes the conversion, so both tuples are converted element by element: `std::get<0>(t1)`
-    refers to `j`, and `std::get<0>(t2)` is a copy of `j` (see [#2226](https://github.com/nlohmann/json/issues/2226)).
-
-!!! warning "Opt-in only"
-
-    This macro must be defined **before** including `<nlohmann/json.hpp>`. Defining it after the include has no effect.
-
-!!! note "Affected conversions"
-
-    Only one-element tuples holding a reference to the **same** `basic_json` type are affected. Constructing a JSON value
-    from them no longer compiles:
-
-    ```cpp
-    json j = true;
-    json a = std::forward_as_tuple(j);  // error with the macro enabled
-    json b = json::array({j});          // use this instead: [true]
-    ```
-
-    Tuples holding a JSON value (`std::make_tuple(j)`), tuples with more than one element, and tuples holding references
-    to other types (including other `basic_json` specializations) are converted to arrays as before.
-
-!!! hint "CMake option"
-
-    This behavior can also be controlled with the CMake option
-    [`JSON_DisableTupleReferenceConversion`](../../integration/cmake.md#json_disabletuplereferenceconversion)
-    (`OFF` by default) which defines `JSON_DISABLE_TUPLE_REFERENCE_CONVERSION` accordingly.
-
-## Examples
-
-??? example "Default behavior (macro not defined)"
-
-    ```cpp
-    #include <nlohmann/json.hpp>
-
-    using json = nlohmann::json;
-
-    int main()
-    {
-        json j = true;
-
-        std::tuple<json> t(std::forward_as_tuple(j));
-        // std::get<0>(t) is [true] -- the whole tuple was converted
-    }
-    ```
-
-??? example "Conversion disabled (macro defined to 1)"
-
-    ```cpp
-    #define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 1
-    #include <nlohmann/json.hpp>
-
-    using json = nlohmann::json;
-
-    int main()
-    {
-        json j = true;
-
-        std::tuple<json> t(std::forward_as_tuple(j));
-        // std::get<0>(t) is true -- a copy of j
-
-        std::tuple<const json&> r(std::forward_as_tuple(j));
-        // std::get<0>(r) refers to j
-    }
-    ```
-
-## See also
-
-- [**basic_json(CompatibleType&&)**](../basic_json/basic_json.md) - the affected constructor
-- [:simple-cmake: JSON_DisableTupleReferenceConversion](../../integration/cmake.md#json_disabletuplereferenceconversion) -
-  CMake option to control the macro
-
-## Version history
-
-- Added in version 3.13.0.

docs/mkdocs/docs/examples/error_handler_t.cpp

@@ -1,3 +1,4 @@
+#include <iomanip>
 #include <iostream>
 #include <nlohmann/json.hpp>
 
@@ -21,4 +22,12 @@ int main()
               << "\nstring with ignored invalid characters: "
               << j_invalid.dump(-1, ' ', false, json::error_handler_t::ignore)
               << '\n';
+
+    // the invalid byte is kept; print the result byte-wise to make it visible
+    std::cout << "string with kept invalid characters:";
+    for (const unsigned char c : j_invalid.dump(-1, ' ', false, json::error_handler_t::keep))
+    {
+        std::cout << ' ' << std::hex << std::setw(2) << std::setfill('0') << static_cast<int>(c);
+    }
+    std::cout << '\n';
 }

docs/mkdocs/docs/examples/error_handler_t.output

@@ -1,3 +1,4 @@
 [json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9
 string with replaced invalid characters: "ä�ü"
 string with ignored invalid characters: "äü"
+string with kept invalid characters: 22 c3 a4 a9 c3 bc 22

docs/mkdocs/docs/features/macros.md

@@ -83,13 +83,6 @@ When defined, default parse and serialize functions for enums are excluded and h
 
 See [full documentation of `JSON_DISABLE_ENUM_SERIALIZATION`](../api/macros/json_disable_enum_serialization.md).
 
-## `JSON_DISABLE_TUPLE_REFERENCE_CONVERSION`
-
-When defined to `1`, a JSON value can no longer be created from a one-element `std::tuple` holding a reference to a JSON
-value, such as the result of `std::forward_as_tuple(j)`. This lets `std::tuple` convert such tuples element-wise.
-
-See [full documentation of `JSON_DISABLE_TUPLE_REFERENCE_CONVERSION`](../api/macros/json_disable_tuple_reference_conversion.md).
-
 ## `JSON_NO_IO`
 
 When defined, headers `<cstdio>`, `<ios>`, `<iosfwd>`, `<istream>`, and `<ostream>` are not included and parse functions

docs/mkdocs/docs/features/serialization.md

@@ -64,6 +64,7 @@ serialization fails by default. The fourth argument of `dump` selects an
 - `strict` (default) — throw a [`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) exception.
 - `replace` — replace invalid bytes with the Unicode replacement character U+FFFD (`�`).
 - `ignore` — silently drop invalid bytes.
+- `keep` — copy invalid bytes to the output unchanged; the result is not valid UTF-8.
 
 ??? example
 

docs/mkdocs/docs/home/exceptions.md

@@ -755,6 +755,7 @@ The `dump()` function only works with UTF-8 encoded strings; that is, if you ass
     - Pass an error handler as last parameter to the `dump()` function to avoid this exception:
         - `json::error_handler_t::replace` will replace invalid bytes sequences with `U+FFFD` 
         - `json::error_handler_t::ignore` will silently ignore invalid byte sequences
+        - `json::error_handler_t::keep` will copy invalid byte sequences to the output unchanged
 
 ### json.exception.type_error.317
 

docs/mkdocs/docs/home/faq.md

@@ -85,7 +85,7 @@ The library supports **Unicode input** as follows:
 - The library will not replace [Unicode noncharacters](http://www.unicode.org/faq/private_use.html#nonchar1).
 - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors.
 - The strings stored in the library are UTF-8 encoded. When using the default string type (`std::string`), note that its length/size functions return the number of stored bytes rather than the number of characters or glyphs.
-- When you store strings with different encodings in the library, calling [`dump()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a50ec80b02d0f3f51130d4abb5d1cfdc5.html#a50ec80b02d0f3f51130d4abb5d1cfdc5) may throw an exception unless `json::error_handler_t::replace` or `json::error_handler_t::ignore` are used as error handlers.
+- When you store strings with different encodings in the library, calling [`dump()`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a50ec80b02d0f3f51130d4abb5d1cfdc5.html#a50ec80b02d0f3f51130d4abb5d1cfdc5) may throw an exception unless `json::error_handler_t::replace`, `json::error_handler_t::ignore`, or `json::error_handler_t::keep` are used as error handlers.
 
 In most cases, the parser is right to complain, because the input is not UTF-8 encoded. This is especially true for Microsoft Windows, where Latin-1 or ISO 8859-1 is often the standard encoding.
 

docs/mkdocs/docs/integration/cmake.md

@@ -169,12 +169,6 @@ Enable position diagnostics by defining macro [`JSON_DIAGNOSTIC_POSITIONS`](../a
 Disable default `enum` serialization by defining the macro
 [`JSON_DISABLE_ENUM_SERIALIZATION`](../api/macros/json_disable_enum_serialization.md). This option is `OFF` by default.
 
-### `JSON_DisableTupleReferenceConversion`
-
-Disable the conversion from a one-element `std::tuple` holding a reference to a JSON value by defining the macro
-[`JSON_DISABLE_TUPLE_REFERENCE_CONVERSION`](../api/macros/json_disable_tuple_reference_conversion.md). This option is
-`OFF` by default.
-
 ### `JSON_FastTests`
 
 Skip expensive/slow test suites. This option is `OFF` by default. Depends on `JSON_BuildTests`.

docs/mkdocs/mkdocs.yml

@@ -287,7 +287,6 @@ nav:
           - 'JSON_DIAGNOSTICS': api/macros/json_diagnostics.md
           - 'JSON_DIAGNOSTIC_POSITIONS': api/macros/json_diagnostic_positions.md
           - 'JSON_DISABLE_ENUM_SERIALIZATION': api/macros/json_disable_enum_serialization.md
-          - 'JSON_DISABLE_TUPLE_REFERENCE_CONVERSION': api/macros/json_disable_tuple_reference_conversion.md
           - 'JSON_HAS_CPP_11, JSON_HAS_CPP_14, JSON_HAS_CPP_17, JSON_HAS_CPP_20': api/macros/json_has_cpp_11.md
           - 'JSON_HAS_EXPERIMENTAL_FILESYSTEM, JSON_HAS_FILESYSTEM': api/macros/json_has_filesystem.md
           - 'JSON_HAS_RANGES': api/macros/json_has_ranges.md

include/nlohmann/detail/conversions/to_json.hpp

@@ -471,13 +471,11 @@ inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<
     j = { std::get<Idx>(t)... };
 }
 
-// A one-element braced list does not reliably wrap its element: with
-// JSON_BRACE_INIT_COPY_SEMANTICS it copies it, which would serialize
-// std::tuple<int>{5} as 5 rather than [5], and some compilers (e.g., Apple clang
-// 15 and 16) copy an element that is itself a basic_json even without it, so
-// std::tuple<json>{true} became true rather than [true]. Build what the default
-// deduction builds instead: an object if the element is a [string, value] pair,
-// a one-element array otherwise.
+#if JSON_BRACE_INIT_COPY_SEMANTICS
+// JSON_BRACE_INIT_COPY_SEMANTICS makes a one-element braced list copy its
+// element instead of wrapping it, which would serialize std::tuple<int>{5} as 5
+// rather than [5]. Build what the default deduction builds instead: an object
+// if the element is a [string, value] pair, a one-element array otherwise.
 template<typename BasicJsonType, typename Tuple>
 inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<0> /*unused*/)
 {
@@ -495,6 +493,7 @@ inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<
         j = BasicJsonType::array({std::move(element)});
     }
 }
+#endif
 
 template<typename BasicJsonType, typename Tuple>
 inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& /*unused*/, index_sequence<> /*unused*/)

include/nlohmann/detail/macro_scope.hpp

@@ -916,10 +916,6 @@ void templated_json_throw(ExceptionType exception)
     #define JSON_DISABLE_ENUM_SERIALIZATION 0
 #endif
 
-#ifndef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-    #define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0
-#endif
-
 #ifndef JSON_USE_GLOBAL_UDLS
     #define JSON_USE_GLOBAL_UDLS 1
 #endif

include/nlohmann/detail/macro_unscope.hpp

@@ -25,7 +25,6 @@
 #undef JSON_INLINE_VARIABLE
 #undef JSON_NO_UNIQUE_ADDRESS
 #undef JSON_DISABLE_ENUM_SERIALIZATION
-#undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
 #undef JSON_USE_GLOBAL_UDLS
 
 #ifndef JSON_TEST_KEEP_MACROS

include/nlohmann/detail/meta/type_traits.hpp

@@ -636,18 +636,6 @@ template<typename BasicJsonType, typename CompatibleType>
 struct is_compatible_type
     : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
 
-// a one-element std::tuple holding a reference to BasicJsonType, as created by
-// std::forward_as_tuple(j); see JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-template<typename BasicJsonType, typename T>
-struct is_basic_json_reference_tuple : std::false_type {};
-
-template<typename BasicJsonType, typename T>
-struct is_basic_json_reference_tuple<BasicJsonType, std::tuple<T>>
-{
-    static constexpr bool value =
-        std::is_reference<T>::value && std::is_same<uncvref_t<T>, BasicJsonType>::value;
-};
-
 template<typename BasicJsonType, typename CompatibleArrayType>
 struct is_compatible_binary_type
 {

include/nlohmann/detail/output/serializer.hpp

@@ -48,7 +48,8 @@ enum class error_handler_t
 {
     strict,  ///< throw a type_error exception in case of invalid UTF-8
     replace, ///< replace invalid UTF-8 sequences with U+FFFD
-    ignore   ///< ignore invalid UTF-8 sequences
+    ignore,  ///< ignore invalid UTF-8 sequences
+    keep     ///< keep invalid UTF-8 sequences; their bytes are copied unchanged
 };
 
 template<typename BasicJsonType>
@@ -1019,6 +1020,47 @@ class serializer
                             break;
                         }
 
+                        case error_handler_t::keep:
+                        {
+                            // drop whatever the incomplete sequence left in
+                            // the buffer (only copied if !EnsureAscii) and copy
+                            // the ill-formed bytes from the input instead
+                            bytes = bytes_after_last_accept;
+
+                            if (undumped_chars > 0)
+                            {
+                                // the pending bytes of the incomplete sequence
+                                // are ill-formed; the current byte may be OK for
+                                // itself, so we would like to read it again
+                                for (std::size_t j = i - undumped_chars; j < i; ++j)
+                                {
+                                    string_buffer[bytes++] = s[j];
+                                }
+                                --i;
+                            }
+                            else
+                            {
+                                // the current byte cannot start any sequence
+                                string_buffer[bytes++] = s[i];
+                            }
+
+                            // write buffer and reset index; there must be 13 bytes
+                            // left, as this is the maximal number of bytes to be
+                            // written ("\uxxxx\uxxxx\0") for one code point
+                            if (string_buffer.size() - bytes < 13)
+                            {
+                                put_buffer(string_buffer, bytes);
+                                bytes = 0;
+                            }
+
+                            bytes_after_last_accept = bytes;
+                            undumped_chars = 0;
+
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
+                        }
+
                         default:            // LCOV_EXCL_LINE
                             JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
                     }
@@ -1064,6 +1106,15 @@ class serializer
                     break;
                 }
 
+                case error_handler_t::keep:
+                {
+                    // write all accepted bytes
+                    put_buffer(string_buffer, bytes_after_last_accept);
+                    // copy the bytes of the incomplete sequence unchanged
+                    put_string(s, s.size() - undumped_chars, s.size());
+                    break;
+                }
+
                 case error_handler_t::replace:
                 {
                     // write all accepted bytes

include/nlohmann/json.hpp

@@ -1595,12 +1595,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     template < typename CompatibleType,
                typename U = detail::uncvref_t<CompatibleType>,
                detail::enable_if_t <
-                   !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value
-#if JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-                   // see https://github.com/nlohmann/json/issues/2226
-                   && !detail::is_basic_json_reference_tuple<basic_json_t, U>::value
-#endif
-                   , int > = 0 >
+                   !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value, int > = 0 >
     basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape)
                 JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
                                            std::forward<CompatibleType>(val))))

single_include/nlohmann/json.hpp

@@ -3327,10 +3327,6 @@ void templated_json_throw(ExceptionType exception)
     #define JSON_DISABLE_ENUM_SERIALIZATION 0
 #endif
 
-#ifndef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-    #define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 0
-#endif
-
 #ifndef JSON_USE_GLOBAL_UDLS
     #define JSON_USE_GLOBAL_UDLS 1
 #endif
@@ -4650,18 +4646,6 @@ template<typename BasicJsonType, typename CompatibleType>
 struct is_compatible_type
     : is_compatible_type_impl<BasicJsonType, CompatibleType> {};
 
-// a one-element std::tuple holding a reference to BasicJsonType, as created by
-// std::forward_as_tuple(j); see JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-template<typename BasicJsonType, typename T>
-struct is_basic_json_reference_tuple : std::false_type {};
-
-template<typename BasicJsonType, typename T>
-struct is_basic_json_reference_tuple<BasicJsonType, std::tuple<T>>
-{
-    static constexpr bool value =
-        std::is_reference<T>::value && std::is_same<uncvref_t<T>, BasicJsonType>::value;
-};
-
 template<typename BasicJsonType, typename CompatibleArrayType>
 struct is_compatible_binary_type
 {
@@ -7016,13 +7000,11 @@ inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<
     j = { std::get<Idx>(t)... };
 }
 
-// A one-element braced list does not reliably wrap its element: with
-// JSON_BRACE_INIT_COPY_SEMANTICS it copies it, which would serialize
-// std::tuple<int>{5} as 5 rather than [5], and some compilers (e.g., Apple clang
-// 15 and 16) copy an element that is itself a basic_json even without it, so
-// std::tuple<json>{true} became true rather than [true]. Build what the default
-// deduction builds instead: an object if the element is a [string, value] pair,
-// a one-element array otherwise.
+#if JSON_BRACE_INIT_COPY_SEMANTICS
+// JSON_BRACE_INIT_COPY_SEMANTICS makes a one-element braced list copy its
+// element instead of wrapping it, which would serialize std::tuple<int>{5} as 5
+// rather than [5]. Build what the default deduction builds instead: an object
+// if the element is a [string, value] pair, a one-element array otherwise.
 template<typename BasicJsonType, typename Tuple>
 inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<0> /*unused*/)
 {
@@ -7040,6 +7022,7 @@ inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<
         j = BasicJsonType::array({std::move(element)});
     }
 }
+#endif
 
 template<typename BasicJsonType, typename Tuple>
 inline void to_json_tuple_impl(BasicJsonType& j, const Tuple& /*unused*/, index_sequence<> /*unused*/)
@@ -23948,7 +23931,8 @@ enum class error_handler_t
 {
     strict,  ///< throw a type_error exception in case of invalid UTF-8
     replace, ///< replace invalid UTF-8 sequences with U+FFFD
-    ignore   ///< ignore invalid UTF-8 sequences
+    ignore,  ///< ignore invalid UTF-8 sequences
+    keep     ///< keep invalid UTF-8 sequences; their bytes are copied unchanged
 };
 
 template<typename BasicJsonType>
@@ -24919,6 +24903,47 @@ class serializer
                             break;
                         }
 
+                        case error_handler_t::keep:
+                        {
+                            // drop whatever the incomplete sequence left in
+                            // the buffer (only copied if !EnsureAscii) and copy
+                            // the ill-formed bytes from the input instead
+                            bytes = bytes_after_last_accept;
+
+                            if (undumped_chars > 0)
+                            {
+                                // the pending bytes of the incomplete sequence
+                                // are ill-formed; the current byte may be OK for
+                                // itself, so we would like to read it again
+                                for (std::size_t j = i - undumped_chars; j < i; ++j)
+                                {
+                                    string_buffer[bytes++] = s[j];
+                                }
+                                --i;
+                            }
+                            else
+                            {
+                                // the current byte cannot start any sequence
+                                string_buffer[bytes++] = s[i];
+                            }
+
+                            // write buffer and reset index; there must be 13 bytes
+                            // left, as this is the maximal number of bytes to be
+                            // written ("\uxxxx\uxxxx\0") for one code point
+                            if (string_buffer.size() - bytes < 13)
+                            {
+                                put_buffer(string_buffer, bytes);
+                                bytes = 0;
+                            }
+
+                            bytes_after_last_accept = bytes;
+                            undumped_chars = 0;
+
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
+                        }
+
                         default:            // LCOV_EXCL_LINE
                             JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
                     }
@@ -24964,6 +24989,15 @@ class serializer
                     break;
                 }
 
+                case error_handler_t::keep:
+                {
+                    // write all accepted bytes
+                    put_buffer(string_buffer, bytes_after_last_accept);
+                    // copy the bytes of the incomplete sequence unchanged
+                    put_string(s, s.size() - undumped_chars, s.size());
+                    break;
+                }
+
                 case error_handler_t::replace:
                 {
                     // write all accepted bytes
@@ -27495,12 +27529,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     template < typename CompatibleType,
                typename U = detail::uncvref_t<CompatibleType>,
                detail::enable_if_t <
-                   !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value
-#if JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-                   // see https://github.com/nlohmann/json/issues/2226
-                   && !detail::is_basic_json_reference_tuple<basic_json_t, U>::value
-#endif
-                   , int > = 0 >
+                   !detail::is_basic_json<U>::value && detail::is_compatible_type<basic_json_t, U>::value, int > = 0 >
     basic_json(CompatibleType && val) noexcept(noexcept( // NOLINT(bugprone-forwarding-reference-overload,bugprone-exception-escape)
             JSONSerializer<U>::to_json(std::declval<basic_json_t&>(),
                                        std::forward<CompatibleType>(val))))
@@ -32550,7 +32579,6 @@ struct formatter<nlohmann::NLOHMANN_BASIC_JSON_TPL, char> // NOLINT(cert-dcl58-c
 #undef JSON_INLINE_VARIABLE
 #undef JSON_NO_UNIQUE_ADDRESS
 #undef JSON_DISABLE_ENUM_SERIALIZATION
-#undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
 #undef JSON_USE_GLOBAL_UDLS
 
 #ifndef JSON_TEST_KEEP_MACROS

tests/src/unit-disable-tuple-reference-conversion.cpp

@@ -1,79 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// This file tests the opt-in JSON_DISABLE_TUPLE_REFERENCE_CONVERSION, so it
-// defines the macro itself rather than relying on a -D flag, and runs in every
-// build.
-#ifdef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-    #undef JSON_DISABLE_TUPLE_REFERENCE_CONVERSION
-#endif
-
-#define JSON_DISABLE_TUPLE_REFERENCE_CONVERSION 1
-
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-using nlohmann::ordered_json;
-
-#include <string>
-#include <tuple>
-#include <type_traits>
-#include <utility>
-
-TEST_CASE("JSON_DISABLE_TUPLE_REFERENCE_CONVERSION")
-{
-    SECTION("json is not constructible from a one-element tuple of a json reference")
-    {
-        CHECK_FALSE(std::is_constructible<json, std::tuple<json&>>::value);
-        CHECK_FALSE(std::is_constructible<json, std::tuple<const json&>>::value);
-        CHECK_FALSE(std::is_constructible < json, std::tuple < json && >>::value);
-        CHECK_FALSE(std::is_constructible<json, const std::tuple<json&>&>::value);
-        CHECK_FALSE(std::is_constructible<ordered_json, std::tuple<ordered_json&>>::value);
-    }
-
-    SECTION("issue #2226 - tuple<const json&> from tuple<json&> keeps the reference")
-    {
-        json j = true;
-        const std::tuple<const json&> tup(std::forward_as_tuple(j));
-        CHECK(&std::get<0>(tup) == &j);
-    }
-
-    SECTION("tuple<json> from tuple<json&> copies the element")
-    {
-        const json j = {{"key", "value"}};
-        const std::tuple<json> t1(std::forward_as_tuple(j));
-        CHECK(std::get<0>(t1) == j);
-
-        json j2 = "text";
-        const std::tuple<json> t2(std::forward_as_tuple(std::move(j2)));
-        CHECK(std::get<0>(t2) == "text");
-    }
-
-    SECTION("other tuple conversions are not affected")
-    {
-        const json j = true;
-
-        // one-element tuple holding a json value
-        CHECK(json(std::make_tuple(j)) == json::array({true}));
-
-        // tuples with more than one element, even when holding references
-        int i = 1;
-        CHECK(json(std::forward_as_tuple(i, j)) == json::array({1, true}));
-        CHECK(json(std::forward_as_tuple(j, j)) == json::array({true, true}));
-
-        // one-element tuples holding references to other types
-        std::string s = "text";
-        CHECK(json(std::forward_as_tuple(s)) == json::array({"text"}));
-        CHECK(json(std::forward_as_tuple(i)) == json::array({1}));
-
-        // a reference to a different basic_json specialization
-        ordered_json oj = true;
-        CHECK(json(std::forward_as_tuple(oj)) == json::array({true}));
-    }
-}

tests/src/unit-regression2.cpp

@@ -18,11 +18,6 @@
 // for some reason including this after the json header leads to linker errors with VS 2017...
 #include <locale>
 
-// skip tests if JSON_DISABLE_TUPLE_REFERENCE_CONVERSION=1 (#2226)
-#if defined(JSON_DISABLE_TUPLE_REFERENCE_CONVERSION) && (JSON_DISABLE_TUPLE_REFERENCE_CONVERSION == 1)
-    #define SKIP_TESTS_FOR_TUPLE_REFERENCE_CONVERSION
-#endif
-
 #define JSON_TESTS_PRIVATE
 #include <nlohmann/json.hpp>
 using json = nlohmann::json;
@@ -33,7 +28,6 @@ using ordered_json = nlohmann::ordered_json;
 
 #include <cstdio>
 #include <list>
-#include <tuple>
 #include <type_traits>
 #include <utility>
 
@@ -548,18 +542,6 @@ TEST_CASE("regression tests 2")
                       )));
     }
 
-#ifndef SKIP_TESTS_FOR_TUPLE_REFERENCE_CONVERSION
-    SECTION("issue #2226 - std::tuple dangling reference - implicit conversion")
-    {
-        // by default, a one-element tuple holding a json reference converts to
-        // a one-element array; JSON_DISABLE_TUPLE_REFERENCE_CONVERSION removes
-        // this conversion (see unit-disable-tuple-reference-conversion.cpp)
-        const json j = true;
-        CHECK(std::is_constructible<json, std::tuple<const json&>>::value);
-        CHECK(json(std::forward_as_tuple(j)) == json::array({true}));
-    }
-#endif
-
     SECTION("PR #2181 - regression bug with lvalue")
     {
         // see https://github.com/nlohmann/json/pull/2181#issuecomment-653326060
@@ -784,6 +766,15 @@ TEST_CASE("regression tests 2")
         CHECK(j == k);
     }
 
+    SECTION("issue #4552 - UTF-8 invalid characters are not always ignored when dumping with error_handler_t::ignore")
+    {
+        json node;
+        node["test"] = "test\334\005";
+        CHECK(node.dump(-1, ' ', false, json::error_handler_t::ignore) == "{\"test\":\"test\\u0005\"}");
+        CHECK(node.dump(-1, ' ', false, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
+        CHECK(node.dump(-1, ' ', true, json::error_handler_t::keep) == "{\"test\":\"test\334\\u0005\"}");
+    }
+
 }
 
 TEST_CASE("regression test - parser callback must not lose a duplicate key's prior value")

tests/src/unit-serialization.cpp

@@ -92,6 +92,8 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"äü\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"ä\xEF\xBF\xBDü\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"ä\xA9ü\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\\u00e4\xA9\\u00fc\"");
         }
 
         SECTION("invalid character (regression guard for shared UTF-8 decoder, see #5529)")
@@ -114,6 +116,8 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xC2\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xC2\"");
         }
 
         SECTION("unexpected character")
@@ -126,6 +130,39 @@ TEST_CASE("serialization")
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"123456\"");
             CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"123\xEF\xBF\xBD\x34\x35\x36\"");
             CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"123\\ufffd456\"");
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"123\xF1\xB0\x34\x35\x36\"");
+        }
+
+        SECTION("keep: valid characters are still escaped")
+        {
+            // an invalid byte followed by characters that must be escaped
+            const json j = "\xC2\"\\\n\xFF\x05";
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
+            CHECK(j.dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xC2\\\"\\\\\\n\xFF\\u0005\"");
+        }
+
+        SECTION("keep: truncated multibyte sequences")
+        {
+            CHECK(json("\xF0\x9F\x98").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
+            CHECK(json("\xF0\x9F\x98").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98\"");
+            CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', false, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
+            CHECK(json("\xF0\x9F\x98" "a").dump(-1, ' ', true, json::error_handler_t::keep) == "\"\xF0\x9F\x98" "a\"");
+        }
+
+        SECTION("keep: long string with many invalid bytes")
+        {
+            // exceeds the internal string buffer several times
+            std::string input;
+            std::string expected = "\"";
+            for (int i = 0; i < 2000; ++i)
+            {
+                input += "\xFF\xE2\x82\n\xC3\xA4";
+                expected += "\xFF\xE2\x82\\n\xC3\xA4";
+            }
+            expected += "\"";
+            const json j = input;
+            CHECK(j.dump(-1, ' ', false, json::error_handler_t::keep) == expected);
         }
 
         SECTION("U+FFFD Substitution of Maximal Subparts")

tests/src/unit-unicode2.cpp

@@ -14,6 +14,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 #include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     static std::string s_replaced2;
     static std::string s_replaced_ascii;
     static std::string s_replaced2_ascii;
+    static std::string s_kept;
+    static std::string s_kept2;
+    static std::string s_kept_ascii;
 
-    // dumping with ignore/replace must not throw in any case
+    // dumping with ignore/replace/keep must not throw in any case
     s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
     s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
     s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
+    s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
 
     if (success_expected)
     {
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
         // all dumps should agree on the string
         CHECK(s_strict == s_ignored);
         CHECK(s_strict == s_replaced);
+        CHECK(s_strict == s_kept);
     }
     else
     {
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
 
         // check that replace string contains a replacement character
         CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
+
+        // ignore drops the invalid bytes, keep copies them
+        CHECK(s_ignored != s_kept);
+        CHECK(s_ignored_ascii != s_kept_ascii);
+
+        // unless a byte needs escaping, keep copies the input unchanged
+        const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
+        });
+        if (!needs_escaping)
+        {
+            CHECK(s_kept == "\"" + json_string + "\"");
+        }
     }
 
     // check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
     CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
     CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
+    CHECK(s_kept2.substr(1, 3) == "abc");
+    CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
 }
 
 void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);

tests/src/unit-unicode3.cpp

@@ -14,6 +14,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 #include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     static std::string s_replaced2;
     static std::string s_replaced_ascii;
     static std::string s_replaced2_ascii;
+    static std::string s_kept;
+    static std::string s_kept2;
+    static std::string s_kept_ascii;
 
-    // dumping with ignore/replace must not throw in any case
+    // dumping with ignore/replace/keep must not throw in any case
     s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
     s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
     s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
+    s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
 
     if (success_expected)
     {
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
         // all dumps should agree on the string
         CHECK(s_strict == s_ignored);
         CHECK(s_strict == s_replaced);
+        CHECK(s_strict == s_kept);
     }
     else
     {
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
 
         // check that replace string contains a replacement character
         CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
+
+        // ignore drops the invalid bytes, keep copies them
+        CHECK(s_ignored != s_kept);
+        CHECK(s_ignored_ascii != s_kept_ascii);
+
+        // unless a byte needs escaping, keep copies the input unchanged
+        const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
+        });
+        if (!needs_escaping)
+        {
+            CHECK(s_kept == "\"" + json_string + "\"");
+        }
     }
 
     // check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
     CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
     CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
+    CHECK(s_kept2.substr(1, 3) == "abc");
+    CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
 }
 
 void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);

tests/src/unit-unicode4.cpp

@@ -14,6 +14,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 #include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     static std::string s_replaced2;
     static std::string s_replaced_ascii;
     static std::string s_replaced2_ascii;
+    static std::string s_kept;
+    static std::string s_kept2;
+    static std::string s_kept_ascii;
 
-    // dumping with ignore/replace must not throw in any case
+    // dumping with ignore/replace/keep must not throw in any case
     s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
     s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
     s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
+    s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
 
     if (success_expected)
     {
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
         // all dumps should agree on the string
         CHECK(s_strict == s_ignored);
         CHECK(s_strict == s_replaced);
+        CHECK(s_strict == s_kept);
     }
     else
     {
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
 
         // check that replace string contains a replacement character
         CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
+
+        // ignore drops the invalid bytes, keep copies them
+        CHECK(s_ignored != s_kept);
+        CHECK(s_ignored_ascii != s_kept_ascii);
+
+        // unless a byte needs escaping, keep copies the input unchanged
+        const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
+        });
+        if (!needs_escaping)
+        {
+            CHECK(s_kept == "\"" + json_string + "\"");
+        }
     }
 
     // check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
     CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
     CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
+    CHECK(s_kept2.substr(1, 3) == "abc");
+    CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
 }
 
 void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);

tests/src/unit-unicode5.cpp

@@ -14,6 +14,7 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <algorithm>
 #include <fstream>
 #include <sstream>
 #include <iostream>
@@ -75,8 +76,11 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     static std::string s_replaced2;
     static std::string s_replaced_ascii;
     static std::string s_replaced2_ascii;
+    static std::string s_kept;
+    static std::string s_kept2;
+    static std::string s_kept_ascii;
 
-    // dumping with ignore/replace must not throw in any case
+    // dumping with ignore/replace/keep must not throw in any case
     s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
     s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
@@ -85,6 +89,9 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
     s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
     s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
+    s_kept = j.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept2 = j2.dump(-1, ' ', false, json::error_handler_t::keep);
+    s_kept_ascii = j.dump(-1, ' ', true, json::error_handler_t::keep);
 
     if (success_expected)
     {
@@ -94,6 +101,7 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
         // all dumps should agree on the string
         CHECK(s_strict == s_ignored);
         CHECK(s_strict == s_replaced);
+        CHECK(s_strict == s_kept);
     }
     else
     {
@@ -105,6 +113,20 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
 
         // check that replace string contains a replacement character
         CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
+
+        // ignore drops the invalid bytes, keep copies them
+        CHECK(s_ignored != s_kept);
+        CHECK(s_ignored_ascii != s_kept_ascii);
+
+        // unless a byte needs escaping, keep copies the input unchanged
+        const bool needs_escaping = std::any_of(json_string.begin(), json_string.end(), [](char c)
+        {
+            return static_cast<unsigned char>(c) < 0x20 || c == '"' || c == '\\';
+        });
+        if (!needs_escaping)
+        {
+            CHECK(s_kept == "\"" + json_string + "\"");
+        }
     }
 
     // check that prefix and suffix are preserved
@@ -116,6 +138,8 @@ void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3
     CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
     CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
     CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
+    CHECK(s_kept2.substr(1, 3) == "abc");
+    CHECK(s_kept2.substr(s_kept2.size() - 4, 3) == "xyz");
 }
 
 void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);