Merge remote-tracking branch 'origin/develop' into claude/fix-issue-3989-db7e45

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

# Conflicts:
#	include/nlohmann/detail/input/binary_reader.hpp
#	include/nlohmann/detail/string_utils.hpp
#	single_include/nlohmann/json.hpp

.clang-tidy

@@ -1,18 +1,9 @@
-# bugprone-use-after-move (hicpp-invalid-access-moved is its alias) still flags
-# the basic_json move constructor, which forwards the whole object to its base
-# class (#5724), and two forwards in the error-message construction of
-# at(KeyType&&) (json.hpp, both overloads: find(std::forward<KeyType>(key))
-# followed by string_t(std::forward<KeyType>(key)) in the throw), which #5689
-# rewrites. Re-enable both checks once those changes have landed.
 # portability-avoid-pragma-once: kept disabled on purpose. #pragma once is accepted
 # by every supported compiler, and tools/amalgamate/amalgamate.py strips it from
 # single_include, so there is nothing left to fix here.
 
 Checks: '*,
 
-         -bugprone-use-after-move,
-         -hicpp-invalid-access-moved,
-
          -altera-id-dependent-backward-branch,
          -altera-struct-pack-align,
          -altera-unroll-loops,

BUILD.bazel

@@ -53,11 +53,11 @@ cc_library(
         "include/nlohmann/detail/meta/detected.hpp",
         "include/nlohmann/detail/meta/identity_tag.hpp",
         "include/nlohmann/detail/meta/is_sax.hpp",
-        "include/nlohmann/detail/meta/logic.hpp",
         "include/nlohmann/detail/meta/std_fs.hpp",
         "include/nlohmann/detail/meta/type_traits.hpp",
         "include/nlohmann/detail/meta/void_t.hpp",
         "include/nlohmann/detail/output/binary_writer.hpp",
+        "include/nlohmann/detail/output/error_handler.hpp",
         "include/nlohmann/detail/output/output_adapters.hpp",
         "include/nlohmann/detail/output/serializer.hpp",
         "include/nlohmann/detail/recursion_depth_limit.hpp",

CMakeLists.txt

@@ -61,6 +61,7 @@ option(JSON_Install                        "Install CMake targets during install
 option(JSON_MultipleHeaders                "Use non-amalgamated version of the library." ON)
 option(JSON_SystemInclude                  "Include as system headers (skip for clang-tidy)." OFF)
 option(JSON_StrictNulHandling              "Build with strict NUL-byte handling enabled." OFF)
+option(JSON_StrictBinaryUTF8               "Build with UTF-8 checks in the CBOR, UBJSON, BJData, and BSON writers enabled." OFF)
 
 if (JSON_CI)
     include(ci)
@@ -118,6 +119,10 @@ if (JSON_StrictNulHandling)
     message(STATUS "Strict NUL-byte handling enabled (JSON_STRICT_NUL_HANDLING=1)")
 endif()
 
+if (JSON_StrictBinaryUTF8)
+    message(STATUS "Strict UTF-8 checks in binary writers enabled (JSON_STRICT_BINARY_UTF8=1)")
+endif()
+
 if (JSON_Diagnostic_Positions)
     message(STATUS "Diagnostic positions enabled (JSON_DIAGNOSTIC_POSITIONS=1)")
 endif()
@@ -153,6 +158,7 @@ target_compile_definitions(
     $<$<BOOL:${JSON_Diagnostic_Positions}>:JSON_DIAGNOSTIC_POSITIONS=1>
     $<$<BOOL:${JSON_LegacyDiscardedValueComparison}>:JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON=1>
     $<$<BOOL:${JSON_StrictNulHandling}>:JSON_STRICT_NUL_HANDLING=1>
+    $<$<BOOL:${JSON_StrictBinaryUTF8}>:JSON_STRICT_BINARY_UTF8=1>
 )
 
 target_include_directories(

README.md

@@ -494,7 +494,7 @@ bool key(string_t& val);
 bool parse_error(std::size_t position, const std::string& last_token, const detail::exception& ex);
 ```
 
-The return value of each function determines whether parsing should proceed.
+The return value of each function determines whether parsing should proceed. For `parse_error`, returning `true` [recovers from the error](https://json.nlohmann.me/features/parsing/error_recovery/): the parser repairs the input and continues.
 
 To implement your own SAX handler, proceed as follows:
 
@@ -502,7 +502,7 @@ To implement your own SAX handler, proceed as follows:
 2. Create an object of your SAX interface class, e.g. `my_sax`.
 3. Call `bool json::sax_parse(input, &my_sax)`; where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface.
 
-Note the `sax_parse` function only returns a `bool` indicating the result of the last executed SAX event. It does not return a  `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp).
+Note the `sax_parse` function only returns a `bool` indicating whether the input was parsed without errors and no SAX event returned `false`. It does not return a  `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp).
 
 ### STL-like access
 

cmake/ci.cmake

@@ -701,7 +701,7 @@ ci_get_cmake(4.0.0  CMAKE_4_0_0_BINARY)
 # the tests require CMake 3.13 or later, so they are excluded for CMake 3.5.0
 set(JSON_CMAKE_FLAGS_3_5_0 JSON_Diagnostics JSON_Diagnostic_Positions JSON_GlobalUDLs JSON_ImplicitConversions JSON_DisableEnumSerialization
     JSON_LegacyDiscardedValueComparison JSON_Install JSON_MultipleHeaders JSON_SystemInclude JSON_Valgrind
-    JSON_StrictNulHandling)
+    JSON_StrictNulHandling JSON_StrictBinaryUTF8)
 set(JSON_CMAKE_FLAGS_3_31_6 JSON_BuildTests ${JSON_CMAKE_FLAGS_3_5_0})
 set(JSON_CMAKE_FLAGS_4_0_0 JSON_BuildTests ${JSON_CMAKE_FLAGS_3_5_0})
 

docs/docset/docSet.sql

@@ -19,6 +19,7 @@ INSERT INTO searchIndex(name, type, path) VALUES ('format_as', 'Function', 'api/
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::accept', 'Function', 'api/basic_json/accept/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::array', 'Function', 'api/basic_json/array/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::array_t', 'Type', 'api/basic_json/array_t/index.html');
+INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::as_base_class', 'Method', 'api/basic_json/as_base_class/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::at', 'Method', 'api/basic_json/at/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::back', 'Method', 'api/basic_json/back/index.html');
 INSERT INTO searchIndex(name, type, path) VALUES ('basic_json::basic_json', 'Constructor', 'api/basic_json/basic_json/index.html');

docs/mkdocs/docs/api/basic_json/as_base_class.md

@@ -0,0 +1,53 @@
+# <small>nlohmann::basic_json::</small>as_base_class
+
+```cpp
+json_base_class_t& as_base_class() noexcept;
+const json_base_class_t& as_base_class() const noexcept;
+```
+
+Returns a reference to this object as its custom base class [`json_base_class_t`](json_base_class_t.md). No copy is
+made.
+
+Since `basic_json` derives from `json_base_class_t`, a member of `basic_json` hides any member of the custom base class
+with the same name. This function makes such hidden members accessible again.
+
+## Return value
+
+reference to this object as [`json_base_class_t`](json_base_class_t.md)
+
+## Exception safety
+
+No-throw guarantee: this function never throws exceptions.
+
+## Complexity
+
+Constant.
+
+## Notes
+
+The function is equivalent to `static_cast<json_base_class_t&>(j)` (or `static_cast<const json_base_class_t&>(j)`).
+
+## Examples
+
+??? example
+
+    The example shows how to use `as_base_class` to access members of the custom base class that are hidden by members
+    of `basic_json`.
+    
+    ```cpp
+    --8<-- "examples/as_base_class.cpp"
+    ```
+    
+    Output:
+    
+    ```json
+    --8<-- "examples/as_base_class.output"
+    ```
+
+## See also
+
+- [json_base_class_t](json_base_class_t.md) - type of the custom base class
+
+## Version history
+
+- Added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/at.md

@@ -238,5 +238,7 @@ Strong exception safety: if an exception occurs, the original value stays intact
 
 1. Added in version 1.0.0.
 2. Added in version 1.0.0.
-3. Added in version 3.11.0.
+3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
+   already supported by [`operator[]`](operator[].md), [`value`](value.md), [`find`](find.md), and other lookup
+   functions.
 4. Added in version 2.0.0.

docs/mkdocs/docs/api/basic_json/basic_json.md

@@ -293,6 +293,15 @@ basic_json(basic_json&& other) noexcept;
         When used without parentheses around an empty initializer list, `basic_json()` is called instead of this
         function, yielding the JSON `#!json null` value.
 
+- Overload 4:
+
+    !!! info "Implicit conversion"
+
+        The conversion is implicit unless [`JSON_USE_IMPLICIT_CONVERSIONS`](../macros/json_use_implicit_conversions.md)
+        is defined to `0` and `BasicJsonType::string_t` differs from `string_t`. In that case, the constructor is
+        `explicit`, so a JSON value with a different string type is no longer silently converted, for example when it is
+        passed to a function taking `#!cpp const json&`. Write `#!cpp json(other)` or `#!cpp other.get<json>()` instead.
+
 - Overload 7:
 
     !!! info "Preconditions"
@@ -466,7 +475,8 @@ basic_json(basic_json&& other) noexcept;
 1. Since version 1.0.0.
 2. Since version 1.0.0.
 3. Since version 2.1.0.
-4. Since version 3.2.0.
+4. Since version 3.2.0. Explicit for different string types if `JSON_USE_IMPLICIT_CONVERSIONS` is `0` since
+   version 3.13.0.
 5. Since version 1.0.0.
 6. Since version 1.0.0.
 7. Since version 1.0.0. Fixed in version 3.13.0 to also check the iterator range for binary values; before, a range

docs/mkdocs/docs/api/basic_json/contains.md

@@ -131,7 +131,9 @@ Logarithmic in the size of the JSON object.
 ## Version history
 
 1. Added in version 3.11.0.
-2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0.
+2. Added in version 3.6.0. Extended template `KeyType` to support comparable types in version 3.11.0. Fixed in
+   version 3.13.0 to consistently accept `std::string_view`-convertible keys, as already supported by
+   [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup functions.
 3. Added in version 3.7.0.
 4. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of
    causing undefined behavior at runtime.

docs/mkdocs/docs/api/basic_json/count.md

@@ -84,6 +84,8 @@ Logarithmic in the size of the JSON object.
 ## Version history
 
 1. Added in version 3.11.0.
-2. Added in version 1.0.0. Changed parameter `key` type to `KeyType&&` in version 3.11.0.
+2. Added in version 1.0.0. Changed parameter `key` type to `KeyType&&` in version 3.11.0. Fixed in version 3.13.0 to
+   consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md),
+   [`at`](at.md), [`value`](value.md), and other lookup functions.
 3. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time instead of
    causing undefined behavior at runtime.

docs/mkdocs/docs/api/basic_json/dump.md

@@ -25,10 +25,12 @@ and `ensure_ascii` parameters.
     result consists of ASCII characters only.
 
 `error_handler` (in)
-:   how to react on decoding errors; there are three possible values (see [`error_handler_t`](error_handler_t.md):
+:   how to react on decoding errors; there are four possible values (see [`error_handler_t`](error_handler_t.md):
     `strict` (throws an exception in case a decoding error occurs; default), `replace` (replace invalid UTF-8 sequences
-    with U+FFFD), and `ignore` (ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the
+    with U+FFFD), `ignore` (ignore invalid UTF-8 sequences during serialization; all valid bytes are copied to the
-    output unchanged, and invalid bytes are dropped)).
+    output unchanged, and invalid bytes are dropped), and `keep` (write the ill-formed bytes to the output as is,
+    without escaping them, even if `ensure_ascii` is `#!cpp true`; the result is then not valid UTF-8, but equals the
+    input bytes exactly, and well-formed characters around the ill-formed bytes are still escaped as usual)).
     
 ## Return value
 
@@ -94,3 +96,4 @@ Binary values are serialized as an object containing two keys:
 - Indentation character `indent_char`, option `ensure_ascii` and exceptions added in version 3.0.0.
 - Error handlers added in version 3.4.0.
 - Serialization of binary values added in version 3.8.0.
+- Error handler `keep` added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/emplace.md

@@ -70,3 +70,5 @@ Logarithmic in the size of the container, O(log(`size()`)).
 ## Version history
 
 - Since version 2.0.8.
+- Fixed in version 3.13.0: for [`ordered_json`](../ordered_json.md), the value could previously only be passed as an
+  rvalue; it can now also be passed as an lvalue or a `#!cpp const` lvalue, matching the behavior of `json`.

docs/mkdocs/docs/api/basic_json/erase.md

@@ -213,5 +213,7 @@ Strong exception safety: if an exception occurs, the original value stays intact
 1. Added in version 1.0.0. Added support for binary types in version 3.8.0.
 2. Added in version 1.0.0. Added support for binary types in version 3.8.0.
 3. Added in version 1.0.0.
-4. Added in version 3.11.0.
+4. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
+   already supported by [`operator[]`](operator[].md), [`at`](at.md), [`value`](value.md), and other lookup
+   functions.
 5. Added in version 1.0.0.

docs/mkdocs/docs/api/basic_json/error_handler_t.md

@@ -4,15 +4,31 @@
 enum class error_handler_t {
     strict,
     replace,
-    ignore
+    ignore,
+    keep
 };
 ```
 
-This enumeration is used in the [`dump`](dump.md) function to choose how to treat decoding errors while serializing a
+This enumeration is used to choose how to treat ill-formed UTF-8 in a string value or object key:
-`basic_json` value. Three values are differentiated:
+
+- [`dump`](dump.md) uses it while serializing a `basic_json` value to text.
+- [`to_cbor`](to_cbor.md), [`to_msgpack`](to_msgpack.md), [`to_ubjson`](to_ubjson.md), [`to_bjdata`](to_bjdata.md),
+  and [`to_bson`](to_bson.md) use it while serializing a `basic_json` value to that binary format. Their default is
+  `keep`, as no binary writer checked before this parameter was added. CBOR, UBJSON, BJData, and BSON require valid
+  UTF-8, so for these four the default is `strict` if [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md)
+  is enabled; MessagePack's specification explicitly allows a string to contain ill-formed UTF-8, so `to_msgpack`
+  stays at `keep`. `to_bon8` does not take this parameter: BON8 always validates, since UTF-8 lead bytes are
+  structural to that format.
+- [`from_cbor`](from_cbor.md), [`from_msgpack`](from_msgpack.md), [`from_ubjson`](from_ubjson.md),
+  [`from_bjdata`](from_bjdata.md), and [`from_bson`](from_bson.md) use it while parsing that binary format, to decide
+  whether to check a string value or object key for well-formed UTF-8 at all; by default (`keep`) they do not, as no
+  binary reader did before this parameter was added. `from_bon8` does not take this parameter, for the same reason
+  `to_bon8` does not.
+
+Four values are differentiated:
 
 strict
-:   throw a `type_error` exception in case of invalid UTF-8
+:   throw a `type_error`/`parse_error` exception in case of invalid UTF-8
 
 replace
 :   replace invalid UTF-8 sequences with U+FFFD (� REPLACEMENT CHARACTER)
@@ -20,6 +36,12 @@ replace
 ignore
 :   ignore invalid UTF-8 sequences; all valid bytes are copied to the output unchanged, and invalid bytes are dropped
 
+keep
+:   keep invalid UTF-8 sequences unchanged; only meaningful for the binary formats mentioned above, since [`dump`]
+    (dump.md) itself must produce text, and `keep` there writes the ill-formed bytes to the output as is, so the
+    result is then not valid UTF-8 (but still equals the input bytes exactly, including around any well-formed
+    characters, which are still escaped as usual)
+
 ## Examples
 
 ??? example
@@ -45,3 +67,5 @@ ignore
 ## Version history
 
 - Added in version 3.4.0.
+- Added `keep`, and made this enumeration apply to the binary readers and writers in addition to `dump`, in version
+  3.13.0.

docs/mkdocs/docs/api/basic_json/find.md

@@ -88,6 +88,8 @@ Logarithmic in the size of the JSON object.
 ## Version history
 
 1. Added in version 3.11.0.
-2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0.
+2. Added in version 1.0.0. Changed to support comparable types in version 3.11.0. Fixed in version 3.13.0 to
+   consistently accept `std::string_view`-convertible keys, as already supported by [`operator[]`](operator[].md),
+   [`at`](at.md), [`value`](value.md), and other lookup functions.
 3. Deleted overloads for integral key types added in version 3.13.0 to reject such calls at compile time instead of
    causing undefined behavior at runtime.

docs/mkdocs/docs/api/basic_json/from_bjdata.md

@@ -5,12 +5,14 @@
 template<typename InputType>
 static basic_json from_bjdata(InputType&& i,
                               const bool strict = true,
-                              const bool allow_exceptions = true);
+                              const bool allow_exceptions = true,
+                              const error_handler_t error_handler = error_handler_t::keep);
 // (2)
 template<typename IteratorType, typename SentinelType = IteratorType>
 static basic_json from_bjdata(IteratorType first, SentinelType last,
                               const bool strict = true,
-                              const bool allow_exceptions = true);
+                              const bool allow_exceptions = true,
+                              const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Deserializes a given input to a JSON value using the BJData (Binary JData) serialization format.
@@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `allow_exceptions` (in)
 :   whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default)
 
+`error_handler` (in)
+:   how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    BJData does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not
+    check at all, as every binary reader did before this parameter was added; `strict` checks and throws;
+    `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would
+
 ## Return value
 
 deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be
@@ -73,7 +81,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
   the end of the file was not reached when `strict` was set to true
 - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if a parse error occurs
 - Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a string could not be parsed
-  successfully
+  successfully, or if a string value or object key is not valid UTF-8 and `error_handler` is `strict`
 - Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the size of an optimized container
   or n-dimensional array cannot be represented by `std::size_t`
 
@@ -111,3 +119,4 @@ Linear in the size of the input.
 - Added in version 3.11.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0.

docs/mkdocs/docs/api/basic_json/from_bson.md

@@ -5,12 +5,14 @@
 template<typename InputType>
 static basic_json from_bson(InputType&& i,
                             const bool strict = true,
-                            const bool allow_exceptions = true);
+                            const bool allow_exceptions = true,
+                            const error_handler_t error_handler = error_handler_t::keep);
 // (2)
 template<typename IteratorType, typename SentinelType = IteratorType>
 static basic_json from_bson(IteratorType first, SentinelType last,
                             const bool strict = true,
-                            const bool allow_exceptions = true);
+                            const bool allow_exceptions = true,
+                            const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Deserializes a given input to a JSON value using the BSON (Binary JSON) serialization format.
@@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `allow_exceptions` (in)
 :   whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default)
 
+`error_handler` (in)
+:   how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    BSON does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not
+    check at all, as every binary reader did before this parameter was added; `strict` checks and throws;
+    `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would
+
 ## Return value
 
 deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be
@@ -75,6 +83,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
   invalid string or byte array length)
 - Throws [`parse_error.114`](../../home/exceptions.md#jsonexceptionparse_error114) if an unsupported BSON record type is
   encountered
+- Throws [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) if a string value or object key is
+  not valid UTF-8 and `error_handler` is `strict`
 
 ## Complexity
 
@@ -111,6 +121,7 @@ Linear in the size of the input.
 - Added in version 3.4.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0.
 
 !!! warning "Deprecation"
 

docs/mkdocs/docs/api/basic_json/from_cbor.md

@@ -6,14 +6,16 @@ template<typename InputType>
 static basic_json from_cbor(InputType&& i,
                             const bool strict = true,
                             const bool allow_exceptions = true,
-                            const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error);
+                            const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error,
+                            const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
 template<typename IteratorType, typename SentinelType = IteratorType>
 static basic_json from_cbor(IteratorType first, SentinelType last,
                             const bool strict = true,
                             const bool allow_exceptions = true,
-                            const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error);
+                            const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error,
+                            const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Deserializes a given input to a JSON value using the CBOR (Concise Binary Object Representation) serialization format.
@@ -65,6 +67,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 :   how to treat CBOR tags (optional, `error` by default); see [`cbor_tag_handler_t`](cbor_tag_handler_t.md) for more
     information
 
+`error_handler` (in)
+:   how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    CBOR does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not
+    check at all, as every binary reader did before this parameter was added; `strict` checks and throws;
+    `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would
+
 ## Return value
 
 deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be
@@ -80,8 +88,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
   the end of the file was not reached when `strict` was set to true
 - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if unsupported features from CBOR were
   used in the given input or if the input is not valid CBOR
-- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of other
+- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of
-  types are not supported, as JSON object keys are always strings) or a string is malformed
+  other types are not supported, as JSON object keys are always strings), or if a string value or object key is not
+  valid UTF-8 and `error_handler` is `strict`
 
 ## Complexity
 
@@ -121,6 +130,7 @@ Linear in the size of the input.
 - Added `tag_handler` parameter in version 3.9.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0.
 
 !!! warning "Deprecation"
 

docs/mkdocs/docs/api/basic_json/from_msgpack.md

@@ -5,12 +5,14 @@
 template<typename InputType>
 static basic_json from_msgpack(InputType&& i,
                                const bool strict = true,
-                               const bool allow_exceptions = true);
+                               const bool allow_exceptions = true,
+                               const error_handler_t error_handler = error_handler_t::keep);
 // (2)
 template<typename IteratorType, typename SentinelType = IteratorType>
 static basic_json from_msgpack(IteratorType first, SentinelType last,
                                const bool strict = true,
-                               const bool allow_exceptions = true);
+                               const bool allow_exceptions = true,
+                               const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Deserializes a given input to a JSON value using the MessagePack serialization format.
@@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `allow_exceptions` (in)
 :   whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default)
 
+`error_handler` (in)
+:   how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    MessagePack's specification explicitly allows ill-formed UTF-8, so checking is opt-in: the default, `keep`, does
+    not check at all, as every binary reader did before this parameter was added; `strict` checks and throws;
+    `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would
+
 ## Return value
 
 deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be
@@ -73,8 +81,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
   the end of the file was not reached when `strict` was set to true
 - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if unsupported features from
   MessagePack were used in the given input or if the input is not valid MessagePack
-- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of other
+- Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a map key is not a string (keys of
-  types are not supported, as JSON object keys are always strings) or a string is malformed
+  other types are not supported, as JSON object keys are always strings), or if a string value or object key is not
+  valid UTF-8 and `error_handler` is `strict`
 
 ## Complexity
 
@@ -113,6 +122,7 @@ Linear in the size of the input.
 - Added `allow_exceptions` parameter in version 3.2.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0.
 
 !!! warning "Deprecation"
 

docs/mkdocs/docs/api/basic_json/from_ubjson.md

@@ -5,12 +5,14 @@
 template<typename InputType>
 static basic_json from_ubjson(InputType&& i,
                               const bool strict = true,
-                              const bool allow_exceptions = true);
+                              const bool allow_exceptions = true,
+                              const error_handler_t error_handler = error_handler_t::keep);
 // (2)
 template<typename IteratorType, typename SentinelType = IteratorType>
 static basic_json from_ubjson(IteratorType first, SentinelType last,
                               const bool strict = true,
-                              const bool allow_exceptions = true);
+                              const bool allow_exceptions = true,
+                              const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Deserializes a given input to a JSON value using the UBJSON (Universal Binary JSON) serialization format.
@@ -58,6 +60,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `allow_exceptions` (in)
 :   whether to throw exceptions in case of a parse error (optional, `#!cpp true` by default)
 
+`error_handler` (in)
+:   how to treat a string value or object key that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    UBJSON does not require a decoder to reject ill-formed UTF-8, so checking is opt-in: the default, `keep`, does not
+    check at all, as every binary reader did before this parameter was added; `strict` checks and throws;
+    `replace`/`ignore` sanitize the string the same way [`dump`](dump.md) would
+
 ## Return value
 
 deserialized JSON value; in case of a parse error and `allow_exceptions` set to `#!cpp false`, the return value will be
@@ -73,7 +81,7 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
   the end of the file was not reached when `strict` was set to true
 - Throws [parse_error.112](../../home/exceptions.md#jsonexceptionparse_error112) if a parse error occurs
 - Throws [parse_error.113](../../home/exceptions.md#jsonexceptionparse_error113) if a string could not be parsed
-  successfully
+  successfully, or if a string value or object key is not valid UTF-8 and `error_handler` is `strict`
 - Throws [out_of_range.408](../../home/exceptions.md#jsonexceptionout_of_range408) if the size of an optimized container
   or n-dimensional array cannot be represented by `std::size_t`
 
@@ -112,6 +120,7 @@ Linear in the size of the input.
 - Added `allow_exceptions` parameter in version 3.2.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0.
 
 !!! warning "Deprecation"
 

docs/mkdocs/docs/api/basic_json/index.md

@@ -200,6 +200,7 @@ Direct access to the stored value of a JSON value.
 - [**get_ref**](get_ref.md) - get a reference value
 - [**operator ValueType**](operator_ValueType.md) - get a value
 - [**get_binary**](get_binary.md) - get a binary value
+- [**as_base_class**](as_base_class.md) - access the custom base class
 
 ### Element access
 

docs/mkdocs/docs/api/basic_json/json_base_class_t.md

@@ -27,6 +27,18 @@ A `CustomBaseClass` with non-static data members forfeits `basic_json`'s
 [standard layout](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType) guarantee. See
 [Template Parameter Requirements](../../features/types/template_parameters.md#custombaseclass).
 
+#### Name conflicts
+
+Since `basic_json` derives from `CustomBaseClass`, members of `basic_json` hide members of `CustomBaseClass` with the
+same name. Hidden members remain accessible via [`as_base_class`](as_base_class.md) or by casting the value to
+`json_base_class_t`.
+
+!!! warning "Avoid generic member names"
+
+    Future versions of the library may add members to `basic_json` that hide members of `CustomBaseClass` that are
+    accessible today. To reduce the risk of such conflicts, avoid generic names for the members of `CustomBaseClass`,
+    for instance by using a distinctive prefix.
+
 ## Examples
 
 ??? example
@@ -45,8 +57,10 @@ A `CustomBaseClass` with non-static data members forfeits `basic_json`'s
 
 ## See also
 
+- [as_base_class](as_base_class.md) - access the custom base class
 - [Template Parameter Requirements](../../features/types/template_parameters.md#custombaseclass) - the requirements for `CustomBaseClass`
 
 ## Version history
 
 - Added in version 3.12.0.
+- Made a public member type in version 3.13.0; it was private before, so it could not be named outside the class.

docs/mkdocs/docs/api/basic_json/meta.md

@@ -13,7 +13,7 @@ JSON object holding version information
 
 | key         | description                                                                                                                                                                                                                                                                    |
 |-------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). On HP aCC compilers, `compiler` is instead the plain string `hp`. |
+| `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `hp`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). |
 | `copyright` | The copyright line for the library as string.                                                                                                                                                                                                                                  |
 | `name`      | The name of the library as string.                                                                                                                                                                                                                                             |
 | `platform`  | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`.                                                                                                                                                                             |

docs/mkdocs/docs/api/basic_json/operator[].md

@@ -89,6 +89,9 @@ Strong exception safety: if an exception occurs, the original value stays intact
     - Throws [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) if an array index in the passed
       JSON pointer `ptr` exceeds the range of `size_type` (e.g., on 32-bit platforms).
 
+    For the **const** version, an object key or array index in `ptr` that does not exist is not reported by an
+    exception, but is undefined behavior (see the notes below). Use [`at`](at.md) for checked access.
+
 ## Complexity
 
 1. Constant if `idx` is in the range of the array. Otherwise, linear in `idx - size()`.
@@ -103,9 +106,12 @@ Strong exception safety: if an exception occurs, the original value stays intact
     The following cases apply to the **const** overloads; the non-const overloads instead insert the missing element
     (see the notes below).
 
-    1. If the element at index `idx` does not exist, the behavior is undefined.
+    1. If the element at index `idx` does not exist, the behavior is undefined and is **guarded by a
+       [runtime assertion](../../features/assertions.md)**!
     2. If the element with key `key` does not exist, the behavior is undefined and is **guarded by a
        [runtime assertion](../../features/assertions.md)**!
+    3. If the JSON pointer `ptr` refers to an object key or an array index that does not exist, the behavior is
+       undefined and is **guarded by a [runtime assertion](../../features/assertions.md)**!
 
 1. The non-const version may add values: If `idx` is beyond the range of the array (i.e., `idx >= size()`), then the
    array is silently filled up with `#!json null` values to make `idx` a valid reference to the last stored element. In
@@ -273,9 +279,11 @@ Strong exception safety: if an exception occurs, the original value stays intact
 ## Version history
 
 1. Added in version 1.0.0. Fixed in version 3.13.0 to throw `#!cpp std::length_error` instead of emptying the array and
-   accessing it out of bounds when `idx` equals the maximum value of `size_type`.
+   accessing it out of bounds when `idx` equals the maximum value of `size_type`. A missing index in the const version
+   is guarded by a runtime assertion since version 3.13.0.
 2. Added in version 1.0.0. Added overloads for `T* key` in version 1.1.0. Removed overloads for `T* key` (replaced by 3)
    in version 3.11.0.
 3. Added in version 3.11.0. Fixed in version 3.13.0 to consistently accept `std::string_view`-convertible keys, as
    already supported by [`at`](at.md), [`value`](value.md), [`find`](find.md), and other lookup functions.
-4. Added in version 2.0.0.
+4. Added in version 2.0.0. A missing array index in the const version is guarded by a runtime assertion since
+   version 3.13.0.

docs/mkdocs/docs/api/basic_json/operator_eq.md

@@ -5,17 +5,17 @@
 bool operator==(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator==(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator==(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator==(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator==(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 
 // since C++20
 class basic_json {
     bool operator==(const_reference rhs) const noexcept;              // (1)
 
     template<typename ScalarType>
-    bool operator==(ScalarType rhs) const noexcept;                   // (2)
+    bool operator==(ScalarType rhs) const noexcept(/* see below */); // (2)
 };
 ```
 
@@ -46,7 +46,12 @@ whether the values `lhs`/`*this` and `rhs` are equal
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -171,3 +176,4 @@ Linear.
 
 1. Added in version 1.0.0. Added C++20 member functions in version 3.11.0.
 2. Added in version 1.0.0. Added C++20 member functions in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_ge.md

@@ -5,10 +5,10 @@
 bool operator>=(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator>=(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator>=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator>=(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator>=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 ```
 
 1. Compares whether one JSON value `lhs` is greater than or equal to another JSON value `rhs` according to the following
@@ -39,7 +39,12 @@ whether `lhs` is greater than or equal to `rhs`
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -94,3 +99,4 @@ Linear.
 
 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_gt.md

@@ -5,10 +5,10 @@
 bool operator>(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator>(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator>(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator>(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator>(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 ```
 
 1. Compares whether one JSON value `lhs` is greater than another JSON value `rhs` according to the
@@ -39,7 +39,12 @@ whether `lhs` is greater than `rhs`
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -84,3 +89,4 @@ Linear.
 
 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_le.md

@@ -5,10 +5,10 @@
 bool operator<=(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator<=(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator<=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator<=(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator<=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 ```
 
 1. Compares whether one JSON value `lhs` is less than or equal to another JSON value `rhs`
@@ -40,7 +40,12 @@ whether `lhs` is less than or equal to `rhs`
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -95,3 +100,4 @@ Linear.
 
 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_lt.md

@@ -5,10 +5,10 @@
 bool operator<(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator<(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator<(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator<(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator<(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 ```
 
 1. Compares whether one JSON value `lhs` is less than another JSON value `rhs` according to the
@@ -49,7 +49,12 @@ whether `lhs` is less than `rhs`
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -94,3 +99,4 @@ Linear.
 
 1. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
 2. Added in version 1.0.0. Conditionally removed since C++20 in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_ne.md

@@ -5,10 +5,10 @@
 bool operator!=(const_reference lhs, const_reference rhs) noexcept;   // (1)
 
 template<typename ScalarType>
-bool operator!=(const_reference lhs, const ScalarType rhs) noexcept;  // (2)
+bool operator!=(const_reference lhs, const ScalarType rhs) noexcept(/* see below */); // (2)
 
 template<typename ScalarType>
-bool operator!=(ScalarType lhs, const const_reference rhs) noexcept;  // (2)
+bool operator!=(ScalarType lhs, const const_reference rhs) noexcept(/* see below */); // (2)
 ```
 
 1. Compares two JSON values for inequality. Returns `#!cpp !(lhs == rhs)`.
@@ -36,7 +36,12 @@ whether the values `lhs`/`*this` and `rhs` are not equal
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -98,3 +103,4 @@ Linear.
    member function in version 3.13.0; since C++20, the compiler rewrites `a != b` using `operator==`.
 2. Added in version 1.0.0. Changed in version 3.13.0 to remove special-casing for `NaN` and `discarded` values;
    `operator!=` now consistently means `!(a == b)`. Since C++20, the compiler rewrites `a != b` using `operator==`.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/operator_spaceship.md

@@ -6,7 +6,7 @@ class basic_json {
     std::partial_ordering operator<=>(const_reference rhs) const noexcept;  // (1)
 
     template<typename ScalarType>
-    std::partial_ordering operator<=>(const ScalarType rhs) const noexcept; // (2)
+    std::partial_ordering operator<=>(const ScalarType rhs) const noexcept(/* see below */); // (2)
 };
 ```
 
@@ -39,7 +39,12 @@ the `std::partial_ordering` of the 3-way comparison of `*this` and `rhs`
 
 ## Exception safety
 
-No-throw guarantee: this function never throws exceptions.
+1. No-throw guarantee: this function never throws exceptions.
+2. No-throw guarantee if converting the scalar to a JSON value cannot throw, as for numbers, Booleans, and
+   `#!cpp nullptr`; the function is `#!cpp noexcept` exactly in that case. Otherwise, it throws what the conversion
+   throws, for example `std::bad_alloc` when converting a string, or
+   [`out_of_range.410`](../../home/exceptions.md#jsonexceptionout_of_range410) for an enum value not mapped by
+   [`NLOHMANN_JSON_SERIALIZE_ENUM_STRICT`](../macros/nlohmann_json_serialize_enum_strict.md).
 
 ## Complexity
 
@@ -98,3 +103,4 @@ Linear.
 
 1. Added in version 3.11.0.
 2. Added in version 3.11.0.
+   Made conditionally `#!cpp noexcept` in version 3.13.0; before, a throwing conversion called `std::terminate`.

docs/mkdocs/docs/api/basic_json/sax_parse.md

@@ -90,7 +90,9 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index
 
 ## Return value
 
-return value of the last processed SAX event
+`#!cpp true` if the input was parsed without errors and no SAX event returned `#!cpp false`; `#!cpp false` otherwise.
+In particular, the result is `#!cpp false` for input with errors, even if the SAX parser recovered from all of them
+(see [error recovery](../../features/parsing/error_recovery.md)).
 
 ## Exception safety
 
@@ -138,6 +140,7 @@ A UTF-8 byte order mark is silently ignored.
 - Ignoring comments via `ignore_comments` added in version 3.9.0.
 - Added `ignore_trailing_commas` in version 3.13.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
+- Recovering from parse errors (see [`parse_error`](../json_sax/parse_error.md)) added in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
 - `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right
   after the parsed value when `strict` is `#!cpp false`.

docs/mkdocs/docs/api/basic_json/to_bjdata.md

@@ -5,15 +5,18 @@
 static std::vector<std::uint8_t> to_bjdata(const basic_json& j,
                                            const bool use_size = false,
                                            const bool use_type = false,
-                                           const bjdata_version_t version = bjdata_version_t::draft2);
+                                           const bjdata_version_t version = bjdata_version_t::draft2,
+                                           const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
 static void to_bjdata(const basic_json& j, detail::output_adapter<std::uint8_t> o,
                       const bool use_size = false, const bool use_type = false,
-                      const bjdata_version_t version = bjdata_version_t::draft2);
+                      const bjdata_version_t version = bjdata_version_t::draft2,
+                      const error_handler_t error_handler = error_handler_t::keep);
 static void to_bjdata(const basic_json& j, detail::output_adapter<char> o,
                       const bool use_size = false, const bool use_type = false,
-                      const bjdata_version_t version = bjdata_version_t::draft2);
+                      const bjdata_version_t version = bjdata_version_t::draft2,
+                      const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Serializes a given JSON value `j` to a byte vector using the BJData (Binary JData) serialization format. BJData aims to
@@ -43,6 +46,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 :   which version of BJData to use (see note on "Binary values" on [BJData](../../features/binary_formats/bjdata.md));
 optional, `#!cpp bjdata_version_t::draft2` by default.
 
+`error_handler` (in)
+:   how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_bjdata` did before
+    this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would.
+    If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead.
+
 ## Return value
 
 1. BJData serialization as byte vector
@@ -56,6 +65,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 
 - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size`
   is false, and `j` contains a non-empty array, object, or binary value.
+- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
+  not valid UTF-8 and `error_handler` is `strict` (the default only if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled)
 
 ## Complexity
 
@@ -105,3 +117,6 @@ Linear in the size of the JSON value `j`.
 
 - Added in version 3.11.0.
 - BJData version parameter (for draft3 binary encoding) added in version 3.12.0.
+- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
+  that is not valid UTF-8 unchanged, as before; `strict` (the default if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`.

docs/mkdocs/docs/api/basic_json/to_bson.md

@@ -2,11 +2,14 @@
 
 ```cpp
 // (1)
-static std::vector<std::uint8_t> to_bson(const basic_json& j);
+static std::vector<std::uint8_t> to_bson(const basic_json& j,
+                                         const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
-static void to_bson(const basic_json& j, detail::output_adapter<std::uint8_t> o);
+static void to_bson(const basic_json& j, detail::output_adapter<std::uint8_t> o,
-static void to_bson(const basic_json& j, detail::output_adapter<char> o);
+                    const error_handler_t error_handler = error_handler_t::keep);
+static void to_bson(const basic_json& j, detail::output_adapter<char> o,
+                    const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are stored as a single entity (a
@@ -25,6 +28,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `o` (in)
 :   output adapter to write serialization to
 
+`error_handler` (in)
+:   how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_bson` did before
+    this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would.
+    If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead.
+
 ## Return value
 
 1. BSON serialization as a byte vector
@@ -46,6 +55,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value
   exceeds 255, the maximum of the BSON binary subtype; example:
   `"subtype 70000 is too large for the BSON binary subtype (max 255)"`
+- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key is
+  not valid UTF-8 and `error_handler` is `strict` (the default only if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled)
 
 ## Complexity
 
@@ -98,3 +110,7 @@ pass before anything is written.
 - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0.
 - Linear in the size of `j`, and no longer limited by the call stack for deeply nested values, since version 3.13.0.
 - `out_of_range.415` is now detected before anything is written, like the other exceptions above, since version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
+  that is not valid UTF-8 unchanged, as before; `strict` (the default if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316` before anything
+  is written.

docs/mkdocs/docs/api/basic_json/to_cbor.md

@@ -2,11 +2,14 @@
 
 ```cpp
 // (1)
-static std::vector<std::uint8_t> to_cbor(const basic_json& j);
+static std::vector<std::uint8_t> to_cbor(const basic_json& j,
+                                          const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
-static void to_cbor(const basic_json& j, detail::output_adapter<std::uint8_t> o);
+static void to_cbor(const basic_json& j, detail::output_adapter<std::uint8_t> o,
-static void to_cbor(const basic_json& j, detail::output_adapter<char> o);
+                     const error_handler_t error_handler = error_handler_t::keep);
+static void to_cbor(const basic_json& j, detail::output_adapter<char> o,
+                     const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Serializes a given JSON value `j` to a byte vector using the CBOR (Concise Binary Object Representation) serialization
@@ -26,6 +29,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `o` (in)
 :   output adapter to write serialization to
 
+`error_handler` (in)
+:   how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_cbor` did before
+    this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would.
+    If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead.
+
 ## Return value
 
 1. CBOR serialization as a byte vector
@@ -35,6 +44,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 
 Strong guarantee: if an exception is thrown, there are no changes in the JSON value.
 
+## Exceptions
+
+- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
+  not valid UTF-8 and `error_handler` is `strict` (the default only if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled)
+
 ## Complexity
 
 Linear in the size of the JSON value `j`.
@@ -68,3 +83,6 @@ Linear in the size of the JSON value `j`.
 
 - Added in version 2.0.9.
 - Compact representation of floating-point numbers added in version 3.8.0.
+- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
+  that is not valid UTF-8 unchanged, as before; `strict` (the default if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`.

docs/mkdocs/docs/api/basic_json/to_msgpack.md

@@ -2,11 +2,14 @@
 
 ```cpp
 // (1)
-static std::vector<std::uint8_t> to_msgpack(const basic_json& j);
+static std::vector<std::uint8_t> to_msgpack(const basic_json& j,
+                                            const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
-static void to_msgpack(const basic_json& j, detail::output_adapter<std::uint8_t> o);
+static void to_msgpack(const basic_json& j, detail::output_adapter<std::uint8_t> o,
-static void to_msgpack(const basic_json& j, detail::output_adapter<char> o);
+                       const error_handler_t error_handler = error_handler_t::keep);
+static void to_msgpack(const basic_json& j, detail::output_adapter<char> o,
+                       const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Serializes a given JSON value `j` to a byte vector using the MessagePack serialization format. MessagePack is a binary
@@ -25,6 +28,13 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 `o` (in)
 :   output adapter to write serialization to
 
+`error_handler` (in)
+:   how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_msgpack` did before
+    this parameter was added and as the MessagePack specification allows; `strict` throws; `replace`/`ignore` sanitize
+    it the same way [`dump`](dump.md) would. Unlike the other binary writers, the default stays `keep` even if
+    [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled.
+
 ## Return value
 
 1. MessagePack serialization as a byte vector
@@ -42,6 +52,8 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value
   exceeds 255, the maximum of the MessagePack ext type; example:
   `"subtype 70000 is too large for the MessagePack ext type (max 255)"`
+- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
+  not valid UTF-8 and `error_handler` is `strict`
 
 ## Complexity
 
@@ -91,6 +103,8 @@ Linear in the size of the JSON value `j`.
 
 - Added in version 2.0.9.
 - Throws `out_of_range.412` and `out_of_range.415` since version 3.13.0.
+- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
+  that is not valid UTF-8 unchanged, as before.
 - Fixed in version 3.13.0 to serialize `number_integer_t`/`number_unsigned_t` pairs of different width correctly;
   before, integers could be serialized with the wrong value if `number_integer_t` was narrower than
   `number_unsigned_t`.

docs/mkdocs/docs/api/basic_json/to_ubjson.md

@@ -4,13 +4,16 @@
 // (1)
 static std::vector<std::uint8_t> to_ubjson(const basic_json& j,
                                            const bool use_size = false,
-                                           const bool use_type = false);
+                                           const bool use_type = false,
+                                           const error_handler_t error_handler = error_handler_t::keep);
 
 // (2)
 static void to_ubjson(const basic_json& j, detail::output_adapter<std::uint8_t> o,
-                      const bool use_size = false, const bool use_type = false);
+                      const bool use_size = false, const bool use_type = false,
+                      const error_handler_t error_handler = error_handler_t::keep);
 static void to_ubjson(const basic_json& j, detail::output_adapter<char> o,
-                      const bool use_size = false, const bool use_type = false);
+                      const bool use_size = false, const bool use_type = false,
+                      const error_handler_t error_handler = error_handler_t::keep);
 ```
 
 Serializes a given JSON value `j` to a byte vector using the UBJSON (Universal Binary JSON) serialization format. UBJSON
@@ -36,6 +39,12 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 :   whether to add type annotations to container types (must be combined with `#!cpp use_size = true`); optional,
     `#!cpp false` by default.
 
+`error_handler` (in)
+:   how to treat a string or object key in `j` that is not valid UTF-8; see [`error_handler_t`](error_handler_t.md).
+    The default, `keep`, writes the ill-formed bytes to the output as is, as every version of `to_ubjson` did before
+    this parameter was added; `strict` throws; `replace`/`ignore` sanitize it the same way [`dump`](dump.md) would.
+    If [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled, the default is `strict` instead.
+
 ## Return value
 
 1. UBJSON serialization as a byte vector
@@ -49,6 +58,9 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 
 - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size`
   is false, and `j` contains a non-empty array, object, or binary value.
+- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
+  not valid UTF-8 and `error_handler` is `strict` (the default only if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled)
 
 ## Complexity
 
@@ -97,3 +109,6 @@ Linear in the size of the JSON value `j`.
 ## Version history
 
 - Added in version 3.1.0.
+- Added `error_handler` parameter in version 3.13.0. Its default, `keep`, writes the bytes of a string or object key
+  that is not valid UTF-8 unchanged, as before; `strict` (the default if
+  [`JSON_STRICT_BINARY_UTF8`](../macros/json_strict_binary_utf8.md) is enabled) throws `type_error.316`.

docs/mkdocs/docs/api/basic_json/value.md

@@ -222,7 +222,9 @@ changes to any JSON value.
 1. Added in version 1.0.0. Changed parameter `default_value` type from `const ValueType&` to `ValueType&&` in version
    3.11.0. Deleted overload for integral key types added in version 3.13.0 to reject such calls at compile time
    instead of causing undefined behavior at runtime.
-2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2.
+2. Added in version 3.11.0. Made `ValueType` the first template parameter in version 3.11.2. Fixed in version 3.13.0
+   to consistently accept `std::string_view`-convertible keys, as already supported by
+   [`operator[]`](operator[].md), [`at`](at.md), [`find`](find.md), and other lookup functions.
 3. Added in version 2.0.2. Extended to work with arrays in version 3.13.0, including fixing an issue where resolving
    `ptr` through an array unexpectedly threw `out_of_range` instead of returning the resolved element (or
    `default_value`, as documented).

docs/mkdocs/docs/api/json_sax/index.md

@@ -7,7 +7,8 @@ struct json_sax;
 
 This class describes the SAX interface used by [sax_parse](../basic_json/sax_parse.md). Each function is called in
 different situations while the input is parsed. The boolean return value informs the parser whether to continue
-processing the input.
+processing the input; for [`parse_error`](parse_error.md), it decides whether to
+[recover from the error](../../features/parsing/error_recovery.md).
 
 For instance, parsing the JSON text `{"a": [1, true]}` triggers the following callbacks, in order:
 

docs/mkdocs/docs/api/json_sax/parse_error.md

@@ -21,11 +21,18 @@ A parse error occurred.
 
 ## Return value
 
-Whether parsing should proceed (**must return `#!cpp false`**).
+Whether to recover from the error:
+
+- `#!cpp false` stops parsing.
+- `#!cpp true` recovers from the error: the error is repaired and parsing continues. If that is not possible, which
+  happens in the binary formats when the end of the item with the error is unknown, the value read so far is completed
+  and parsing stops. See [error recovery](../../features/parsing/error_recovery.md) for how errors are repaired.
+
+Either way, [`sax_parse`](../basic_json/sax_parse.md) returns `#!cpp false`.
 
 ## Examples
 
-??? example
+??? example "Example: (1) the SAX interface"
 
     The example below shows how the SAX interface is used.
 
@@ -39,12 +46,29 @@ Whether parsing should proceed (**must return `#!cpp false`**).
     --8<-- "examples/sax_parse.output"
     ```
 
+??? example "Example: (2) recovering from errors"
+
+    The example below shows how a SAX parser recovers from errors.
+
+    ```cpp
+    --8<-- "examples/sax_parse__error_recovery.cpp"
+    ```
+    
+    Output:
+    
+    ```
+    --8<-- "examples/sax_parse__error_recovery.output"
+    ```
+
 ## See also
 
 - [sax_parse](../basic_json/sax_parse.md) - SAX parser
 - [Parsing and Exceptions](../../features/parsing/parse_exceptions.md) - the article on handling parse errors without
   exceptions
+- [Error Recovery](../../features/parsing/error_recovery.md) - the article on recovering from parse errors
 
 ## Version history
 
 - Added in version 3.2.0.
+- Returning `#!cpp true` recovers from the error since version 3.13.0; before, parsing stopped, but the result of
+  [`sax_parse`](../basic_json/sax_parse.md) could be wrong.

docs/mkdocs/docs/api/macros/index.md

@@ -18,6 +18,8 @@ header. See also the [macro overview page](../../features/macros.md).
 
 - [**JSON_PRECISE_STREAM_POSITION**](json_precise_stream_position.md) - opt in to leaving an input stream positioned
   right after a parsed number
+- [**JSON_STRICT_BINARY_UTF8**](json_strict_binary_utf8.md) - opt in to checking strings for valid UTF-8 in the CBOR,
+  UBJSON, BJData, and BSON writers
 - [**JSON_STRICT_NUL_HANDLING**](json_strict_nul_handling.md) - opt in to rejecting a NUL byte in the input instead of
   treating it as end of input
 

docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md

@@ -115,3 +115,4 @@ The default value is `0` (disabled — existing behavior is preserved).
 ## Version history
 
 - Added in version 3.13.0.
+- Planned to become the default (with the macro removed) in version 4.0.0.

docs/mkdocs/docs/api/macros/json_strict_binary_utf8.md

@@ -0,0 +1,101 @@
+# JSON_STRICT_BINARY_UTF8
+
+```cpp
+#define JSON_STRICT_BINARY_UTF8 /* value */
+```
+
+When defined to `1`, the `error_handler` parameter of the binary writers [`to_cbor`](../basic_json/to_cbor.md),
+[`to_ubjson`](../basic_json/to_ubjson.md), [`to_bjdata`](../basic_json/to_bjdata.md), and
+[`to_bson`](../basic_json/to_bson.md) defaults to [`error_handler_t::strict`](../basic_json/error_handler_t.md) instead
+of `error_handler_t::keep`. These writers then check every string value and object key for valid UTF-8 and throw
+[`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8, like
+[`dump`](../basic_json/dump.md) does. Without it, they write the bytes unchanged. An `error_handler` passed explicitly
+always takes precedence.
+
+The macro does not affect:
+
+- [`to_msgpack`](../basic_json/to_msgpack.md): the MessagePack specification allows a `str` value to contain bytes that
+  are not valid UTF-8, so its `error_handler` always defaults to `keep`.
+- [`to_bon8`](../basic_json/to_bon8.md): BON8 always checks, because the UTF-8 lead bytes mark where a string ends.
+- The binary readers ([`from_cbor`](../basic_json/from_cbor.md), [`from_msgpack`](../basic_json/from_msgpack.md),
+  [`from_ubjson`](../basic_json/from_ubjson.md), [`from_bjdata`](../basic_json/from_bjdata.md),
+  [`from_bson`](../basic_json/from_bson.md)): none of these formats requires a decoder to reject ill-formed UTF-8, so
+  they always return the bytes unchanged.
+
+## Default definition
+
+The default value is `0` (disabled, the behavior of version 3.12.0 and earlier is preserved).
+
+```cpp
+#define JSON_STRICT_BINARY_UTF8 0
+```
+
+## Notes
+
+!!! note "Background"
+
+    CBOR, UBJSON, BJData, and BSON all require strings to be UTF-8. Up to version 3.12.0, the writers did not check
+    this, so they could produce output that other decoders reject. Checking by default would break code that stores
+    other encodings (for instance ISO 8859-1) in a string and only ever writes it to a binary format. You can pass
+    `error_handler_t::strict` to each call, or use this macro to check by default ahead of version 4.0.0, where
+    `strict` is planned to become the default (see
+    [#5529](https://github.com/nlohmann/json/issues/5529) and [#5651](https://github.com/nlohmann/json/issues/5651)).
+
+!!! warning "Opt-in only"
+
+    This macro must be defined **before** including `<nlohmann/json.hpp>`. Defining it after the include has no
+    effect.
+
+!!! note "ABI compatibility"
+
+    The value of this macro is encoded in the [namespace](../../features/namespace.md) (tag `_sbu8`), resulting in
+    distinct symbol names. Translation units compiled with and without it can therefore be linked into the same program
+    without One Definition Rule (ODR) violations, but they cannot exchange instances of library types.
+
+## Examples
+
+??? example "Default behavior (macro not defined)"
+
+    Without the macro, the bytes are written unchanged:
+
+    ```cpp
+    #include <nlohmann/json.hpp>
+
+    using json = nlohmann::json;
+
+    int main()
+    {
+        auto v = json::to_cbor(json("\xFF"));
+        // v is {0x61, 0xFF}
+    }
+    ```
+
+??? example "Opt-in check (macro defined to 1)"
+
+    With the macro, ill-formed UTF-8 is rejected:
+
+    ```cpp
+    #define JSON_STRICT_BINARY_UTF8 1
+    #include <nlohmann/json.hpp>
+
+    using json = nlohmann::json;
+
+    int main()
+    {
+        auto v = json::to_cbor(json("\xFF"));
+        // throws type_error.316: invalid UTF-8 byte at index 0: 0xFF
+    }
+    ```
+
+## See also
+
+- [**to_cbor**](../basic_json/to_cbor.md) - create a CBOR serialization of a JSON value
+- [**to_ubjson**](../basic_json/to_ubjson.md) - create a UBJSON serialization of a JSON value
+- [**to_bjdata**](../basic_json/to_bjdata.md) - create a BJData serialization of a JSON value
+- [**to_bson**](../basic_json/to_bson.md) - create a BSON serialization of a JSON value
+- [**error_handler_t**](../basic_json/error_handler_t.md) - how [`dump`](../basic_json/dump.md) treats ill-formed UTF-8
+
+## Version history
+
+- Added in version 3.13.0.
+- Planned to become the default (with the macro removed) in version 4.0.0.

docs/mkdocs/docs/api/macros/json_use_implicit_conversions.md

@@ -5,7 +5,9 @@
 ```
 
 When defined to `0`, implicit conversions are switched off. By default, implicit conversions are switched on. The
-value directly affects [`operator ValueType`](../basic_json/operator_ValueType.md).
+value directly affects [`operator ValueType`](../basic_json/operator_ValueType.md) and the
+[converting constructor](../basic_json/basic_json.md) from a `basic_json` specialization with a different string
+type (overload 4).
 
 ## Default definition
 
@@ -59,6 +61,25 @@ By default, implicit conversions are enabled.
     auto s = j.get<std::string>();
     ```
 
+??? example "Conversion between `basic_json` specializations"
+
+    A `basic_json` specialization with a different string type is also no longer converted implicitly when
+    `JSON_USE_IMPLICIT_CONVERSIONS` is defined to `0`:
+
+    ```cpp
+    using wjson = nlohmann::basic_json<std::map, std::vector, std::wstring>;
+
+    void load(const nlohmann::json& j);
+
+    wjson wj = /* ... */;
+    load(wj);                            // error: no implicit conversion
+    load(nlohmann::json(wj));            // OK: explicit conversion
+    load(wj.get<nlohmann::json>());      // OK: explicit conversion
+    ```
+
+    Specializations that share the same string type, such as `json` and `ordered_json`, remain implicitly
+    convertible.
+
 ## See also
 
 - [**operator ValueType**](../basic_json/operator_ValueType.md) - get a value (implicit)
@@ -68,3 +89,4 @@ By default, implicit conversions are enabled.
 ## Version history
 
 - Added in version 3.9.0.
+- Also affects the conversion between `basic_json` specializations with different string types since version 3.13.0.

docs/mkdocs/docs/api/macros/json_use_legacy_discarded_value_comparison.md

@@ -81,3 +81,6 @@ When the macro is not defined, the library will define it to its default value.
 ## Version history
 
 - Added in version 3.11.0.
+- Fixed in version 3.13.0 so `<=` and `>=` also emulate the legacy behavior in C++20 when the JSON value is the
+  right-hand operand of a scalar comparison; before, only the 3-way-comparison-rewritten candidate was found, which
+  yielded `#!cpp false` instead of `#!cpp true`.

docs/mkdocs/docs/community/roadmap.md

@@ -14,7 +14,7 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js
   opt-in.
 - **Keep the 3.x public API stable.** Releases follow [semantic versioning](https://semver.org). Changes that would
   break existing code are only added behind a feature macro, so users can opt in and test their code before a next
-  major release.
+  major release, see [Version 4.0](#version-40).
 - **Support a broad range of compilers and platforms.** The [CI](quality_assurance.md) keeps testing old and new
   versions of GCC, Clang, MSVC, and other compilers on Linux, macOS, and Windows.
 - **Keep the quality assurance up.** Every change keeps the test coverage at 100%, passes the static and dynamic
@@ -37,7 +37,67 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js
 
 ## Version 4.0
 
-There is no decision yet on whether or when a version 4.0 with breaking changes will be released. Proposals that need
+There is no release date for version 4.0 yet. Proposals that need a major version, for instance stricter type
-a major version, for instance stricter type conversions, are collected in issue
+conversions, are collected in issue [#3453](https://github.com/nlohmann/json/issues/3453).
-[#3453](https://github.com/nlohmann/json/issues/3453). Until then, such changes are only added as opt-in behavior
+
-behind feature macros.
+!!! note "Not final"
+
+    The plan for version 4.0 described below is not final and may still change: macros may be added to or removed from
+    the list, and planned defaults may be revised. Any such change will be documented on this page.
+
+### Trying out 4.0 today
+
+Version 4.0 will not be developed on a separate branch. Instead, every breaking change is first added to a 3.x release
+behind a macro whose default keeps the 3.x behavior. Version 4.0 then switches the defaults and removes the macros.
+Version 4.0 is therefore the sum of these macros: you can try it on the 3.x release train today by defining each macro
+to its 4.0 value and fixing what no longer compiles or behaves differently. Once your code works with all of them, it
+is ready for version 4.0.
+
+The following macros guard changes that are planned to become the default in version 4.0:
+
+| Macro                                                                                                            | 3.x default | 4.0 behavior                                                                                                                  | CMake option                                                                                                       | Added  |
+|------------------------------------------------------------------------------------------------------------------|-------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------|--------|
+| [`JSON_USE_IMPLICIT_CONVERSIONS`](../api/macros/json_use_implicit_conversions.md)                                | `1`         | `0`: no implicit conversions from `basic_json` to other types; use [`get`](../api/basic_json/get.md) instead                  | [`JSON_ImplicitConversions`](../integration/cmake.md#json_implicitconversions)                                     | 3.9.0  |
+| [`JSON_USE_GLOBAL_UDLS`](../api/macros/json_use_global_udls.md)                                                  | `1`         | `0`: the string literals `_json` and `_json_pointer` are only available in namespace `nlohmann::literals`                    | [`JSON_GlobalUDLs`](../integration/cmake.md#json_globaludls)                                                       | 3.11.0 |
+| [`JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON`](../api/macros/json_use_legacy_discarded_value_comparison.md)      | `0`         | removed: the deprecated legacy comparison of discarded values can no longer be enabled                                        | [`JSON_LegacyDiscardedValueComparison`](../integration/cmake.md#json_legacydiscardedvaluecomparison)               | 3.11.0 |
+| [`JSON_BRACE_INIT_COPY_SEMANTICS`](../api/macros/json_brace_init_copy_semantics.md)                              | `0`         | `1`: single-element brace initialization such as `#!cpp json j{obj};` copies the element instead of creating an array         | –                                                                                                                  | 3.13.0 |
+| [`JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md)                                  | `0`         | `1`: reading from a stream does not consume the character after a number                                                      | –                                                                                                                  | 3.13.0 |
+| [`JSON_STRICT_NUL_HANDLING`](../api/macros/json_strict_nul_handling.md)                                          | `0`         | `1`: a NUL byte in the input is a parse error instead of the end of input                                                     | [`JSON_StrictNulHandling`](../integration/cmake.md#json_strictnulhandling)                                         | 3.13.0 |
+| [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md)                                            | `0`         | `1`: `to_cbor`, `to_ubjson`, `to_bjdata`, and `to_bson` throw for strings that are not valid UTF-8 by default                 | [`JSON_StrictBinaryUTF8`](../integration/cmake.md#json_strictbinaryutf8)                                           | 3.13.0 |
+
+For example, the following makes a 3.x release behave like version 4.0 with respect to these changes:
+
+```cpp
+#define JSON_USE_IMPLICIT_CONVERSIONS 0
+#define JSON_USE_GLOBAL_UDLS 0
+#define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0
+#define JSON_BRACE_INIT_COPY_SEMANTICS 1
+#define JSON_PRECISE_STREAM_POSITION 1
+#define JSON_STRICT_NUL_HANDLING 1
+#define JSON_STRICT_BINARY_UTF8 1
+#include <nlohmann/json.hpp>
+```
+
+The macros must be defined before the library header is included; setting them once in the build system is the easiest
+way to achieve this.
+
+### Removal of deprecated functions
+
+Version 4.0 will remove all deprecated functions. Compiling with deprecation warnings enabled shows which of them your
+code still uses. The [migration guide](../integration/migration_guide.md#replace-deprecated-functions) shows how to
+replace each of them.
+
+| Deprecated                                                                                                                                                                                                                                         | Since  | Migration                                                                        |
+|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|----------------------------------------------------------------------------------|
+| `#!cpp operator<<(basic_json&, std::istream&)`                                                                                                                                                                                                     | 3.0.0  | [Parsing](../integration/migration_guide.md#parsing)                             |
+| `#!cpp operator>>(const basic_json&, std::ostream&)`                                                                                                                                                                                               | 3.0.0  | [Miscellaneous functions](../integration/migration_guide.md#miscellaneous-functions) |
+| `iterator_wrapper`                                                                                                                                                                                                                                 | 3.1.0  | [Miscellaneous functions](../integration/migration_guide.md#miscellaneous-functions) |
+| [`parse`](../api/basic_json/parse.md), [`accept`](../api/basic_json/accept.md), and [`sax_parse`](../api/basic_json/sax_parse.md) with an initializer list `{ptr, len}` or `{first, last}`                                                         | 3.8.0  | [Parsing](../integration/migration_guide.md#parsing)                             |
+| [`from_bson`](../api/basic_json/from_bson.md), [`from_cbor`](../api/basic_json/from_cbor.md), [`from_msgpack`](../api/basic_json/from_msgpack.md), and [`from_ubjson`](../api/basic_json/from_ubjson.md) with `(ptr, len)` or an initializer list | 3.8.0  | [Parsing](../integration/migration_guide.md#parsing)                             |
+| [`json_pointer::operator string_t`](../api/json_pointer/operator_string_t.md)                                                                                                                                                                      | 3.11.0 | [JSON Pointers](../integration/migration_guide.md#json-pointers)                 |
+| [`json_pointer`](../api/json_pointer/index.md) with a `basic_json` type as template argument, and the overloads of `value`, `contains`, `operator[]`, and `at` accepting such a pointer                                                           | 3.11.0 | [JSON Pointers](../integration/migration_guide.md#json-pointers)                 |
+| Comparing a [`json_pointer`](../api/json_pointer/index.md) with a string via [`operator==`](../api/json_pointer/operator_eq.md) or [`operator!=`](../api/json_pointer/operator_ne.md)                                                              | 3.11.2 | [JSON Pointers](../integration/migration_guide.md#json-pointers)                 |
+
+The deprecated legacy comparison of discarded values is controlled by a macro and therefore listed in the table above.
+
+New breaking changes will follow the same path: they are added to these tables when they land in a 3.x release.

docs/mkdocs/docs/examples/as_base_class.cpp

@@ -0,0 +1,41 @@
+#include <iostream>
+#include <nlohmann/json.hpp>
+
+class base_class_with_hidden_members
+{
+  public:
+    const char* type_name() const noexcept
+    {
+        return "my_type_name";
+    }
+
+    std::size_t size() const noexcept
+    {
+        return 42;
+    }
+};
+
+using json = nlohmann::basic_json <
+             std::map,
+             std::vector,
+             std::string,
+             bool,
+             std::int64_t,
+             std::uint64_t,
+             double,
+             std::allocator,
+             nlohmann::adl_serializer,
+             std::vector<std::uint8_t>,
+             base_class_with_hidden_members
+             >;
+
+int main()
+{
+    json j = {1, 2, 3};
+
+    // the members of basic_json hide the members of the base class
+    std::cout << j.type_name() << ' ' << j.size() << '\n';
+
+    // access the hidden members of the base class
+    std::cout << j.as_base_class().type_name() << ' ' << j.as_base_class().size() << '\n';
+}

docs/mkdocs/docs/examples/as_base_class.output

@@ -0,0 +1,2 @@
+array 3
+my_type_name 42

docs/mkdocs/docs/examples/error_handler_t.cpp

@@ -20,5 +20,6 @@ int main()
               << j_invalid.dump(-1, ' ', false, json::error_handler_t::replace)
               << "\nstring with ignored invalid characters: "
               << j_invalid.dump(-1, ' ', false, json::error_handler_t::ignore)
-              << '\n';
+              << "\nstring with the invalid byte kept as is (" << j_invalid.dump(-1, ' ', false, json::error_handler_t::keep).size()
+              << " bytes, not valid UTF-8 itself)\n";
 }

docs/mkdocs/docs/examples/error_handler_t.output

@@ -1,3 +1,4 @@
 [json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9
 string with replaced invalid characters: "ä�ü"
 string with ignored invalid characters: "äü"
+string with the invalid byte kept as is (7 bytes, not valid UTF-8 itself)

docs/mkdocs/docs/examples/sax_parse__error_recovery.cpp

@@ -0,0 +1,43 @@
+#include <iostream>
+#include <iomanip>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+// a SAX parser that creates a JSON value like json::parse does, but that
+// recovers from parse errors instead of stopping at the first one
+class recovering_parser : public nlohmann::detail::json_sax_dom_parser<json>
+{
+  public:
+    explicit recovering_parser(json& result)
+        : nlohmann::detail::json_sax_dom_parser<json>(result, false)
+    {}
+
+    bool parse_error(std::size_t position,
+                     const std::string& /*last_token*/,
+                     const json::exception& ex)
+    {
+        std::cout << "byte " << position << ": " << ex.what() << '\n';
+
+        // repair the input and continue
+        return true;
+    }
+};
+
+int main()
+{
+    // JSON text with several mistakes that ends too early
+    const std::string text = R"({
+    "name": "Hello World",
+    "tags": ["a" "b",],
+    "valid": tru,
+    "size": 1.,
+    "nested": {"x": 1)";
+
+    json result;
+    recovering_parser sax(result);
+    const bool valid = json::sax_parse(text, &sax);
+
+    std::cout << "\nvalid JSON: " << std::boolalpha << valid << '\n'
+              << std::setw(4) << result << std::endl;
+}

docs/mkdocs/docs/examples/sax_parse__error_recovery.output

@@ -0,0 +1,19 @@
+byte 49: [json.exception.parse_error.101] parse error at line 3, column 20: syntax error while parsing array - unexpected string literal; expected ']'
+byte 51: [json.exception.parse_error.101] parse error at line 3, column 22: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal
+byte 70: [json.exception.parse_error.101] parse error at line 4, column 17: syntax error while parsing value - invalid literal; last read: '"valid": tru,'
+byte 86: [json.exception.parse_error.101] parse error at line 5, column 15: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.,'
+byte 109: [json.exception.parse_error.101] parse error at line 6, column 22: syntax error while parsing object - unexpected end of input; expected '}'
+
+valid JSON: false
+{
+    "name": "Hello World",
+    "nested": {
+        "x": 1
+    },
+    "size": 1,
+    "tags": [
+        "a",
+        "b"
+    ],
+    "valid": null
+}

docs/mkdocs/docs/features/arbitrary_types.md

@@ -79,6 +79,7 @@ Some important things:
 * When using `get<your_type>()`, `your_type` **MUST** be [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). (There is a way to bypass this requirement described later.)
 * In function `from_json`, use function [`at()`](../api/basic_json/at.md) to access the object values rather than `operator[]`. In case a key does not exist, `at` throws an exception that you can handle, whereas `operator[]` exhibits undefined behavior.
 * You do not need to add serializers or deserializers for STL types like `std::vector`: the library already implements these.
+* If you control the type, consider defining `to_json`/`from_json` as `friend` functions inside the class ("hidden friends"). Argument-dependent lookup then only finds them for your type, which also avoids a [GCC < 11 compilation error](../home/faq.md#incomplete-detector-type-with-gcc-11).
 
 ??? example "Example: serialize a `person` to JSON with `to_json`"
 

docs/mkdocs/docs/features/assertions.md

@@ -16,14 +16,15 @@ before including the `json.hpp` header.
 
 ## Function with runtime assertions
 
-### Unchecked object access to a const value
+### Unchecked access to a const value
 
-Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for objects. Whereas a missing
+Function [`operator[]`](../api/basic_json/operator%5B%5D.md) implements unchecked access for arrays and objects. Whereas
-key is added in the case of non-const objects, accessing a const object with a missing key is undefined behavior (think
+a missing element is added in the case of non-const values, accessing a const value with a missing object key or an
-of a dereferenced null pointer) and yields a runtime assertion.
+invalid array index is undefined behavior (think of a dereferenced null pointer) and yields a runtime assertion. This
+also applies to a [JSON pointer](json_pointer.md) that refers to a missing key or an invalid index.
 
-If you are not sure whether an element in an object exists, use checked access with the
+If you are not sure whether an element exists, use checked access with the [`at` function](../api/basic_json/at.md)
-[`at` function](../api/basic_json/at.md) or call the [`contains` function](../api/basic_json/contains.md) before.
+or call the [`contains` function](../api/basic_json/contains.md) before.
 
 See also the documentation on [element access](element_access/index.md).
 
@@ -46,7 +47,30 @@ See also the documentation on [element access](element_access/index.md).
     Output:
 
     ```
-    Assertion failed: (m_value.object->find(key) != m_value.object->end()), function operator[], file json.hpp, line 2144.
+    Assertion failed: (it != m_data.m_value.object->end()), function operator[], file json.hpp, line 28795.
+    ```
+
+??? example "Example 2: Invalid array index in a JSON pointer"
+
+    The following code will trigger an assertion at runtime:
+
+    ```cpp
+    #include <nlohmann/json.hpp>
+    
+    using json = nlohmann::json;
+    using namespace nlohmann::literals;
+    
+    int main()
+    {
+        const json j = {{"array", {1, 2, 3}}};
+        auto v = j["/array/5"_json_pointer];
+    }
+    ```
+
+    Output:
+
+    ```
+    Assertion failed: (idx < m_data.m_value.array->size()), function operator[], file json.hpp, line 28758.
     ```
 
 ### Constructing from an uninitialized iterator range

docs/mkdocs/docs/features/binary_formats/bjdata.md

@@ -63,6 +63,15 @@ The library uses the following mapping from JSON values types to BJData types ac
 
       - strings with more than 18446744073709551615 bytes, i.e., 2<sup>64</sup>-1 bytes (theoretical)
 
+!!! warning "UTF-8 validation of string values and object keys"
+
+    BJData strings must use UTF-8 encoding. By default (the [`error_handler`](../../api/basic_json/to_bjdata.md)
+    parameter left at `keep`), `to_bjdata()` writes the bytes of string values and object keys unchanged, even if they
+    are not valid UTF-8. With `error_handler_t::strict`, it throws
+    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead;
+    `replace`/`ignore` sanitize the string. [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md)
+    makes `strict` the default.
+
 !!! info "Unused BJData markers"
 
     The following markers are not used in the conversion:
@@ -208,6 +217,19 @@ The library maps BJData types to JSON value types as follows:
 
     The mapping is **complete** in the sense that any BJData value can be converted to a JSON value.
 
+!!! warning "Ill-formed UTF-8 in string values and object keys"
+
+    BJData strings must use UTF-8 encoding, but checking it on read is opt-in: with the
+    [`error_handler`](../../api/basic_json/from_bjdata.md) parameter left at `keep` (the default), `from_bjdata()`
+    accepts a string value or object key whose bytes are not valid UTF-8 and hands them back unchanged. Passing
+    `error_handler_t::strict` makes `from_bjdata()` check and throw
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and
+    `replace`/`ignore` sanitize the string instead of keeping it. However,
+    [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
+    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read with the default
+    `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes. `to_bjdata()`'s
+    own `error_handler` parameter defaults to `keep` (see above), so such a value is written back unchanged.
+
 !!! info "Round trips"
 
     A value returned by [`from_bjdata`](../../api/basic_json/from_bjdata.md) can be serialized with

docs/mkdocs/docs/features/binary_formats/bson.md

@@ -109,14 +109,21 @@ The library maps BSON record types to JSON value types as follows:
     If BSON input must be validated for strict specification compliance, validate it separately before passing it to
     `from_bson()`.
 
-!!! warning "UTF-8 validation of string values"
+!!! warning "Ill-formed UTF-8 in string values"
 
-    The BSON specification requires `string` values (type `0x02`) to be valid UTF-8. This library validates the
+    The BSON specification requires `string` values (type `0x02`) to be valid UTF-8, but this is not required of a
-    bytes of every such string at decode time and rejects ill-formed UTF-8 with a
+    decoder, so checking is opt-in: with the [`error_handler`](../../api/basic_json/from_bson.md) parameter left at
-    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions`
+    `keep` (the default), `from_bson()` accepts a `string` value whose bytes are not valid UTF-8 and hands them back
-    set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. Element
+    unchanged. Passing `error_handler_t::strict` makes `from_bson()` check and throw
-    (key) names and `binary` values (type `0x05`) are unaffected and are never validated, since they are read
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and
-    byte-by-byte as a C string, or are not required to hold text, respectively.
+    `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md)
+    still requires valid UTF-8 and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a
+    value read with the default `keep` handler, unless an error handler is passed that replaces or ignores the
+    ill-formed bytes. `to_bson()`'s own `error_handler` parameter defaults to `keep`, so such a string value or element
+    (key) name is written unchanged; with `strict` (the default if
+    [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled), it throws the same exception
+    instead. Element (key) names are never validated on read, since they are read byte-by-byte as a C string. `binary`
+    values (type `0x05`) are unaffected, since they are not required to hold text.
 
 ??? example "Example: deserialize a JSON value from BSON"
 

docs/mkdocs/docs/features/binary_formats/cbor.md

@@ -189,15 +189,21 @@ The library maps CBOR types to JSON value types as follows:
     ([RFC 8392](https://www.rfc-editor.org/rfc/rfc8392.html)), cannot be read with this library and need a
     general-purpose CBOR library instead.
 
-!!! warning "UTF-8 validation of text strings"
+!!! warning "Ill-formed UTF-8 in text strings"
 
-    [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings
+    [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings (major
-    (major type 3) to be valid UTF-8. This library validates the bytes of every text string (object keys included) at
+    type 3) to be valid UTF-8, but leaves it up to the decoder whether to enforce this, so checking is opt-in: with the
-    decode time and rejects ill-formed UTF-8 with a
+    [`error_handler`](../../api/basic_json/from_cbor.md) parameter left at `keep` (the default), `from_cbor()` accepts a
-    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with
+    text string (object keys included) whose bytes are not valid UTF-8 and hands them back unchanged. Passing
-    `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting value is
+    `error_handler_t::strict` makes `from_cbor()` check and throw
-    dumped. Byte strings (major type 2) are unaffected and are never validated, since they are not required to hold
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and
-    text.
+    `replace`/`ignore` sanitize the string instead of keeping it. However, [`dump()`](../../api/basic_json/dump.md)
+    still requires valid UTF-8 and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a
+    value read with the default `keep` handler, unless an error handler is passed that replaces or ignores the
+    ill-formed bytes. `to_cbor()`'s own [`error_handler`](../../api/basic_json/to_cbor.md) parameter defaults to `keep`,
+    so such a value is written back unchanged; with `strict` (the default if
+    [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md) is enabled), it throws the same exception
+    instead. Byte strings (major type 2) are unaffected, since they are not required to hold text.
 
 !!! warning "Tagged items"
 

docs/mkdocs/docs/features/binary_formats/messagepack.md

@@ -153,14 +153,23 @@ The library maps MessagePack types to JSON value types as follows:
     This applies to the [SAX interface](../parsing/sax_interface.md) as well, as the key is read before it is passed
     on. Such input needs a general-purpose MessagePack library instead.
 
-!!! warning "UTF-8 validation of string values"
+!!! warning "Ill-formed UTF-8 in string values"
 
-    The MessagePack specification requires `str` values (`fixstr`, `str 8`, `str 16`, `str 32`) to be valid UTF-8.
+    The MessagePack specification explicitly allows a `str` value (`fixstr`, `str 8`, `str 16`, `str 32`) to contain
-    This library validates the bytes of every such string (object keys included) at decode time and rejects
+    a byte sequence that is not valid UTF-8, and expects a deserializer to hand the original bytes back unchanged.
-    ill-formed UTF-8 with a [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or,
+    This library follows that by default: with its
-    with `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting
+    [`error_handler`](../../api/basic_json/from_msgpack.md) parameter left at `keep` (the default),
-    value is dumped. `bin`/`ext`/`fixext` values are unaffected and are never validated, since they are not required
+    `from_msgpack()` reads `str` bytes (object keys included) as-is, without validating them, so such a value
-    to hold text.
+    round-trips through `from_msgpack(to_msgpack(j))` byte for byte. Passing `error_handler_t::strict` makes
+    `from_msgpack()` check anyway and throw
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and
+    `replace`/`ignore` sanitize the string instead of keeping it. `to_msgpack()` also writes `str` bytes as-is by
+    default, since the specification permits it; its [`error_handler`](../../api/basic_json/to_msgpack.md) parameter
+    can be set to `strict` to throw [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) instead, or
+    to `replace`/`ignore` to sanitize the string, for instance for a decoder that rejects ill-formed UTF-8. However,
+    [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
+    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read this way with the
+    default `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes.
 
 ??? example "Example: deserialize a JSON value from MessagePack"
 

docs/mkdocs/docs/features/binary_formats/ubjson.md

@@ -47,6 +47,15 @@ The library uses the following mapping from JSON values types to UBJSON types ac
 
       - strings with more than 9223372036854775807 bytes (theoretical)
 
+!!! warning "UTF-8 validation of string values and object keys"
+
+    UBJSON's required string encoding is UTF-8. By default (the [`error_handler`](../../api/basic_json/to_ubjson.md)
+    parameter left at `keep`), `to_ubjson()` writes the bytes of string values and object keys unchanged, even if they
+    are not valid UTF-8. With `error_handler_t::strict`, it throws
+    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8 instead;
+    `replace`/`ignore` sanitize the string. [`JSON_STRICT_BINARY_UTF8`](../../api/macros/json_strict_binary_utf8.md)
+    makes `strict` the default.
+
 !!! info "Unused UBJSON markers"
 
     The following markers are not used in the conversion:
@@ -120,6 +129,19 @@ The library maps UBJSON types to JSON value types as follows:
 
     The mapping is **complete** in the sense that any UBJSON value can be converted to a JSON value.
 
+!!! warning "Ill-formed UTF-8 in string values and object keys"
+
+    UBJSON's required string encoding is UTF-8, but checking it on read is opt-in: with the
+    [`error_handler`](../../api/basic_json/from_ubjson.md) parameter left at `keep` (the default), `from_ubjson()`
+    accepts a string value or object key whose bytes are not valid UTF-8 and hands them back unchanged. Passing
+    `error_handler_t::strict` makes `from_ubjson()` check and throw
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) for ill-formed UTF-8, and
+    `replace`/`ignore` sanitize the string instead of keeping it. However,
+    [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
+    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read with the default
+    `keep` handler, unless an error handler is passed that replaces or ignores the ill-formed bytes. `to_ubjson()`'s
+    own `error_handler` parameter defaults to `keep` (see above), so such a value is written back unchanged.
+
 ??? example "Example: deserialize a JSON value from UBJSON"
 
     ```cpp

docs/mkdocs/docs/features/macros.md

@@ -138,6 +138,20 @@ using the library with compilers that do not fully support C++11 and may only wo
 
 See [full documentation of `JSON_SKIP_UNSUPPORTED_COMPILER_CHECK`](../api/macros/json_skip_unsupported_compiler_check.md).
 
+## `JSON_STRICT_BINARY_UTF8`
+
+When defined to `1`, [`to_cbor`](../api/basic_json/to_cbor.md), [`to_ubjson`](../api/basic_json/to_ubjson.md),
+[`to_bjdata`](../api/basic_json/to_bjdata.md), and [`to_bson`](../api/basic_json/to_bson.md) throw
+[`type_error.316`](../home/exceptions.md#jsonexceptiontype_error316) for a string value or object key that is not
+valid UTF-8. The default value is `0`, which writes the bytes unchanged as before version 3.13.0; this is planned to
+become the default in version 4.0.0.
+
+The check can also be enabled with the CMake option
+[`JSON_StrictBinaryUTF8`](../integration/cmake.md#json_strictbinaryutf8) (`OFF` by default) which sets
+`JSON_STRICT_BINARY_UTF8` accordingly.
+
+See [full documentation of `JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md).
+
 ## `JSON_STRICT_NUL_HANDLING`
 
 When defined to `1`, a `'\0'` (NUL) byte anywhere in the input is rejected with `parse_error.101`, like any other

docs/mkdocs/docs/features/namespace.md

@@ -20,6 +20,7 @@ The complete default namespace name is derived as follows:
       `_bics`.
     - [`JSON_PRECISE_STREAM_POSITION`](../api/macros/json_precise_stream_position.md) defined non-zero appends `_psp`.
     - [`JSON_STRICT_NUL_HANDLING`](../api/macros/json_strict_nul_handling.md) defined non-zero appends `_snul`.
+    - [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) defined non-zero appends `_sbu8`.
 - The inline namespace ends with the suffix `_v` followed by the 3 components of the version number separated by
   underscores. To omit the version component, see [Disabling the version component](#disabling-the-version-component)
   below.

docs/mkdocs/docs/features/parsing/error_recovery.md

@@ -0,0 +1,122 @@
+# Error Recovery
+
+By default, parsing stops at the first error. With the [SAX interface](sax_interface.md), you can instead ask the
+parser to *recover*: to repair the error and continue, so that you get as much as possible out of malformed input, for
+instance a file that was cut off, JSON edited by hand, or the output of a language model.
+
+## Recovering from errors
+
+The SAX parser's [`parse_error`](../../api/json_sax/parse_error.md) function is called for every error. Its return value
+decides what happens next:
+
+- `#!cpp false` stops parsing. This is what the SAX parsers of the library do, so [`parse`](../../api/basic_json/parse.md)
+  and [`accept`](../../api/basic_json/accept.md) never recover.
+- `#!cpp true` repairs the error and continues parsing.
+
+When recovering, the SAX parser still receives well-formed events: every `start_object` or `start_array` is followed by
+the matching `end_object` or `end_array`, and every `key` is followed by exactly one value. A SAX parser that creates a
+JSON value, such as the one in the example below, therefore gets a complete value. Parsing always ends, and
+[`sax_parse`](../../api/basic_json/sax_parse.md) returns `#!cpp false` for input that is not valid JSON, even if every
+error was repaired. Each token is reported at most once, and the SAX parser can stop at any error by returning
+`#!cpp false`.
+
+!!! example
+
+    The example below derives a SAX parser from the library's parser for `json` values (`json_sax_dom_parser`),
+    and recovers from all errors.
+
+    ```cpp
+    --8<-- "examples/sax_parse__error_recovery.cpp"
+    ```
+    
+    Output:
+    
+    ```
+    --8<-- "examples/sax_parse__error_recovery.output"
+    ```
+
+## How errors are repaired
+
+Each error is repaired with the smallest local edit: a missing separator is inserted, a stray token is removed, what can
+be read of a broken string or number is kept, and a value that cannot be read at all becomes `#!json null`.
+
+| Mistake                   | Repair                                                                         | Example                                  | Result                     |
+|---------------------------|--------------------------------------------------------------------------------|------------------------------------------|----------------------------|
+| missing `,` or `:`        | inserted                                                                       | `#!json [1 2]`, `#!json {"a" 1}`         | `[1,2]`, `{"a":1}`         |
+| missing value             | `#!json null` for an object key or between commas in an array                  | `#!json {"a":}`, `#!json [1,,2]`         | `{"a":null}`, `[1,null,2]` |
+| trailing comma            | removed                                                                        | `#!json [1,2,]`                          | `[1,2]`                    |
+| broken string             | invalid escapes and bytes are replaced (see below); a line break ends the string | `#!json ["a\qb"]`                        | `["aqb"]`                  |
+| broken number             | the longest valid beginning is kept                                            | `#!json [1., 2e+]`                       | `[1,2]`                    |
+| unreadable value          | `#!json null`                                                                  | `#!json [1, NaN, tru]`                   | `[1,null,null]`            |
+| number too large          | passed as infinity, together with its text                                     | `#!json [1e999]`                         | infinity (see below)       |
+| stray `:`                 | removed                                                                        | `#!json ["a":1]`                         | `["a",1]`                  |
+| member without a key      | skipped up to the next `,` or `}`                                              | `#!json {1:2, "b":3}`                    | `{"b":3}`                  |
+| wrong closing bracket     | closes the innermost array or object                                           | `#!json {"a":[1,2}, "b":3}`              | `{"a":[1,2],"b":3}`        |
+| input ends too early      | all open arrays and objects are closed                                         | `#!json {"a":[1,2`                       | `{"a":[1,2]}`              |
+| text before the value     | skipped                                                                        | `#!json )]}'{"a":1}`                     | `{"a":1}`                  |
+
+In a string, an unknown escape like `\q` stands for the escaped character (`q`), as in JavaScript. An invalid `\u`
+escape, a lone surrogate, and ill-formed UTF-8 are each replaced by U+FFFD (REPLACEMENT CHARACTER), and control
+characters are kept. A string without its closing quote ends at the next line break or at the end of the input.
+
+The input after the top-level value is not repaired: as without recovery, it is reported as an error, and parsing stops.
+
+## Binary formats
+
+The binary formats ([BJData](../binary_formats/bjdata.md), [BON8](../binary_formats/bon8.md),
+[BSON](../binary_formats/bson.md), [CBOR](../binary_formats/cbor.md), [MessagePack](../binary_formats/messagepack.md),
+and [UBJSON](../binary_formats/ubjson.md)) have no delimiters to find the next value by. So what can be repaired depends
+on whether the end of the item with the error is known, a distinction that
+[RFC 8949, Section 5.3](https://www.rfc-editor.org/rfc/rfc8949.html#section-5.3) makes for CBOR, too.
+
+If the item is complete, but cannot be passed on as it is, it is replaced, and parsing continues after it:
+
+| Mistake                                                             | Formats                 | Repair                                                                  |
+|---------------------------------------------------------------------|-------------------------|-------------------------------------------------------------------------|
+| tag                                                                 | CBOR                    | ignored                                                                 |
+| simple value other than `false`, `true`, and `null`, like undefined | CBOR                    | `#!json null`                                                           |
+| negative integer below the range of `number_integer_t`              | CBOR                    | the nearest floating-point number                                       |
+| character (`C`) that is not ASCII                                   | BJData, UBJSON          | U+FFFD                                                                  |
+| invalid high-precision number (`H`)                                 | BJData, UBJSON          | the longest valid beginning is kept, as for JSON text, or `#!json null` |
+| high-precision number too large                                     | BJData, UBJSON          | passed as infinity, together with its text                              |
+| object key that is not a string                                     | BON8, CBOR, MessagePack | the member is skipped                                                   |
+| element of a type the library does not read, like ObjectId or date  | BSON                    | `#!json null`                                                           |
+| string without its terminator                                       | BSON                    | kept                                                                    |
+| document whose size does not match its content                      | BSON                    | kept                                                                    |
+
+CBOR tags and simple values are repaired as [RFC 8949, Section 6.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-6.1)
+suggests for converting CBOR to JSON. Note that [`sax_parse`](../../api/basic_json/sax_parse.md) has no parameter for
+CBOR tags, so every tag is an error there; when recovering, tags are ignored like with
+[`cbor_tag_handler_t::ignore`](../../api/basic_json/cbor_tag_handler_t.md). Strings that are not valid UTF-8 are no
+error: like [`from_cbor`](../../api/basic_json/from_cbor.md) and the other functions by default, `sax_parse` passes
+them on as they are.
+
+After any other error, the end of the item is unknown: the input ended, a byte is not a valid type marker, or a size
+cannot be right. Parsing then stops, and the value read so far is completed: a key that waits for its value gets
+`#!json null`, and all open arrays and objects are closed. This keeps everything before the error of an input that was
+cut off. The exception is BSON, which stores the size of every document: an element whose end is unknown gets
+`#!json null`, the rest of its document is skipped, and parsing continues after the document.
+
+## Limitations
+
+- A repair is a guess. For example, `#!json {"a" "b": 1}` could be meant as `#!json {"a": "b"}` or as
+  `#!json {"a": null, "b": 1}`; it is repaired to the former. Treat recovered values as a best effort, and check the
+  reported errors.
+- A closing bracket always closes the innermost array or object. If a bracket is missing rather than wrong, the
+  repair differs from the intention: `#!json {"a": {"b": [1, 2}, "c": 3}` is repaired to
+  `#!json {"a": {"b": [1, 2], "c": 3}}`, although `#!json {"a": {"b": [1, 2]}, "c": 3}` may have been meant.
+- Keys without quotes, and strings in single quotes, are not supported; such members are skipped.
+- In the binary formats, a member that is skipped because its key is not a string is lost, and so are the elements of a
+  BSON document after one whose end is unknown.
+- A number that is too large for `number_float_t` is passed as positive or negative infinity. The SAX parser's
+  `number_float` also gets the number's text, but a JSON value cannot store it, and
+  [`dump`](../../api/basic_json/dump.md) serializes infinity as `#!json null`.
+- When parsing is not strict (see [`sax_parse`](../../api/basic_json/sax_parse.md)), a repair may read parts of the
+  input after the value, for instance of the next value in a stream of concatenated values.
+
+## See also
+
+- [SAX interface](sax_interface.md) - implement a custom SAX handler
+- [`parse_error`](../../api/json_sax/parse_error.md) - the SAX event for parse errors
+- [`sax_parse`](../../api/basic_json/sax_parse.md) - generate SAX events
+- [parsing and exceptions](parse_exceptions.md) - control error handling

docs/mkdocs/docs/features/parsing/index.md

@@ -75,7 +75,7 @@ You can influence a DOM parse without switching to the SAX interface by passing
 When the input is not valid JSON, the `parse` function throws an exception by default. If exceptions are undesired or
 unavailable, the parser can instead return a discarded value, or [`accept`](../../api/basic_json/accept.md) can be used
 to only check whether an input is valid JSON. See [parsing and exceptions](parse_exceptions.md) for the available
-options.
+options. To get as much as possible out of malformed input, a SAX parser can [recover from errors](error_recovery.md).
 
 ## See also
 
@@ -86,4 +86,5 @@ options.
 - [parser callbacks](parser_callbacks.md) - influence the parsing by a callback function
 - [SAX interface](sax_interface.md) - implement a custom SAX handler
 - [parsing and exceptions](parse_exceptions.md) - control error handling
+- [error recovery](error_recovery.md) - get as much as possible out of malformed input
 - [parsing untrusted input](untrusted_input.md) - what to consider when parsing input from untrusted sources

docs/mkdocs/docs/features/parsing/parse_exceptions.md

@@ -64,7 +64,8 @@ bool parse_error(std::size_t position,
                  const json::exception& ex);
 ```
 
-The return value indicates whether the parsing should continue, so the function should usually return `#!cpp false`.
+The return value decides whether to stop parsing (`#!cpp false`) or to repair the error and continue
+(`#!cpp true`); see [error recovery](error_recovery.md) for the latter.
 
 ??? example "Example: report parse errors without exceptions"
 

docs/mkdocs/docs/features/parsing/sax_interface.md

@@ -60,7 +60,8 @@ bool key(string_t& val);
 bool parse_error(std::size_t position, const std::string& last_token, const json::exception& ex);
 ```
 
-The return value of each function determines whether parsing should proceed.
+The return value of each function determines whether parsing should proceed. For `parse_error`, returning
+`#!cpp true` [recovers from the error](error_recovery.md).
 
 To implement your own SAX handler, proceed as follows:
 
@@ -68,7 +69,7 @@ To implement your own SAX handler, proceed as follows:
 2. Create an object of your SAX interface class, e.g. `my_sax`.
 3. Call `#!cpp bool json::sax_parse(input, &my_sax);` where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface.
 
-Note the `sax_parse` function only returns a `#!cpp bool` indicating the result of the last executed SAX event. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`.
+Note the `sax_parse` function only returns a `#!cpp bool` indicating whether the input was parsed without errors and no SAX event returned `#!cpp false`. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`.
 
 ## See also
 

docs/mkdocs/docs/home/exceptions.md

@@ -341,7 +341,10 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde
 
 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a
 string was read where one was required (for instance as a map key), the string's length specification is invalid, or
-the string's bytes are not valid UTF-8.
+the string's bytes are not valid UTF-8 and the `error_handler` parameter of the corresponding `from_*` function is
+set to `strict`. By default (`error_handler_t::keep`), the bytes of a string are not checked for valid UTF-8 on read;
+see the ill-formed UTF-8 notes on the individual [binary format](../features/binary_formats/index.md) pages for how
+such a string is handled depending on `error_handler`.
 
 CBOR and MessagePack allow map keys of any type, but JSON object keys are always strings. Maps with keys of any other
 type (for instance integers or `null`) are therefore not supported; see the notes on
@@ -749,6 +752,12 @@ The [`unflatten()`](../api/basic_json/unflatten.md) function only works for an o
 
 The [`dump()`](../api/basic_json/dump.md) function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. See the FAQ entry on [serializing untrusted or invalid UTF-8](faq.md#serializing-untrusted-or-invalid-utf-8) for background and the recommended fix.
 
+The binary writers [`to_cbor()`](../api/basic_json/to_cbor.md), [`to_ubjson()`](../api/basic_json/to_ubjson.md),
+[`to_bjdata()`](../api/basic_json/to_bjdata.md), and [`to_bson()`](../api/basic_json/to_bson.md) throw this exception
+as well for a string value or object key that is not valid UTF-8 if their `error_handler` is `strict` (the default if
+[`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md) is enabled). So does
+[`to_msgpack()`](../api/basic_json/to_msgpack.md) if `error_handler_t::strict` is passed.
+
 !!! failure "Example message"
 
     Calling `dump()` on a JSON value containing an ISO 8859-1 encoded string:

docs/mkdocs/docs/home/faq.md

@@ -305,6 +305,51 @@ Only very old NDKs (before r18), which defaulted to GCC and `gnustl`, lacked C++
 `std::to_string`. If you run into this, update to a current NDK.
 
 
+### Incomplete `detector` type with GCC < 11
+
+!!! question
+
+    Why does GCC 10 or older fail with `invalid use of incomplete type 'struct nlohmann::detail::detector<..., to_json_function, ...>'` for a type that holds an `optional` member?
+
+This happens with GCC 10 and older in C++11/C++14 mode when all of these hold:
+
+- a class `Holder` has an `optional<Dummy>` member (e.g., `boost::optional`),
+- `Dummy` has a constructor taking a `json` value, and
+- `to_json` for `Holder` is a free function in the namespace of `Dummy`.
+
+```cpp
+class Dummy {
+  public:
+    explicit Dummy(const nlohmann::json& j);
+};
+
+class Holder {
+    boost::optional<Dummy> d;
+};
+
+void to_json(nlohmann::json& j, const Holder& h);  // triggers the error
+```
+
+To decide whether `Dummy` is copyable, the compiler checks whether a `Dummy` can be converted to `json`. That check
+looks up `to_json` via argument-dependent lookup, finds the unrelated `to_json` for `Holder`, and eventually asks again
+whether `Dummy` is copyable. GCC before version 11 turns this cycle into a hard error; GCC 11 and later, Clang, and
+C++17 mode compile the code. The same error shows up without this library whenever a constrained converting constructor
+is involved, so the library can't avoid it.
+
+To work around this, define `to_json` (and `from_json`) as a *hidden friend* inside the class. That way,
+argument-dependent lookup only finds it for `Holder`:
+
+```cpp
+class Holder {
+    boost::optional<Dummy> d;
+
+    friend void to_json(nlohmann::json& j, const Holder& h) { /* ... */ }
+};
+```
+
+The [`NLOHMANN_DEFINE_TYPE_INTRUSIVE`](../api/macros/nlohmann_define_type_intrusive.md) macros define hidden friends as
+well. See [#3669](https://github.com/nlohmann/json/issues/3669) for details.
+
 ### Missing STL function
 
 !!! question "Questions"

docs/mkdocs/docs/integration/cmake.md

@@ -212,6 +212,11 @@ Use the non-amalgamated version of the library. This option is `ON` by default.
 
 Treat the library headers like system headers (i.e., adding `SYSTEM` to the [`target_include_directories`](https://cmake.org/cmake/help/latest/command/target_include_directories.html) call) to check for this library by tools like Clang-Tidy. This option is `OFF` by default.
 
+### `JSON_StrictBinaryUTF8`
+
+Check string values and object keys for valid UTF-8 in the CBOR, UBJSON, BJData, and BSON writers, by defining the
+macro [`JSON_STRICT_BINARY_UTF8`](../api/macros/json_strict_binary_utf8.md). This option is `OFF` by default.
+
 ### `JSON_StrictNulHandling`
 
 Reject a `'\0'` (NUL) byte in the input instead of treating it as end of input, by defining the macro

docs/mkdocs/docs/integration/migration_guide.md

@@ -2,11 +2,14 @@
 
 This page collects some guidelines on how to future-proof your code for future versions of this library. For how to
 add the library to your project in the first place, see [Integration](index.md), [CMake](cmake.md), or
-[Package Managers](package_managers.md).
+[Package Managers](package_managers.md). The [roadmap](../community/roadmap.md#version-40) lists what will change in
+version 4.0, including the macros that let you try its behavior with a 3.x release; this page describes how to adjust
+your code.
 
 ## Replace deprecated functions
 
-The following functions have been deprecated and will be removed in the next major version (i.e., 4.0.0). All
+The following functions have been deprecated and will be removed in the next major version (i.e., 4.0.0), see the
+[roadmap](../community/roadmap.md#removal-of-deprecated-functions) for an overview. All
 deprecations are annotated with
 [`HEDLEY_DEPRECATED_FOR`](https://nemequ.github.io/hedley/api-reference.html#HEDLEY_DEPRECATED_FOR) to report which
 function to use instead.

docs/mkdocs/mkdocs.yml

@@ -88,6 +88,7 @@ nav:
       - features/performance.md
       - Parsing:
           - features/parsing/index.md
+          - features/parsing/error_recovery.md
           - features/parsing/json_lines.md
           - features/parsing/parse_exceptions.md
           - features/parsing/parser_callbacks.md
@@ -116,6 +117,7 @@ nav:
           - 'accept': api/basic_json/accept.md
           - 'array': api/basic_json/array.md
           - 'array_t': api/basic_json/array_t.md
+          - 'as_base_class': api/basic_json/as_base_class.md
           - 'at': api/basic_json/at.md
           - 'back': api/basic_json/back.md
           - 'begin': api/basic_json/begin.md
@@ -306,6 +308,7 @@ nav:
           - 'JSON_PRECISE_STREAM_POSITION': api/macros/json_precise_stream_position.md
           - 'JSON_SKIP_LIBRARY_VERSION_CHECK': api/macros/json_skip_library_version_check.md
           - 'JSON_SKIP_UNSUPPORTED_COMPILER_CHECK': api/macros/json_skip_unsupported_compiler_check.md
+          - 'JSON_STRICT_BINARY_UTF8': api/macros/json_strict_binary_utf8.md
           - 'JSON_STRICT_NUL_HANDLING': api/macros/json_strict_nul_handling.md
           - 'JSON_USE_GLOBAL_UDLS': api/macros/json_use_global_udls.md
           - 'JSON_USE_IMPLICIT_CONVERSIONS': api/macros/json_use_implicit_conversions.md

include/nlohmann/detail/abi_macros.hpp

@@ -46,6 +46,10 @@
     #define JSON_STRICT_NUL_HANDLING 0
 #endif
 
+#ifndef JSON_STRICT_BINARY_UTF8
+    #define JSON_STRICT_BINARY_UTF8 0
+#endif
+
 #if JSON_DIAGNOSTICS
     #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag
 #else
@@ -82,14 +86,20 @@
     #define NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING
 #endif
 
+#if JSON_STRICT_BINARY_UTF8
+    #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 _sbu8
+#else
+    #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8
+#endif
+
 #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION
     #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0
 #endif
 
 // Construct the namespace ABI tags component
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) json_abi ## a ## b ## c ## d ## e ## f
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) json_abi ## a ## b ## c ## d ## e ## f ## g
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f) \
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f, g) \
-    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f)
+    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g)
 
 #define NLOHMANN_JSON_ABI_TAGS                                       \
     NLOHMANN_JSON_ABI_TAGS_CONCAT(                                   \
@@ -98,7 +108,8 @@
             NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS,              \
             NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS,         \
             NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION,           \
-            NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING)
+            NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING,               \
+            NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8)
 
 // Construct the namespace version component
 #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \

include/nlohmann/detail/conversions/from_json.hpp

@@ -27,7 +27,6 @@
 #include <nlohmann/detail/meta/identity_tag.hpp>
 #include <nlohmann/detail/meta/std_fs.hpp>
 #include <nlohmann/detail/meta/type_traits.hpp>
-#include <nlohmann/detail/meta/logic.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 #include <nlohmann/detail/value_t.hpp>
 
@@ -211,62 +210,29 @@ inline void from_json(const BasicJsonType& j, std::valarray<T>& l)
     });
 }
 
+// element is not itself a C array: read it directly
+template<typename BasicJsonType, typename T>
+auto from_json_c_array_element(const BasicJsonType& j, T& e)
+-> decltype(e = j.template get<T>(), void())
+{
+    e = j.template get<T>();
+}
+
+// element is itself a C array: recurse one dimension at a time, so any rank is supported
 template<typename BasicJsonType, typename T, std::size_t N>
-auto from_json(const BasicJsonType& j, T (&arr)[N])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+void from_json_c_array_element(const BasicJsonType& j, T (&arr)[N])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
--> decltype(j.template get<T>(), void())
 {
     for (std::size_t i = 0; i < N; ++i)
     {
-        arr[i] = j.at(i).template get<T>();
+        from_json_c_array_element(j.at(i), arr[i]);
     }
 }
 
-template<typename BasicJsonType, typename T, std::size_t N1, std::size_t N2>
+template<typename BasicJsonType, typename T, std::size_t N>
-auto from_json(const BasicJsonType& j, T (&arr)[N1][N2])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+auto from_json(const BasicJsonType& j, T (&arr)[N])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
--> decltype(j.template get<T>(), void())
+-> decltype(j.template get<typename std::remove_all_extents<T>::type>(), void())
 {
-    for (std::size_t i1 = 0; i1 < N1; ++i1)
+    from_json_c_array_element(j, arr);
-    {
-        for (std::size_t i2 = 0; i2 < N2; ++i2)
-        {
-            arr[i1][i2] = j.at(i1).at(i2).template get<T>();
-        }
-    }
-}
-
-template<typename BasicJsonType, typename T, std::size_t N1, std::size_t N2, std::size_t N3>
-auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
--> decltype(j.template get<T>(), void())
-{
-    for (std::size_t i1 = 0; i1 < N1; ++i1)
-    {
-        for (std::size_t i2 = 0; i2 < N2; ++i2)
-        {
-            for (std::size_t i3 = 0; i3 < N3; ++i3)
-            {
-                arr[i1][i2][i3] = j.at(i1).at(i2).at(i3).template get<T>();
-            }
-        }
-    }
-}
-
-template<typename BasicJsonType, typename T, std::size_t N1, std::size_t N2, std::size_t N3, std::size_t N4>
-auto from_json(const BasicJsonType& j, T (&arr)[N1][N2][N3][N4])  // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
--> decltype(j.template get<T>(), void())
-{
-    for (std::size_t i1 = 0; i1 < N1; ++i1)
-    {
-        for (std::size_t i2 = 0; i2 < N2; ++i2)
-        {
-            for (std::size_t i3 = 0; i3 < N3; ++i3)
-            {
-                for (std::size_t i4 = 0; i4 < N4; ++i4)
-                {
-                    arr[i1][i2][i3][i4] = j.at(i1).at(i2).at(i3).at(i4).template get<T>();
-                }
-            }
-        }
-    }
 }
 
 template<typename BasicJsonType>
@@ -286,20 +252,33 @@ auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr,
     }
 }
 
+// reserve() is called through this pair (modeled on from_json_object_reserve)
+// so from_json_array_impl below has a single body for both ConstructibleArrayType
+// that support reserve() and those that don't.
+template<typename ConstructibleArrayType>
+auto from_json_array_reserve(ConstructibleArrayType& arr, typename ConstructibleArrayType::size_type size, priority_tag<1> /*unused*/)
+-> decltype(arr.reserve(size), void())
+{
+    arr.reserve(size);
+}
+
+template<typename ConstructibleArrayType>
+void from_json_array_reserve(ConstructibleArrayType& /*arr*/, std::size_t /*size*/, priority_tag<0> /*unused*/)
+{}
+
 template<typename BasicJsonType, typename ConstructibleArrayType,
          enable_if_t<
              std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,
              int> = 0>
 auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/)
 -> decltype(
-    arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()),
     j.template get<typename ConstructibleArrayType::value_type>(),
     void())
 {
     using std::end;
 
     ConstructibleArrayType ret;
-    ret.reserve(j.size());
+    from_json_array_reserve(ret, j.size(), priority_tag<1> {});
     std::transform(j.begin(), j.end(),
                    std::inserter(ret, end(ret)), [](const BasicJsonType & i)
     {
@@ -310,27 +289,6 @@ auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, p
     arr = std::move(ret);
 }
 
-template<typename BasicJsonType, typename ConstructibleArrayType,
-         enable_if_t<
-             std::is_assignable<ConstructibleArrayType&, ConstructibleArrayType>::value,
-             int> = 0>
-inline void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr,
-                                 priority_tag<0> /*unused*/)
-{
-    using std::end;
-
-    ConstructibleArrayType ret;
-    std::transform(
-        j.begin(), j.end(), std::inserter(ret, end(ret)),
-        [](const BasicJsonType & i)
-    {
-        // get<BasicJsonType>() returns *this, this won't call a from_json
-        // method when value_type is BasicJsonType
-        return i.template get<typename ConstructibleArrayType::value_type>();
-    });
-    arr = std::move(ret);
-}
-
 template < typename BasicJsonType, typename ConstructibleArrayType,
            enable_if_t <
                is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value&&
@@ -433,9 +391,7 @@ inline void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)
 }
 
 // overload for arithmetic types, not chosen for basic_json template arguments
-// (BooleanType, etc.); note: Is it really necessary to provide explicit
+// (BooleanType, etc.)
-// overloads for boolean_t etc. in case of a custom BooleanType which is not
-// an arithmetic type?
 template < typename BasicJsonType, typename ArithmeticType,
            enable_if_t <
                std::is_arithmetic<ArithmeticType>::value&&
@@ -531,7 +487,7 @@ inline void from_json_tuple_impl(const BasicJsonType& j, std::pair<A1, A2>& p, p
 template<typename BasicJsonType, typename... Args>
 std::tuple<Args...> from_json_tuple_impl(const BasicJsonType& j, identity_tag<std::tuple<Args...>> /*unused*/, priority_tag<2> /*unused*/)
 {
-    static_assert(cxpr_and<cxpr_or<cxpr_not<std::is_reference<Args>>, is_compatible_reference_type<const BasicJsonType&, Args>>...>::value,
+    static_assert(conjunction<disjunction<negation<std::is_reference<Args>>, is_compatible_reference_type<const BasicJsonType&, Args>>...>::value,
                   "Can not return a tuple containing references to types not contained in a Json, try Json::get_to()");
     return from_json_tuple_impl_base<1, Args...>(j, index_sequence_for<Args...> {});
 }
@@ -554,10 +510,10 @@ auto from_json(const BasicJsonType& j, TupleRelated&& t)
     return from_json_tuple_impl(j, std::forward<TupleRelated>(t), priority_tag<3> {});
 }
 
-template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+// shared body for std::map/std::unordered_map with a non-string Key: both
-           typename = enable_if_t < !std::is_constructible <
+// containers are read from an array of [key, value] pairs the same way
-                                        typename BasicJsonType::string_t, Key >::value >>
+template<typename BasicJsonType, typename MapType>
-inline void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
+void from_json_pair_array_to_map(const BasicJsonType& j, MapType& m)
 {
     if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
     {
@@ -570,33 +526,29 @@ inline void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allo
         {
             JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &p));
         }
-        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
+        m.emplace(p.at(0).template get<typename MapType::key_type>(), p.at(1).template get<typename MapType::mapped_type>());
     }
 }
 
+template < typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator,
+           typename = enable_if_t < !std::is_constructible <
+                                        typename BasicJsonType::string_t, Key >::value >>
+void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m)
+{
+    from_json_pair_array_to_map(j, m);
+}
+
 template < typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator,
            typename = enable_if_t < !std::is_constructible <
                                         typename BasicJsonType::string_t, Key >::value >>
-inline void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
+void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m)
 {
-    if (JSON_HEDLEY_UNLIKELY(!j.is_array()))
+    from_json_pair_array_to_map(j, m);
-    {
-        JSON_THROW(type_error::create(302, concat("type must be array, but is ", j.type_name()), &j));
-    }
-    m.clear();
-    for (const auto& p : j)
-    {
-        if (JSON_HEDLEY_UNLIKELY(!p.is_array()))
-        {
-            JSON_THROW(type_error::create(302, concat("type must be array, but is ", p.type_name()), &p));
-        }
-        m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>());
-    }
 }
 
 #if JSON_HAS_FILESYSTEM || JSON_HAS_EXPERIMENTAL_FILESYSTEM
 
-// Workaround for MSVC 19.51 (and possibly later): in large in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996)
+// Workaround for MSVC 19.51 (and possibly later): in large cpp files, the compiler may fail to resolve with generic has_from_json (issue #4996)
 template<typename BasicJsonType>
 struct has_from_json<BasicJsonType, std_fs::path, void> : std::true_type {};
 

include/nlohmann/detail/conversions/to_json.hpp

@@ -178,7 +178,7 @@ struct external_constructor<value_t::array>
 
     template < typename BasicJsonType, typename CompatibleArrayType,
                enable_if_t < !std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
                              && !is_compatible_range_view<CompatibleArrayType>::value
 #endif
                              , int > = 0 >
@@ -222,9 +222,7 @@ struct external_constructor<value_t::array>
         j.assert_invariant();
     }
 
-    // std::ranges does not work properly on MinGW due to incomplete C++20 support
+#if JSON_HAS_RANGE_VIEW_CONVERSION
-    // see https://github.com/nlohmann/json/issues/4916
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
     template<typename BasicJsonType, typename CompatibleArrayType,
              enable_if_t<is_compatible_range_view<std::remove_cvref_t<CompatibleArrayType>>::value, int> = 0>
     static void construct(BasicJsonType& j, CompatibleArrayType && arr)
@@ -294,7 +292,9 @@ void to_json(BasicJsonType& j, const std::optional<T>& opt) noexcept(std::is_not
 {
     if (opt.has_value())
     {
-        j = *opt;
+        // explicit construction, as the conversion from a basic_json with a different
+        // string type is explicit if JSON_USE_IMPLICIT_CONVERSIONS is 0 (#2649)
+        j = BasicJsonType(*opt);
     }
     else
     {
@@ -382,7 +382,7 @@ template < typename BasicJsonType, typename CompatibleArrayType,
                          !std::is_same<typename BasicJsonType::binary_t, CompatibleArrayType>::value&&
                          !is_compatible_binary_type<BasicJsonType, CompatibleArrayType>::value&&
                          !is_basic_json<CompatibleArrayType>::value
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
     && !is_compatible_range_view<CompatibleArrayType>::value
 #endif
                          ,
@@ -392,7 +392,7 @@ inline void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
     external_constructor<value_t::array>::construct(j, arr);
 }
 
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
 template < typename BasicJsonType, typename T,
            enable_if_t < is_compatible_range_view<std::remove_cvref_t<T>>::value
                          && !is_compatible_string_type<BasicJsonType, std::remove_cvref_t<T>>::value

include/nlohmann/detail/exceptions.hpp

@@ -286,6 +286,27 @@ class other_error : public exception
     other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
 };
 
+/*!
+@brief helper function to call JSON_THROW from a template
+@note JSON_THROW is a macro that, depending on the JSON_THROW_USER /
+      JSON_TRY_USER / JSON_NOEXCEPTION configuration, may expand to code
+      that does not reference its argument (e.g. `std::abort()`), which
+      would trigger a compilation error if the argument's type depends on
+      a template parameter that is otherwise unused. Wrapping the call in
+      a templated function avoids this and gives the compiler a single
+      place to see the (possibly unused) parameter.
+*/
+template<typename ExceptionType>
+void templated_json_throw(ExceptionType exception)
+{
+    JSON_THROW(exception);
+
+    // JSON_THROW may expand to code that discards its argument (e.g. when
+    // exceptions are disabled) - the cast below avoids an unused-parameter
+    // warning with -Werror in that case
+    (void)exception;
+}
+
 }  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END
 

include/nlohmann/detail/input/input_adapters.hpp

@@ -453,8 +453,10 @@ struct wide_string_input_helper<BaseInputAdapter, 4>
         }
         else
         {
-            // get the current character
+            // get the current character; converted to an unsigned type so that
-            const auto wc = input.get_character();
+            // a negative unit (wint_t is signed on some platforms) is not
+            // mistaken for an ASCII character or for EOF
+            const auto wc = static_cast<std::uint32_t>(input.get_character());
 
             if (wc <= 0x10FFFF)
             {
@@ -522,9 +524,11 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
                 bool valid_pair = false;
                 if (wc <= 0xDBFF && JSON_HEDLEY_UNLIKELY(!input.empty()))
                 {
-                    const auto wc2 = static_cast<unsigned int>(input.get_character());
+                    // only consume the next unit if it completes the pair
+                    const auto wc2 = static_cast<unsigned int>(*input.current);
                     if (0xDC00 <= wc2 && wc2 <= 0xDFFF)
                     {
+                        input.get_character();
                         const auto charcode = 0x10000u + (((static_cast<unsigned int>(wc) & 0x3FFu) << 10u) | (wc2 & 0x3FFu));
                         utf8_bytes_filled = 0;
                         encode_utf8(charcode, [&utf8_bytes, &utf8_bytes_filled](std::uint32_t byte)
@@ -537,7 +541,8 @@ struct wide_string_input_helper<BaseInputAdapter, 2>
 
                 if (!valid_pair)
                 {
-                    utf8_bytes[0] = static_cast<std::char_traits<char>::int_type>(wc);
+                    // emit a byte that is never valid UTF-8 (see the UTF-32 case)
+                    utf8_bytes[0] = 0xFF;
                     utf8_bytes_filled = 1;
                 }
             }
@@ -746,7 +751,7 @@ struct container_input_adapter_factory< ContainerType,
 {
     // container is forwarded twice on purpose: the resulting begin/end
     // iterator types must match adapter_type, computed the same way
-    // NOLINTNEXTLINE(bugprone-use-after-move)
+    // NOLINTNEXTLINE(bugprone-use-after-move,hicpp-invalid-access-moved)
     return input_adapter(begin(std::forward<ContainerType>(container)), end(std::forward<ContainerType>(container)));
 }
        };

include/nlohmann/detail/input/json_sax.hpp

@@ -132,7 +132,9 @@ struct json_sax
     @param[in] position    the position in the input where the error occurs
     @param[in] last_token  the last read token
     @param[in] ex          an exception object describing the error
-    @return whether parsing should proceed (must return false)
+    @return whether to recover from the error: false stops parsing; true
+            repairs the error and continues, or, if that is not possible,
+            stops after completing the value read so far
     */
     virtual bool parse_error(std::size_t position,
                              const std::string& last_token,
@@ -270,8 +272,11 @@ After successful parsing, the value that is passed by reference to the
 constructor contains the parsed value.
 
 @tparam BasicJsonType     the JSON type
+@tparam InputAdapterType  the input adapter of the lexer that can be passed to
+                          the constructor to record diagnostic positions; it
+                          does not matter if no lexer is passed
 */
-template<typename BasicJsonType, typename InputAdapterType>
+template<typename BasicJsonType, typename InputAdapterType = string_input_adapter_type>
 class json_sax_dom_parser
 {
   public:
@@ -518,7 +523,7 @@ class json_sax_dom_parser
     lexer_t* m_lexer_ref = nullptr;
 };
 
-template<typename BasicJsonType, typename InputAdapterType>
+template<typename BasicJsonType, typename InputAdapterType = string_input_adapter_type>
 class json_sax_dom_callback_parser
 {
   public:

include/nlohmann/detail/input/lexer.hpp

@@ -10,7 +10,7 @@
 
 #include <array> // array
 #include <cstddef> // size_t
-#include <cstdint> // uint32_t
+#include <cstdint> // uint8_t, uint32_t
 #include <cstdio> // snprintf
 #include <initializer_list> // initializer_list
 #include <string> // char_traits, string
@@ -439,8 +439,16 @@ class lexer : public lexer_base<BasicJsonType>
                             if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF)
                             {
                                 // expect next \uxxxx entry
-                                if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u'))
+                                if (JSON_HEDLEY_LIKELY(get() == '\\'))
                                 {
+                                    if (JSON_HEDLEY_UNLIKELY(get() != 'u'))
+                                    {
+                                        // current is the character escaped by the backslash
+                                        error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+                                        string_error_resume = resume_kind::escaped_character;
+                                        return token_type::parse_error;
+                                    }
+
                                     const int codepoint2 = get_codepoint();
 
                                     if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1))
@@ -465,7 +473,11 @@ class lexer : public lexer_base<BasicJsonType>
                                     }
                                     else
                                     {
+                                        // the second escape was read completely and is a
+                                        // code point of its own
                                         error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+                                        string_error_resume = resume_kind::after_escape;
+                                        string_error_codepoint = codepoint2;
                                         return token_type::parse_error;
                                     }
                                 }
@@ -479,7 +491,9 @@ class lexer : public lexer_base<BasicJsonType>
                             {
                                 if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF))
                                 {
+                                    // the escape was read completely
                                     error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    string_error_resume = resume_kind::after_escape;
                                     return token_type::parse_error;
                                 }
                             }
@@ -941,10 +955,15 @@ class lexer : public lexer_base<BasicJsonType>
                         case '\n':
                         case '\r':
                         case char_traits<char_type>::eof():
+                            return true;
+
 #if !JSON_STRICT_NUL_HANDLING
                         case '\0':
-#endif
+                            // a NUL byte is the end of the input (see scan()),
+                            // so leave it for scan() to see
+                            unget();
                             return true;
+#endif
 
                         default:
                             break;
@@ -2095,6 +2114,573 @@ scan_number_done:
         }
     }
 
+    /////////////////////
+    // error recovery
+    /////////////////////
+
+    /*!
+    @brief make the best of the token that scan() rejected
+
+    Called by the parser after scan() returned token_type::parse_error and the
+    SAX parser asked to recover from the error (see #3989). Keeps what can be
+    read of the token and skips the rest:
+
+    - A string keeps its characters. An unknown escape stands for the escaped
+      character itself (as in JavaScript), an invalid Unicode escape and ill-formed
+      UTF-8 become U+FFFD, and a control character is kept. A line break or the
+      end of the input ends a string that lacks its closing quote.
+    - A number keeps its longest valid prefix, e.g. `1` for `1.` or `1e+`.
+    - A block comment that is not closed runs to the end of the input.
+    - Anything else is skipped.
+
+    The rest of an invalid token is skipped up to the next delimiter
+    (whitespace, a structural character, or a quote). A delimiter that the
+    invalid token consumed is returned to the input, so that the next scan()
+    reads it.
+
+    @return token_type::value_string or a number token type if a string or a
+            number could be read, token_type::end_of_input for a block comment
+            that is not closed, token_type::uninitialized otherwise
+    */
+    token_type recover_token()
+    {
+        const resume_kind resume = string_error_resume;
+        const int codepoint = string_error_codepoint;
+        string_error_resume = resume_kind::character;
+        string_error_codepoint = -1;
+
+        if (error_message_starts_with("invalid string"))
+        {
+            return recover_string(resume, codepoint);
+        }
+
+        if (error_message_starts_with("invalid number"))
+        {
+            return recover_number();
+        }
+
+        if (error_message_starts_with("invalid comment; missing"))
+        {
+            // the comment runs to the end of the input
+            return token_type::end_of_input;
+        }
+
+        skip_to_delimiter();
+        return token_type::uninitialized;
+    }
+
+    /*!
+    @brief return the token that scan() read last to the input, so that the
+           next scan() reads it again
+
+    Called by the parser when recovering from an error. The token must be a
+    single character (',', ':', '[', ']', '{', or '}') or the end of the
+    input, and scan() must have read it last.
+    */
+    void unget_token()
+    {
+        JSON_ASSERT(!next_unget);
+        unget();
+    }
+
+    /*!
+    @brief let the token string for the next error begin at the current character
+
+    The token string of an error reaches back to the beginning of the last
+    string or number. After an error, the parser calls this function so that
+    the next error does not report (and, with many errors, copy) everything
+    read since then.
+    */
+    void restart_token_string()
+    {
+        restart_token_string_impl(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+  private:
+    /// how recover_string() continues after the error scan_string() reported
+    enum class resume_kind : std::uint8_t
+    {
+        /// current is the next character of the string (or the end of input)
+        character,
+        /// current is the character escaped by the preceding backslash
+        escaped_character,
+        /// current is the last character of a complete escape
+        after_escape
+    };
+
+    /// whether error_message begins with @a prefix
+    bool error_message_starts_with(const char* prefix) const noexcept
+    {
+        const char* message = error_message;
+        while (*prefix != '\0')
+        {
+            if (*message++ != *prefix++)
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /// whether current ends an invalid token (see recover_token())
+    bool current_is_delimiter() const noexcept
+    {
+        switch (current)
+        {
+            case ' ':
+            case '\t':
+            case '\n':
+            case '\r':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case ',':
+            case ':':
+            case '\"':
+#if !JSON_STRICT_NUL_HANDLING
+            case '\0':
+#endif
+            case char_traits<char_type>::eof():
+                return true;
+
+            case '/':
+                return ignore_comments;
+
+            default:
+                return false;
+        }
+    }
+
+    /// skip the rest of an invalid token and return its delimiter to the input
+    void skip_to_delimiter()
+    {
+        while (!current_is_delimiter())
+        {
+            get();
+        }
+
+        if (current != char_traits<char_type>::eof())
+        {
+            unget();
+        }
+    }
+
+    /// append U+FFFD REPLACEMENT CHARACTER to token_buffer
+    void add_replacement_character()
+    {
+        add(0xEF);
+        add(0xBF);
+        add(0xBD);
+    }
+
+    /// append the UTF-8 encoding of @a codepoint (not a surrogate) to token_buffer
+    void add_codepoint(const int codepoint)
+    {
+        JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
+        const auto cp = static_cast<unsigned int>(codepoint);
+        if (cp < 0x80)
+        {
+            add(static_cast<char_int_type>(cp));
+        }
+        else if (cp <= 0x7FF)
+        {
+            add(static_cast<char_int_type>(0xC0u | (cp >> 6u)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+        else if (cp <= 0xFFFF)
+        {
+            add(static_cast<char_int_type>(0xE0u | (cp >> 12u)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 6u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+        else
+        {
+            add(static_cast<char_int_type>(0xF0u | (cp >> 18u)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 12u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 6u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+    }
+
+    /// append a code point read from a Unicode escape; a surrogate becomes U+FFFD
+    void add_escaped_codepoint(const int codepoint)
+    {
+        if (0xD800 <= codepoint && codepoint <= 0xDFFF)
+        {
+            add_replacement_character();
+        }
+        else
+        {
+            add_codepoint(codepoint);
+        }
+    }
+
+    /*!
+    @brief remove an incomplete UTF-8 sequence from the end of token_buffer
+
+    next_byte_in_range() adds the bytes of a sequence as it checks them, so
+    when it rejects a byte, the beginning of the sequence is already in
+    token_buffer, which otherwise holds only complete sequences.
+
+    @return whether an incomplete sequence was removed
+    */
+    bool remove_incomplete_utf8_sequence()
+    {
+        std::size_t lead = token_buffer.size();
+        std::size_t continuation_bytes = 0;
+        while (lead > 0 && continuation_bytes < 3
+                && (static_cast<unsigned char>(token_buffer[lead - 1]) & 0xC0u) == 0x80u)
+        {
+            --lead;
+            ++continuation_bytes;
+        }
+        if (lead == 0)
+        {
+            return false;
+        }
+
+        const auto lead_byte = static_cast<unsigned char>(token_buffer[lead - 1]);
+        std::size_t expected = 0;
+        if (lead_byte >= 0xF0)
+        {
+            expected = 3;
+        }
+        else if (lead_byte >= 0xE0)
+        {
+            expected = 2;
+        }
+        else if (lead_byte >= 0xC0)
+        {
+            expected = 1;
+        }
+        if (continuation_bytes >= expected)
+        {
+            return false;
+        }
+
+        token_buffer.resize(lead - 1);
+        return true;
+    }
+
+    /*!
+    @brief read the UTF-8 sequence that begins with current, which is not ASCII
+    @return whether the next character must be read; false if current still
+            needs to be handled, because it does not belong to the sequence
+    */
+    bool recover_utf8_sequence()
+    {
+        // the number of continuation bytes and the range of the first one;
+        // see the ranges in scan_string()
+        std::size_t count = 0;
+        char_int_type low = 0x80;
+        char_int_type high = 0xBF;
+        if (current >= 0xC2 && current <= 0xDF)
+        {
+            count = 1;
+        }
+        else if (current >= 0xE0 && current <= 0xEF)
+        {
+            count = 2;
+            low = (current == 0xE0) ? 0xA0 : 0x80;
+            high = (current == 0xED) ? 0x9F : 0xBF;
+        }
+        else if (current >= 0xF0 && current <= 0xF4)
+        {
+            count = 3;
+            low = (current == 0xF0) ? 0x90 : 0x80;
+            high = (current == 0xF4) ? 0x8F : 0xBF;
+        }
+        else
+        {
+            // an ill-formed byte
+            add_replacement_character();
+            return true;
+        }
+
+        const std::size_t start = token_buffer.size();
+        add(current);
+        for (std::size_t i = 0; i < count; ++i)
+        {
+            get();
+            if (current < low || current > high)
+            {
+                token_buffer.resize(start);
+                add_replacement_character();
+                return false;
+            }
+            add(current);
+            low = 0x80;
+            high = 0xBF;
+        }
+        return true;
+    }
+
+    /*!
+    @brief read the low surrogate that must follow the high surrogate @a high
+    @return whether the next character must be read; false if current still
+            needs to be handled
+    */
+    bool recover_low_surrogate(int high)
+    {
+        while (true)
+        {
+            if (get() != '\\')
+            {
+                add_replacement_character();
+                return false;
+            }
+            if (get() != 'u')
+            {
+                add_replacement_character();
+                // not 'u', so this does not come back here
+                return recover_escape();
+            }
+
+            const int low = get_codepoint();
+            if (low == -1)
+            {
+                add_replacement_character();
+                return false;
+            }
+            if (0xDC00 <= low && low <= 0xDFFF)
+            {
+                add_codepoint(static_cast<int>((static_cast<unsigned int>(high) << 10u)
+                                               + static_cast<unsigned int>(low) - 0x35FDC00u));
+                return true;
+            }
+
+            // high has no low surrogate
+            add_replacement_character();
+            if (low < 0xD800 || low > 0xDBFF)
+            {
+                add_codepoint(low);
+                return true;
+            }
+            // another high surrogate
+            high = low;
+        }
+    }
+
+    /*!
+    @brief read the escape whose backslash was read; current is the escaped character
+    @return whether the next character must be read; false if current still
+            needs to be handled
+    */
+    bool recover_escape()
+    {
+        switch (current)
+        {
+            case '\"':
+                add('\"');
+                return true;
+            case '\\':
+                add('\\');
+                return true;
+            case '/':
+                add('/');
+                return true;
+            case 'b':
+                add('\b');
+                return true;
+            case 'f':
+                add('\f');
+                return true;
+            case 'n':
+                add('\n');
+                return true;
+            case 'r':
+                add('\r');
+                return true;
+            case 't':
+                add('\t');
+                return true;
+
+            case 'u':
+            {
+                const int codepoint = get_codepoint();
+                if (codepoint == -1)
+                {
+                    add_replacement_character();
+                    return false;
+                }
+                if (0xD800 <= codepoint && codepoint <= 0xDBFF)
+                {
+                    return recover_low_surrogate(codepoint);
+                }
+                add_escaped_codepoint(codepoint);
+                return true;
+            }
+
+            // an unknown escape stands for the escaped character
+            default:
+                return false;
+        }
+    }
+
+    /*!
+    @brief read the rest of a string after scan_string() rejected it
+
+    token_buffer holds what scan_string() read before the error. See
+    recover_token() for how errors are repaired.
+
+    @param[in] resume     how to continue, see resume_kind
+    @param[in] codepoint  for a high surrogate followed by an escape of another
+                          code point: that code point; -1 otherwise
+    */
+    token_type recover_string(const resume_kind resume, const int codepoint)
+    {
+        // whether the next character must be read before it can be handled
+        bool fetch = false;
+
+        if (error_message_starts_with("invalid string: surrogate")
+                || error_message_starts_with("invalid string: '\\u'")
+                || (error_message_starts_with("invalid string: ill-formed UTF-8")
+                    && remove_incomplete_utf8_sequence()))
+        {
+            add_replacement_character();
+        }
+
+        switch (resume)
+        {
+            case resume_kind::escaped_character:
+                fetch = recover_escape();
+                break;
+            case resume_kind::after_escape:
+                if (0xD800 <= codepoint && codepoint <= 0xDBFF)
+                {
+                    fetch = recover_low_surrogate(codepoint);
+                }
+                else
+                {
+                    if (codepoint != -1)
+                    {
+                        add_escaped_codepoint(codepoint);
+                    }
+                    fetch = true;
+                }
+                break;
+            case resume_kind::character:
+            default:
+                break;
+        }
+
+        while (true)
+        {
+            if (fetch)
+            {
+                get();
+            }
+            fetch = true;
+
+            switch (current)
+            {
+                case '\"':
+                // a line break or the end of the input ends a string that
+                // lacks its closing quote
+                case '\n':
+                case '\r':
+                case char_traits<char_type>::eof():
+                    return token_type::value_string;
+
+#if !JSON_STRICT_NUL_HANDLING
+                case '\0':
+                    // the end of the input, see scan()
+                    unget();
+                    return token_type::value_string;
+#endif
+
+                case '\\':
+                    get();
+                    fetch = recover_escape();
+                    break;
+
+                default:
+                    if (current < 0x80)
+                    {
+                        // including control characters
+                        add(current);
+                    }
+                    else
+                    {
+                        fetch = recover_utf8_sequence();
+                    }
+                    break;
+            }
+        }
+    }
+
+    /*!
+    @brief keep the longest valid prefix of a number that scan_number() rejected
+
+    token_buffer holds the characters scan_number() accepted before the error,
+    so the prefix ends at its last digit.
+    */
+    token_type recover_number()
+    {
+        // only size(), operator[], and resize() are used, which every string
+        // type the library supports provides
+        std::size_t length = token_buffer.size();
+        while (length != 0 && (token_buffer[length - 1] < '0' || token_buffer[length - 1] > '9'))
+        {
+            --length;
+        }
+        token_buffer.resize(length);
+
+        if (length == 0)
+        {
+            skip_to_delimiter();
+            return token_type::uninitialized;
+        }
+
+        if (decimal_point_position >= length)
+        {
+            decimal_point_position = std::string::npos;
+        }
+
+        std::size_t exponent = std::string::npos;
+        for (std::size_t i = 0; i < length; ++i)
+        {
+            if (token_buffer[i] == 'e' || token_buffer[i] == 'E')
+            {
+                exponent = i;
+                break;
+            }
+        }
+        const std::size_t mantissa_end = (exponent == std::string::npos) ? length : exponent;
+        token_type number_type = token_type::value_unsigned;
+        if (decimal_point_position != std::string::npos || exponent != std::string::npos)
+        {
+            number_type = token_type::value_float;
+        }
+        else if (token_buffer[0] == '-')
+        {
+            number_type = token_type::value_integer;
+        }
+
+        const token_type result = convert_number(number_type, mantissa_end);
+        skip_to_delimiter();
+        return result;
+    }
+
+    /// seekable adapter: the token string begins at current, which was consumed
+    void restart_token_string_impl(std::true_type /*lazy*/) noexcept
+    {
+        const std::size_t consumed = ia.get_consumed_count();
+        token_string_start = (consumed > 0 && current != char_traits<char_type>::eof()) ? consumed - 1 : consumed;
+    }
+
+    /// streaming adapter: the token string begins at current; a character
+    /// that was put back is copied again when it is read again
+    void restart_token_string_impl(std::false_type /*lazy*/)
+    {
+        token_string.clear();
+        if (!next_unget && current != char_traits<char_type>::eof())
+        {
+            token_string.push_back(char_traits<char_type>::to_char_type(current));
+        }
+    }
+
   private:
     /// input adapter
     InputAdapterType ia;
@@ -2135,6 +2721,13 @@ scan_number_done:
     /// a description of occurred lexer errors
     const char* error_message = "";
 
+    /// how recover_token() continues a string that scan_string() rejected;
+    /// set only on the error paths that need more than error_message
+    resume_kind string_error_resume = resume_kind::character;
+    /// the code point of the second escape when a high surrogate is followed
+    /// by an escape that is not a low surrogate; -1 otherwise
+    int string_error_codepoint = -1;
+
     // number values
     number_integer_t value_integer = 0;
     number_unsigned_t value_unsigned = 0;

include/nlohmann/detail/input/parser.hpp

@@ -139,26 +139,59 @@ class parser
     bool accept(const bool strict = true)
     {
         json_sax_acceptor<BasicJsonType> sax_acceptor;
-        return sax_parse(&sax_acceptor, strict);
+        return sax_parse_impl<false>(&sax_acceptor, strict);
     }
 
+    /*!
+    @brief public SAX interface
+
+    If the SAX parser's parse_error() returns true, the parser recovers from
+    the error: it repairs the input and continues (see #3989).
+
+    @param[in] sax     the SAX parser
+    @param[in] strict  whether to expect the last token to be EOF
+    @return whether the input was parsed without errors and no SAX event
+            returned false
+    */
     template<typename SAX>
     JSON_HEDLEY_NON_NULL(2)
     bool sax_parse(SAX* sax, const bool strict = true)
+    {
+        return sax_parse_impl<true>(sax, strict);
+    }
+
+  private:
+    /// what sax_parse_internal() does after an object key was expected
+    enum class next_step : std::uint8_t
+    {
+        /// stop parsing
+        stop,
+        /// parse a value that begins with last_token
+        parse_value,
+        /// evaluate the state of the innermost container, which reads
+        /// last_token again
+        evaluate_state
+    };
+
+    template<bool AllowRecovery, typename SAX>
+    JSON_HEDLEY_NON_NULL(2)
+    bool sax_parse_impl(SAX* sax, const bool strict)
     {
         (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
-        const bool result = sax_parse_internal(sax);
+        const bool result = sax_parse_internal<AllowRecovery>(sax);
 
         if (result)
         {
             if (strict)
             {
-                // strict mode: next byte must be EOF
+                // strict mode: next byte must be EOF; after recovering from an
-                if (get_token() != token_type::end_of_input)
+                // error, the end of the input may already have been read
+                if (last_token != token_type::end_of_input && get_token() != token_type::end_of_input)
                 {
-                    return sax->parse_error(m_lexer.get_position(),
+                    // the value is complete, so there is nothing to recover
-                                            m_lexer.get_token_string(),
+                    static_cast<void>(report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr),
-                                            parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr));
+                                                   std::integral_constant<bool, AllowRecovery> {}));
+                    return false;
                 }
             }
             else
@@ -169,10 +202,9 @@ class parser
             }
         }
 
-        return result;
+        return result && !error_reported;
     }
 
-  private:
     /*!
     @brief run a DOM SAX parser to completion and position the lexer
 
@@ -190,7 +222,7 @@ class parser
     template<typename DomSax>
     bool parse_dom(DomSax& sdp, const bool strict)
     {
-        sax_parse_internal(&sdp);
+        sax_parse_internal<false>(&sdp);
 
         if (strict)
         {
@@ -213,10 +245,20 @@ class parser
         return !sdp.is_errored();
     }
 
-    template<typename SAX>
+    /*!
+    @brief parse a JSON value and pass it to a SAX parser
+
+    @tparam AllowRecovery  whether to recover from an error if the SAX parser's
+                           parse_error() returns true; false for the SAX parsers
+                           of parse() and accept(), which never do, so that no
+                           code for recovering is generated for them
+    */
+    template<bool AllowRecovery, typename SAX>
     JSON_HEDLEY_NON_NULL(2)
     bool sax_parse_internal(SAX* sax)
     {
+        const std::integral_constant<bool, AllowRecovery> allow_recovery{};
+
         // stack to remember the hierarchy of structured values we are parsing
         // true = array; false = object
         std::vector<bool> states;
@@ -247,12 +289,18 @@ class parser
                             break;
                         }
 
-                        // parse key
+                        // remember we are now inside an object
+                        states.push_back(false);
+
+                        // parse key (the steps of parse_key(), which are
+                        // repeated here and below for speed)
                         if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
+                            if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation))
-                                                    m_lexer.get_token_string(),
+                            {
-                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+                                return false;
+                            }
+                            continue;
                         }
                         if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
                         {
@@ -262,13 +310,12 @@ class parser
                         // parse separator (:)
                         if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
+                            if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation))
-                                                    m_lexer.get_token_string(),
+                            {
-                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+                                return false;
+                            }
+                            continue;
                         }
-
-                        // remember we are now inside an object
-                        states.push_back(false);
 
                         // parse values
                         get_token();
@@ -305,9 +352,11 @@ class parser
 
                         if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
+                            if (!overflow_error(sax, res, allow_recovery))
-                                                    m_lexer.get_token_string(),
+                            {
-                                                    out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr));
+                                return false;
+                            }
+                            break;
                         }
 
                         if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))
@@ -375,23 +424,63 @@ class parser
                     case token_type::parse_error:
                     {
                         // using "uninitialized" to avoid an "expected" message
-                        return sax->parse_error(m_lexer.get_position(),
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr), allow_recovery))
-                                                m_lexer.get_token_string(),
+                        {
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));
+                            return false;
+                        }
+
+                        // recover: keep what can be read of the token
+                        recover_token();
+                        if (last_token != token_type::uninitialized)
+                        {
+                            // a string or a number
+                            continue;
+                        }
+                        if (states.empty())
+                        {
+                            // look for the value after the garbage
+                            if (!skip_to_value())
+                            {
+                                return false;
+                            }
+                            continue;
+                        }
+                        // nothing could be read
+                        if (JSON_HEDLEY_UNLIKELY(!sax->null()))
+                        {
+                            return false;
+                        }
+                        break;
                     }
                     case token_type::end_of_input:
                     {
                         if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
+                            // there is nothing to recover
-                                                    m_lexer.get_token_string(),
+                            static_cast<void>(report_error(sax, parse_error::create(101, m_lexer.get_position(),
-                                                    parse_error::create(101, m_lexer.get_position(),
+                                                           "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr), allow_recovery));
-                                                            "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));
+                            return false;
                         }
 
-                        return sax->parse_error(m_lexer.get_position(),
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery))
-                                                m_lexer.get_token_string(),
+                        {
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+                            return false;
+                        }
+
+                        // recover: the input ends where a value is missing
+                        if (states.empty())
+                        {
+                            // there is no value
+                            return false;
+                        }
+                        if (!recover_missing_value(sax, states))
+                        {
+                            return false;
+                        }
+                        // the state evaluation reads the token again
+                        m_lexer.unget_token();
+                        skip_to_state_evaluation = true;
+                        continue;
                     }
                     case token_type::uninitialized:
                     case token_type::end_array:
@@ -401,9 +490,35 @@ class parser
                     case token_type::literal_or_value:
                     default: // the last token was unexpected
                     {
-                        return sax->parse_error(m_lexer.get_position(),
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery))
-                                                m_lexer.get_token_string(),
+                        {
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+                            return false;
+                        }
+
+                        // recover
+                        if (states.empty())
+                        {
+                            // look for the value after the garbage
+                            if (!skip_to_value())
+                            {
+                                return false;
+                            }
+                            continue;
+                        }
+                        if (last_token == token_type::name_separator)
+                        {
+                            // a stray ':'; the value may follow
+                            get_token();
+                            continue;
+                        }
+                        if (!recover_missing_value(sax, states))
+                        {
+                            return false;
+                        }
+                        // the state evaluation reads the token again
+                        m_lexer.unget_token();
+                        skip_to_state_evaluation = true;
+                        continue;
                     }
                 }
             }
@@ -454,9 +569,30 @@ class parser
                     continue;
                 }
 
-                return sax->parse_error(m_lexer.get_position(),
+                if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr), allow_recovery))
-                                        m_lexer.get_token_string(),
+                {
-                                        parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr));
+                    return false;
+                }
+
+                // recover
+                if (last_token == token_type::end_of_input)
+                {
+                    // the input ends inside the array
+                    return close_containers(sax, states);
+                }
+                if (last_token == token_type::end_object)
+                {
+                    // a wrong closing bracket closes the innermost container
+                    if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))
+                    {
+                        return false;
+                    }
+                    states.pop_back();
+                    skip_to_state_evaluation = true;
+                }
+                // otherwise, a missing ',' (or a stray ':', which value
+                // parsing drops): the next value begins here
+                continue;
             }
 
             // states.back() is false -> object
@@ -473,11 +609,12 @@ class parser
                     // parse key
                     if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
                     {
-                        return sax->parse_error(m_lexer.get_position(),
+                        if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation))
-                                                m_lexer.get_token_string(),
+                        {
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+                            return false;
+                        }
+                        continue;
                     }
-
                     if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
                     {
                         return false;
@@ -486,9 +623,11 @@ class parser
                     // parse separator (:)
                     if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
                     {
-                        return sax->parse_error(m_lexer.get_position(),
+                        if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation))
-                                                m_lexer.get_token_string(),
+                        {
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+                            return false;
+                        }
+                        continue;
                     }
 
                     // parse values
@@ -516,10 +655,477 @@ class parser
                 continue;
             }
 
-            return sax->parse_error(m_lexer.get_position(),
+            if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr), allow_recovery))
-                                    m_lexer.get_token_string(),
+            {
-                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr));
+                return false;
             }
+
+            // recover
+            if (last_token == token_type::end_of_input)
+            {
+                // the input ends inside the object
+                return close_containers(sax, states);
+            }
+            if (last_token == token_type::end_array)
+            {
+                // a wrong closing bracket closes the innermost container
+                if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))
+                {
+                    return false;
+                }
+                states.pop_back();
+                skip_to_state_evaluation = true;
+                continue;
+            }
+            if (!continue_after(recover_member(sax, allow_recovery), skip_to_state_evaluation))
+            {
+                return false;
+            }
+        }
+    }
+
+    /*!
+    @brief continue sax_parse_internal() after a recovery
+    @return whether to continue parsing
+    */
+    bool continue_after(const next_step step, bool& skip_to_state_evaluation)
+    {
+        if (step == next_step::evaluate_state)
+        {
+            // the state evaluation reads the token again
+            m_lexer.unget_token();
+            skip_to_state_evaluation = true;
+        }
+        return step != next_step::stop;
+    }
+
+    /// the parser for parse() and accept() never recovers: stop parsing
+    static std::false_type continue_after(std::false_type /*step*/, bool& /*skip_to_state_evaluation*/) noexcept
+    {
+        return {};
+    }
+
+    /*!
+    @brief parse an object key and the name separator (:) after it
+
+    last_token is the token where the key is expected. sax_parse_internal()
+    repeats these steps rather than calling this function, which is used
+    when recovering from an error.
+
+    @return next_step::parse_value if the value follows, with last_token its
+            first token; next_step::evaluate_state if the object's state is
+            to be evaluated after recovering from an error; next_step::stop
+            to stop parsing
+    */
+    template<typename SAX>
+    next_step parse_key(SAX* sax)
+    {
+        const std::true_type allow_recovery{};
+
+        if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
+        {
+            return key_error(sax, allow_recovery, false);
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
+        {
+            return next_step::stop;
+        }
+
+        // parse separator (:)
+        if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+        {
+            return key_error(sax, allow_recovery, true);
+        }
+
+        // the value begins with the next token
+        get_token();
+        return next_step::parse_value;
+    }
+
+    /*!
+    @brief report a number that is too large for number_float_t, and recover
+           from the error by passing the value on; the SAX parser gets the
+           number's text as well
+
+    This is a separate function, as reading other numbers is measurably
+    slower if the error is handled where they are read.
+
+    @param[in] sax    the SAX parser
+    @param[in] value  the value that is not finite
+    @return whether to continue parsing
+    */
+    template<typename SAX, typename AllowRecovery>
+    bool overflow_error(SAX* sax, const number_float_t value, AllowRecovery allow_recovery)
+    {
+        if (!report_error(sax, out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr), allow_recovery))
+        {
+            return false;
+        }
+        return sax->number_float(value, m_lexer.get_string());
+    }
+
+    /*!
+    @brief report a missing key, or a missing name separator (:) after the
+           key; the parser for parse() and accept() never recovers
+
+    @param[in] key_read  whether the key was read, so that the name separator
+                         is missing
+    @return std::false_type, see report_error()
+    */
+    template<typename SAX>
+    std::false_type key_error(SAX* sax, std::false_type allow_recovery, const bool key_read)
+    {
+        return report_error(sax, parse_error::create(101, m_lexer.get_position(), key_read
+                            ? exception_message(token_type::name_separator, "object separator")
+                            : exception_message(token_type::value_string, "object key"), nullptr), allow_recovery);
+    }
+
+    /*!
+    @brief report a missing key, or a missing name separator (:) after the
+           key, and recover from it
+
+    @param[in] key_read  whether the key was read, so that the name separator
+                         is missing
+    */
+    template<typename SAX>
+    next_step key_error(SAX* sax, std::true_type allow_recovery, const bool key_read)
+    {
+        if (!key_read)
+        {
+            if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr), allow_recovery))
+            {
+                return next_step::stop;
+            }
+            return recover_key(sax);
+        }
+
+        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr), allow_recovery))
+        {
+            return next_step::stop;
+        }
+        return recover_name_separator(sax);
+    }
+
+    /////////////////////
+    // error recovery
+    /////////////////////
+
+    /*
+    The functions below repair an error after the SAX parser's parse_error()
+    returned true (see #3989). Each mistake is repaired by the smallest local
+    edit: a missing ',' or ':' is inserted, a stray token is removed, what can
+    be read of an invalid string or number is kept (see
+    lexer::recover_token()), a missing value becomes null, a wrong closing
+    bracket closes the innermost container, and the end of the input closes
+    all of them. The events stay balanced, and every key() is followed by
+    exactly one value.
+
+    A repair hands a token to the state evaluation, by returning it to the
+    lexer (lexer::unget_token()) so that the state evaluation reads it again,
+    only if it is ',', ']', '}', or the end of the input. The state evaluation
+    hands a token to value or key parsing only if it is none of them, so a
+    token is never handed back and forth. Every other step reads a token or
+    closes a container, so parsing always ends.
+    */
+
+    /*!
+    @brief report an error to the SAX parser; the parser for parse() and
+           accept() never recovers
+
+    @return std::false_type rather than false: its value is known where the
+            function is called even if the call is not inlined, so the code
+            for recovering is not generated
+    */
+    template<typename SAX, typename Exception>
+    std::false_type report_error(SAX* sax, const Exception& ex, std::false_type /*allow_recovery*/)
+    {
+        error_reported = true;
+        static_cast<void>(sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex));
+        return {};
+    }
+
+    /*!
+    @brief report an error to the SAX parser
+    @return whether to recover from the error
+    */
+    template<typename SAX, typename Exception>
+    bool report_error(SAX* sax, const Exception& ex, std::true_type /*allow_recovery*/)
+    {
+        const std::size_t position = m_lexer.get_position().chars_read_total;
+        if (error_reported && position == last_error_position && last_token == last_error_token)
+        {
+            // a repair handed on the token of the error it repaired; the
+            // token was reported already, and the SAX parser asked to recover
+            return true;
+        }
+
+        error_reported = true;
+        last_error_position = position;
+        last_error_token = last_token;
+
+        if (!sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex))
+        {
+            return false;
+        }
+
+        // the token string of the next error begins here
+        m_lexer.restart_token_string();
+        return true;
+    }
+
+    /*!
+    @brief keep what can be read of the token that the lexer rejected
+
+    The error was reported for the rejected token, so it is not reported again
+    for the token it is repaired to (see lexer::recover_token()).
+    */
+    token_type recover_token()
+    {
+        last_token = m_lexer.recover_token();
+        last_error_position = m_lexer.get_position().chars_read_total;
+        last_error_token = last_token;
+        return last_token;
+    }
+
+    /// pass the end events of all open containers
+    template<typename SAX>
+    bool close_containers(SAX* sax, std::vector<bool>& states)
+    {
+        while (!states.empty())
+        {
+            const bool is_array = states.back();
+            states.pop_back();
+            if (JSON_HEDLEY_UNLIKELY(is_array ? !sax->end_array() : !sax->end_object()))
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /*!
+    @brief read tokens until one begins a value, skipping everything before
+           the top-level value
+    @return whether a value begins with last_token
+    */
+    bool skip_to_value()
+    {
+        while (true)
+        {
+            switch (get_token())
+            {
+                case token_type::begin_array:
+                case token_type::begin_object:
+                case token_type::literal_false:
+                case token_type::literal_null:
+                case token_type::literal_true:
+                case token_type::value_float:
+                case token_type::value_integer:
+                case token_type::value_string:
+                case token_type::value_unsigned:
+                    return true;
+
+                case token_type::end_of_input:
+                    return false;
+
+                case token_type::parse_error:
+                    recover_token();
+                    if (last_token != token_type::uninitialized)
+                    {
+                        return true;
+                    }
+                    break;
+
+                case token_type::uninitialized:
+                case token_type::end_array:
+                case token_type::end_object:
+                case token_type::name_separator:
+                case token_type::value_separator:
+                case token_type::literal_or_value:
+                default:
+                    break;
+            }
+        }
+    }
+
+    /*!
+    @brief skip the rest of an object member that cannot be read
+
+    Reads tokens, beginning with last_token, until a ',', '}', or ']' that is
+    not inside a container that begins in the skipped tokens, or the end of
+    the input.
+    */
+    void skip_member()
+    {
+        std::size_t depth = 0;
+        while (true)
+        {
+            switch (last_token)
+            {
+                case token_type::begin_array:
+                case token_type::begin_object:
+                    ++depth;
+                    break;
+
+                case token_type::end_array:
+                case token_type::end_object:
+                    if (depth == 0)
+                    {
+                        return;
+                    }
+                    --depth;
+                    break;
+
+                case token_type::value_separator:
+                    if (depth == 0)
+                    {
+                        return;
+                    }
+                    break;
+
+                case token_type::end_of_input:
+                    return;
+
+                case token_type::parse_error:
+                    recover_token();
+                    break;
+
+                case token_type::uninitialized:
+                case token_type::literal_true:
+                case token_type::literal_false:
+                case token_type::literal_null:
+                case token_type::value_string:
+                case token_type::value_unsigned:
+                case token_type::value_integer:
+                case token_type::value_float:
+                case token_type::name_separator:
+                case token_type::literal_or_value:
+                default:
+                    break;
+            }
+            get_token();
+        }
+    }
+
+    /*!
+    @brief pass a value where it is missing
+
+    last_token is ',', ']', '}', or the end of the input, where a value was
+    expected. In an object, the key gets null; in an array, a ',' where a
+    value is missing stands for null (as in JavaScript), while an array that
+    ends there just ends.
+    */
+    template<typename SAX>
+    bool recover_missing_value(SAX* sax, const std::vector<bool>& states)
+    {
+        JSON_ASSERT(!states.empty());
+        if (!states.back() || last_token == token_type::value_separator)
+        {
+            return sax->null();
+        }
+        return true;
+    }
+
+    /// recover from a missing key; last_token is where it was expected
+    template<typename SAX>
+    next_step recover_key(SAX* sax)
+    {
+        switch (last_token)
+        {
+            case token_type::value_separator:
+            case token_type::end_object:
+            case token_type::end_array:
+            case token_type::end_of_input:
+                // no member: the object's state handles the token
+                return next_step::evaluate_state;
+
+            case token_type::parse_error:
+                recover_token();
+                if (last_token == token_type::value_string)
+                {
+                    // a key that could be repaired
+                    return parse_key(sax);
+                }
+                skip_member();
+                return next_step::evaluate_state;
+
+            case token_type::uninitialized:
+            case token_type::literal_true:
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+            case token_type::value_integer:
+            case token_type::value_float:
+            case token_type::begin_array:
+            case token_type::begin_object:
+            case token_type::name_separator:
+            case token_type::literal_or_value:
+            default:
+                // a member without a key
+                skip_member();
+                return next_step::evaluate_state;
+        }
+    }
+
+    /// recover from a missing name separator (:) after the key; last_token
+    /// is where it was expected
+    template<typename SAX>
+    next_step recover_name_separator(SAX* sax)
+    {
+        switch (last_token)
+        {
+            case token_type::value_separator:
+            case token_type::end_object:
+            case token_type::end_array:
+            case token_type::end_of_input:
+                // the value is missing as well
+                return sax->null() ? next_step::evaluate_state : next_step::stop;
+
+            case token_type::uninitialized:
+            case token_type::literal_true:
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+            case token_type::value_integer:
+            case token_type::value_float:
+            case token_type::begin_array:
+            case token_type::begin_object:
+            case token_type::name_separator:
+            case token_type::parse_error:
+            case token_type::literal_or_value:
+            default:
+                // a missing ':'; the value begins here
+                return next_step::parse_value;
+        }
+    }
+
+    /// recover from a token after an object member that is neither ',' nor
+    /// '}' (nor ']' or the end of the input, which the caller handles)
+    template<typename SAX>
+    next_step recover_member(SAX* sax, std::true_type /*allow_recovery*/)
+    {
+        if (last_token == token_type::parse_error)
+        {
+            recover_token();
+        }
+        if (last_token == token_type::value_string)
+        {
+            // a missing ','; the next key begins here
+            return parse_key(sax);
+        }
+        skip_member();
+        return next_step::evaluate_state;
+    }
+
+    /// the parser for parse() and accept() never recovers (and does not come
+    /// here, as report_error() returned false)
+    template<typename SAX>
+    std::false_type recover_member(SAX* /*sax*/, std::false_type /*allow_recovery*/) const noexcept
+    {
+        return {};
     }
 
     /// get next token from lexer
@@ -568,6 +1174,12 @@ class parser
     const bool allow_exceptions = true;
     /// whether trailing commas in objects and arrays should be ignored (true) or signaled as errors (false)
     const bool ignore_trailing_commas = false;
+    /// whether an error was reported to the SAX parser
+    bool error_reported = false;
+    /// the position of the last reported error
+    std::size_t last_error_position = 0;
+    /// the token of the last reported error
+    token_type last_error_token = token_type::uninitialized;
 };
 
 }  // namespace detail

include/nlohmann/detail/iterators/iter_impl.hpp

@@ -60,9 +60,11 @@ class iter_impl // NOLINT(cppcoreguidelines-special-member-functions,hicpp-speci
     static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
                   "iter_impl only accepts (const) basic_json");
     // superficial check for the LegacyBidirectionalIterator named requirement
-    static_assert(std::is_base_of<std::bidirectional_iterator_tag, std::bidirectional_iterator_tag>::value
+    // note: only array_t::iterator is checked here; object_t::iterator may be
-                  &&  std::is_base_of<std::bidirectional_iterator_tag, typename std::iterator_traits<typename array_t::iterator>::iterator_category>::value,
+    // a forward-only iterator as long as reverse iteration and operator--
-                  "basic_json iterator assumes array and object type iterators satisfy the LegacyBidirectionalIterator named requirement.");
+    // are never used on it
+    static_assert(std::is_base_of<std::bidirectional_iterator_tag, typename std::iterator_traits<typename array_t::iterator>::iterator_category>::value,
+                  "basic_json iterator assumes array type iterators satisfy the LegacyBidirectionalIterator named requirement.");
 
   public:
     /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17.

include/nlohmann/detail/json_pointer.hpp

@@ -240,6 +240,72 @@ class json_pointer
     }
 
   private:
+    /*!
+    @brief result of @ref parse_array_index
+
+    @ref array_index maps each value to the corresponding parse_error/out_of_range
+    exception; @ref contains and @ref get_checked_or_null, which must not throw for
+    an out-of-range or unrepresentable index, switch on it directly instead.
+    */
+    enum class array_index_status
+    {
+        ok,                ///< @a s is a valid, representable array index
+        leading_zero,      ///< @a s begins with '0' but has more than one character
+        not_a_number,      ///< @a s does not begin with a digit
+        unresolved,        ///< @a s could not be converted to an integer
+        exceeds_size_type  ///< @a s converts to an integer that exceeds size_type
+    };
+
+    /*!
+    @param[in] s    reference token to be converted into an array index
+    @param[out] idx the integer representation of @a s if @ref array_index_status::ok
+                     is returned; left unchanged otherwise
+
+    @return whether @a s is a valid array index, and if not, why
+
+    @note this function never throws; @ref array_index and the callers that must not
+          throw (@ref contains, @ref get_checked_or_null) build on it instead of each
+          re-implementing the RFC 6901 digit rules and the @a size_type range check
+    */
+    template<typename BasicJsonType>
+    static array_index_status parse_array_index(const string_t& s, typename BasicJsonType::size_type& idx) noexcept
+    {
+        using size_type = typename BasicJsonType::size_type;
+
+        // error condition (cf. RFC 6901, Sect. 4)
+        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0'))
+        {
+            return array_index_status::leading_zero;
+        }
+
+        // error condition (cf. RFC 6901, Sect. 4)
+        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))
+        {
+            return array_index_status::not_a_number;
+        }
+
+        const char* p = s.data();
+        char* p_end = nullptr; // NOLINT(misc-const-correctness)
+        errno = 0; // strtoull doesn't reset errno
+        const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)
+        if (p == p_end // invalid input or empty string
+                || errno == ERANGE // out of range
+                || JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(p_end - p) != s.size())) // incomplete read
+        {
+            return array_index_status::unresolved;
+        }
+
+        // the index does not fit into size_type; on 64-bit platforms this is
+        // only SIZE_MAX itself (see #2203 and #5395)
+        if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))  // NOLINT(runtime/int)
+        {
+            return array_index_status::exceeds_size_type;
+        }
+
+        idx = static_cast<size_type>(res);
+        return array_index_status::ok;
+    }
+
     /*!
     @param[in] s  reference token to be converted into an array index
 
@@ -253,39 +319,23 @@ class json_pointer
     template<typename BasicJsonType>
     static typename BasicJsonType::size_type array_index(const string_t& s)
     {
-        using size_type = typename BasicJsonType::size_type;
+        typename BasicJsonType::size_type idx{};
-
+        switch (parse_array_index<BasicJsonType>(s, idx))
-        // error condition (cf. RFC 6901, Sect. 4)
-        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && s[0] == '0'))
         {
+            case array_index_status::leading_zero:
                 JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", s, "' must not begin with '0'"), nullptr));
-        }
+            case array_index_status::not_a_number:
-
-        // error condition (cf. RFC 6901, Sect. 4)
-        if (JSON_HEDLEY_UNLIKELY(s.size() > 1 && !(s[0] >= '1' && s[0] <= '9')))
-        {
                 JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", s, "' is not a number"), nullptr));
-        }
+            case array_index_status::unresolved:
-
-        const char* p = s.data();
-        char* p_end = nullptr; // NOLINT(misc-const-correctness)
-        errno = 0; // strtoull doesn't reset errno
-        const unsigned long long res = std::strtoull(p, &p_end, 10); // NOLINT(runtime/int)
-        if (p == p_end // invalid input or empty string
-                || errno == ERANGE // out of range
-                || JSON_HEDLEY_UNLIKELY(static_cast<std::size_t>(p_end - p) != s.size())) // incomplete read
-        {
                 JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", s, "'"), nullptr));
-        }
+            case array_index_status::exceeds_size_type:
-
-        // the index does not fit into size_type; on 64-bit platforms this is
-        // only SIZE_MAX itself (see #2203 and #5395)
-        if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))  // NOLINT(runtime/int)
-        {
                 JSON_THROW(detail::out_of_range::create(410, detail::concat("array index ", s, " exceeds size_type"), nullptr));
+            case array_index_status::ok:
+            default:
+                break;
         }
 
-        return static_cast<size_type>(res);
+        return idx;
     }
 
   JSON_PRIVATE_UNLESS_TESTED:
@@ -536,6 +586,10 @@ class json_pointer
     @return const reference to the JSON value pointed to by the JSON
     pointer
 
+    @pre Every object key and array index the pointer refers to exists.
+         Like the const operator[] for keys and indices, a missing one is
+         undefined behavior, guarded by a runtime assertion.
+
     @throw parse_error.106   if an array index begins with '0'
     @throw parse_error.109   if an array index was not a number
     @throw out_of_range.402  if the array index '-' is used
@@ -550,7 +604,8 @@ class json_pointer
             {
                 case detail::value_t::object:
                 {
-                    // use unchecked object access
+                    // use unchecked object access; the const operator[]
+                    // asserts that the key exists
                     ptr = &ptr->operator[](reference_token);
                     break;
                 }
@@ -563,7 +618,8 @@ class json_pointer
                         JSON_THROW(detail::out_of_range::create(402, detail::concat("array index '-' (", std::to_string(ptr->m_data.m_value.array->size()), ") is out of range"), ptr));
                     }
 
-                    // use unchecked array access
+                    // use unchecked array access; the const operator[]
+                    // asserts that the index exists
                     ptr = &ptr->operator[](array_index<BasicJsonType>(reference_token));
                     break;
                 }
@@ -584,63 +640,6 @@ class json_pointer
         return *ptr;
     }
 
-    /*!
-    @throw parse_error.106   if an array index begins with '0'
-    @throw parse_error.109   if an array index was not a number
-    @throw out_of_range.402  if the array index '-' is used
-    @throw out_of_range.404  if the JSON pointer can not be resolved
-    */
-    template<typename BasicJsonType>
-    const BasicJsonType& get_checked(const BasicJsonType* ptr) const
-    {
-        for (const auto& reference_token : reference_tokens)
-        {
-            switch (ptr->type())
-            {
-                case detail::value_t::object:
-                {
-                    // note: at performs range check
-                    ptr = &ptr->at(reference_token);
-                    break;
-                }
-
-                case detail::value_t::array:
-                {
-                    if (JSON_HEDLEY_UNLIKELY(reference_token == "-"))
-                    {
-                        // "-" always fails the range check
-                        JSON_THROW(detail::out_of_range::create(402, detail::concat(
-                                "array index '-' (", std::to_string(ptr->m_data.m_value.array->size()),
-                                ") is out of range"), ptr));
-                    }
-
-                    const auto idx = array_index<BasicJsonType>(reference_token);
-                    // Bounds check before access to avoid exception with JSON_NOEXCEPTION
-                    if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size()))
-                    {
-                        JSON_THROW(detail::out_of_range::create(401, detail::concat(
-                                "array index ", std::to_string(idx), " is out of range"), ptr));
-                    }
-                    ptr = &ptr->operator[](idx);
-                    break;
-                }
-
-                case detail::value_t::null:
-                case detail::value_t::string:
-                case detail::value_t::boolean:
-                case detail::value_t::number_integer:
-                case detail::value_t::number_unsigned:
-                case detail::value_t::number_float:
-                case detail::value_t::binary:
-                case detail::value_t::discarded:
-                default:
-                    JSON_THROW(detail::out_of_range::create(404, detail::concat("unresolved reference token '", reference_token, "'"), ptr));
-            }
-        }
-
-        return *ptr;
-    }
-
     /*!
     @brief return a pointer to the pointed to value, or `nullptr` if the
            pointer cannot be resolved because a key is missing, an array
@@ -679,18 +678,23 @@ class json_pointer
                         return nullptr;
                     }
 
-                    // may throw parse_error.106/109 for a malformed index; an
+                    // a malformed index throws parse_error.106/109; an
                     // index that is syntactically valid but cannot be
                     // represented (out_of_range.404/410) is treated like an
                     // out-of-range index below
                     typename BasicJsonType::size_type idx{};
-                    JSON_TRY
+                    switch (parse_array_index<BasicJsonType>(reference_token, idx))
-                    {
-                        idx = array_index<BasicJsonType>(reference_token);
-                    }
-                    JSON_INTERNAL_CATCH (detail::out_of_range&)
                     {
+                        case array_index_status::leading_zero:
+                            JSON_THROW(detail::parse_error::create(106, 0, detail::concat("array index '", reference_token, "' must not begin with '0'"), nullptr));
+                        case array_index_status::not_a_number:
+                            JSON_THROW(detail::parse_error::create(109, 0, detail::concat("array index '", reference_token, "' is not a number"), nullptr));
+                        case array_index_status::unresolved:
+                        case array_index_status::exceeds_size_type:
                             return nullptr;
+                        case array_index_status::ok:
+                        default:
+                            break;
                     }
 
                     if (JSON_HEDLEY_UNLIKELY(idx >= ptr->m_data.m_value.array->size()))
@@ -718,8 +722,8 @@ class json_pointer
     }
 
     /*!
-    @throw parse_error.106   if an array index begins with '0'
+    @note unlike array_index(), this never throws: a malformed or unrepresentable
-    @throw parse_error.109   if an array index was not a number
+          array index reference token is treated like a missing key (see #5395)
     */
     template<typename BasicJsonType>
     bool contains(const BasicJsonType* ptr) const
@@ -747,49 +751,17 @@ class json_pointer
                         // "-" always fails the range check
                         return false;
                     }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.empty()))
-                    {
-                        // an empty reference token is not an array index; array_index()
-                        // would throw out_of_range.404 -- contains() must not throw (see #5395)
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() == 1 && !('0' <= reference_token[0] && reference_token[0] <= '9')))
-                    {
-                        // invalid char
-                        return false;
-                    }
-                    if (JSON_HEDLEY_UNLIKELY(reference_token.size() > 1))
-                    {
-                        if (JSON_HEDLEY_UNLIKELY(!('1' <= reference_token[0] && reference_token[0] <= '9')))
-                        {
-                            // the first char should be between '1' and '9'
-                            return false;
-                        }
-                        for (std::size_t i = 1; i < reference_token.size(); i++)
-                        {
-                            if (JSON_HEDLEY_UNLIKELY(!('0' <= reference_token[i] && reference_token[i] <= '9')))
-                            {
-                                // other char should be between '0' and '9'
-                                return false;
-                            }
-                        }
-                    }
 
-                    // the reference token consists only of digits at this point (cf. checks
+                    // any parse failure (malformed index, or one that is syntactically
-                    // above); however, its numeric value might not be representable, in which
+                    // valid but not representable as size_type) means the reference
-                    // case array_index() would throw out_of_range.404/410 -- contains() must
+                    // token cannot denote an existing array element -- contains() must
-                    // not throw (see #5395), so such a reference token is treated as "not found"
+                    // not throw (see #5395), so it is treated as "not found"
-                    errno = 0; // strtoull() does not reset errno on success
+                    typename BasicJsonType::size_type idx{};
-                    char* p_end = nullptr; // NOLINT(misc-const-correctness)
+                    if (JSON_HEDLEY_UNLIKELY(parse_array_index<BasicJsonType>(reference_token, idx) != array_index_status::ok))
-                    const unsigned long long magnitude = std::strtoull(reference_token.data(), &p_end, 10); // NOLINT(runtime/int)
-                    if (JSON_HEDLEY_UNLIKELY(errno == ERANGE // the value exceeds ULLONG_MAX
-                                             || magnitude >= static_cast<unsigned long long>((std::numeric_limits<typename BasicJsonType::size_type>::max)()))) // NOLINT(runtime/int)
                     {
-                        // the array index cannot be represented as size_type
                         return false;
                     }
 
-                    const auto idx = array_index<BasicJsonType>(reference_token);
                     if (idx >= ptr->size())
                     {
                         // index out of range

include/nlohmann/detail/macro_scope.hpp

@@ -9,7 +9,6 @@
 #pragma once
 
 #include <utility> // declval, pair
-#include <nlohmann/detail/meta/detected.hpp>
 #include <nlohmann/thirdparty/hedley/hedley.hpp>
 
 // This file contains all internal macro definitions (except those affecting ABI)
@@ -140,10 +139,12 @@
         // libstdc++ < 11 has incomplete C++20 ranges (issue #4440)
     #elif defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 11
         #define JSON_HAS_RANGES 0
-        // libc++ < 16 has incomplete C++20 ranges (issue #4440)
+        // clang < 16 with libstdc++ does not implement the ranges customization
+        // points libstdc++ declares, so its C++20 ranges support is incomplete (issue #5161)
     #elif defined(__clang__) && !defined(__apple_build_version__) \
         && __clang_major__ < 16 && defined(__GLIBCXX__)
         #define JSON_HAS_RANGES 0
+        // libc++ < 16 has incomplete C++20 ranges (issue #4440)
     #elif defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000
         #define JSON_HAS_RANGES 0
         // nvcc CUDA 12.0/12.1 chokes on the enable_borrowed_range variable-template
@@ -158,6 +159,18 @@
     #endif
 #endif
 
+// std::ranges view conversion (to_json/is_compatible_array_type_impl) additionally
+// needs to be disabled on MinGW, whose std::ranges support is incomplete
+// (issue #4916); this macro combines both conditions so the check and its
+// reason are not duplicated at every use site.
+#ifndef JSON_HAS_RANGE_VIEW_CONVERSION
+    #if JSON_HAS_RANGES && !defined(__MINGW32__)
+        #define JSON_HAS_RANGE_VIEW_CONVERSION 1
+    #else
+        #define JSON_HAS_RANGE_VIEW_CONVERSION 0
+    #endif
+#endif
+
 #ifndef JSON_HAS_STD_FORMAT
     #if defined(JSON_HAS_CPP_20) && defined(__cpp_lib_format)
         #define JSON_HAS_STD_FORMAT 1
@@ -279,21 +292,6 @@
 
 
 
-/*!
-@brief function to wrap JSON_THROW_MACRO - there can be compilation errors about
-       there being no arguments to JSON_THROW that depend on template arguments
-       if this is not used to call JSON_THROW
-*/
-template<typename ExceptionType>
-void templated_json_throw(ExceptionType exception)
-{
-    JSON_THROW(exception);
-
-    /* JSON_THROW(exception) discards exception and aborts - void cast needed to supress
-       compilation error if compiled with -Werror and Wunused-parameter */
-    (void)exception;
-}
-
 /*!
 @brief macro to briefly define a mapping between an enum and JSON with exception
        on invalid input
@@ -314,7 +312,7 @@ void templated_json_throw(ExceptionType exception)
             return ej_pair.first == e;                                                          \
         });                                                                                     \
         if (it != std::end(m)) j = it->second;                                                  \
-        else templated_json_throw<nlohmann::detail::out_of_range>(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \
+        else ::nlohmann::detail::templated_json_throw<nlohmann::detail::out_of_range>(nlohmann::detail::out_of_range::create(410,"enum value out of range for " #ENUM_TYPE, nullptr)); \
     }                                                                                           \
     template<typename BasicJsonType>                                                            \
     inline void from_json(const BasicJsonType& j, ENUM_TYPE& e)                                 \
@@ -329,7 +327,7 @@ void templated_json_throw(ExceptionType exception)
             return ej_pair.second == j;                                                         \
         });                                                                                     \
         if (it != std::end(m)) e = it->first;                                                   \
-        else templated_json_throw<nlohmann::detail::out_of_range>(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \
+        else ::nlohmann::detail::templated_json_throw<nlohmann::detail::out_of_range>(nlohmann::detail::out_of_range::create(410, nlohmann::detail::concat("enum value out of range for " #ENUM_TYPE ": ", j.dump(-1, ' ', false, nlohmann::detail::error_handler_t::replace)), &j)); \
     }
 
 // Ugly macros to avoid uglier copy-paste when specializing basic_json. They
@@ -874,30 +872,6 @@ void templated_json_throw(ExceptionType exception)
     \
     template<typename... T>                                                       \
     using result_of_##std_name = decltype(std_name(std::declval<T>()...));        \
-    }                                                                             \
-    \
-    namespace detail2 {                                                           \
-    struct std_name##_tag                                                         \
-    {                                                                             \
-    };                                                                            \
-    \
-    template<typename... T>                                                       \
-    std_name##_tag std_name(T&&...);                                              \
-    \
-    template<typename... T>                                                       \
-    using result_of_##std_name = decltype(std_name(std::declval<T>()...));        \
-    \
-    template<typename... T>                                                       \
-    struct would_call_std_##std_name                                              \
-    {                                                                             \
-        static constexpr auto const value = ::nlohmann::detail::                  \
-                                            is_detected_exact<std_name##_tag, result_of_##std_name, T...>::value; \
-    };                                                                            \
-    } /* namespace detail2 */ \
-    \
-    template<typename... T>                                                       \
-    struct would_call_std_##std_name : detail2::would_call_std_##std_name<T...>   \
-    {                                                                             \
     }
 
 #ifndef JSON_USE_IMPLICIT_CONVERSIONS

include/nlohmann/detail/macro_unscope.hpp

@@ -35,12 +35,14 @@
     #undef JSON_HAS_EXPERIMENTAL_FILESYSTEM
     #undef JSON_HAS_THREE_WAY_COMPARISON
     #undef JSON_HAS_RANGES
+    #undef JSON_HAS_RANGE_VIEW_CONVERSION
     #undef JSON_HAS_STD_FORMAT
     #undef JSON_HAS_STATIC_RTTI
     #undef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON
     #undef JSON_BRACE_INIT_COPY_SEMANTICS
     #undef JSON_PRECISE_STREAM_POSITION
     #undef JSON_STRICT_NUL_HANDLING
+    #undef JSON_STRICT_BINARY_UTF8
 #endif
 
 #include <nlohmann/thirdparty/hedley/hedley_undef.hpp>

include/nlohmann/detail/meta/call_std/begin.hpp

@@ -12,6 +12,6 @@
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 
-NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin);
+NLOHMANN_CAN_CALL_STD_FUNC_IMPL(begin)
 
 NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/meta/call_std/end.hpp

@@ -12,6 +12,6 @@
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 
-NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end);
+NLOHMANN_CAN_CALL_STD_FUNC_IMPL(end)
 
 NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/meta/is_sax.hpp

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <cstdint> // size_t
+#include <cstddef> // size_t
 #include <utility> // declval
 #include <string> // string
 
@@ -70,37 +70,6 @@ using parse_error_function_t = decltype(std::declval<T&>().parse_error(
         std::declval<std::size_t>(), std::declval<const std::string&>(),
         std::declval<const Exception&>()));
 
-template<typename SAX, typename BasicJsonType>
-struct is_sax
-{
-  private:
-    static_assert(is_basic_json<BasicJsonType>::value,
-                  "BasicJsonType must be of type basic_json<...>");
-
-    using number_integer_t = typename BasicJsonType::number_integer_t;
-    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
-    using number_float_t = typename BasicJsonType::number_float_t;
-    using string_t = typename BasicJsonType::string_t;
-    using binary_t = typename BasicJsonType::binary_t;
-    using exception_t = typename BasicJsonType::exception;
-
-  public:
-    static constexpr bool value =
-        is_detected_exact<bool, null_function_t, SAX>::value &&
-        is_detected_exact<bool, boolean_function_t, SAX>::value &&
-        is_detected_exact<bool, number_integer_function_t, SAX, number_integer_t>::value &&
-        is_detected_exact<bool, number_unsigned_function_t, SAX, number_unsigned_t>::value &&
-        is_detected_exact<bool, number_float_function_t, SAX, number_float_t, string_t>::value &&
-        is_detected_exact<bool, string_function_t, SAX, string_t>::value &&
-        is_detected_exact<bool, binary_function_t, SAX, binary_t>::value &&
-        is_detected_exact<bool, start_object_function_t, SAX>::value &&
-        is_detected_exact<bool, key_function_t, SAX, string_t>::value &&
-        is_detected_exact<bool, end_object_function_t, SAX>::value &&
-        is_detected_exact<bool, start_array_function_t, SAX>::value &&
-        is_detected_exact<bool, end_array_function_t, SAX>::value &&
-        is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value;
-};
-
 template<typename SAX, typename BasicJsonType>
 struct is_sax_static_asserts
 {
@@ -120,8 +89,6 @@ struct is_sax_static_asserts
                   "Missing/invalid function: bool null()");
     static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
                   "Missing/invalid function: bool boolean(bool)");
-    static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value,
-                  "Missing/invalid function: bool boolean(bool)");
     static_assert(
         is_detected_exact<bool, number_integer_function_t, SAX,
         number_integer_t>::value,

include/nlohmann/detail/meta/logic.hpp

@@ -1,54 +0,0 @@
-#pragma once
-
-#include <nlohmann/detail/macro_scope.hpp>
-
-NLOHMANN_JSON_NAMESPACE_BEGIN
-namespace detail
-{
-#ifdef JSON_HAS_CPP_17
-
-template<bool... Booleans>
-struct cxpr_or_impl : std::integral_constant < bool, (Booleans || ...) > {};
-
-template<bool... Booleans>
-struct cxpr_and_impl : std::integral_constant < bool, (Booleans &&...) > {};
-
-#else
-
-template<bool... Booleans>
-struct cxpr_or_impl : std::false_type {};
-
-template<bool... Booleans>
-struct cxpr_or_impl<true, Booleans...> : std::true_type {};
-
-template<bool... Booleans>
-struct cxpr_or_impl<false, Booleans...> : cxpr_or_impl<Booleans...> {};
-
-template<bool... Booleans>
-struct cxpr_and_impl : std::true_type {};
-
-template<bool... Booleans>
-struct cxpr_and_impl<true, Booleans...> : cxpr_and_impl<Booleans...> {};
-
-template<bool... Booleans>
-struct cxpr_and_impl<false, Booleans...> : std::false_type {};
-
-#endif
-
-template<class Boolean>
-struct cxpr_not : std::integral_constant < bool, !Boolean::value > {};
-
-template<class... Booleans>
-struct cxpr_or : cxpr_or_impl<Booleans::value...> {};
-
-template<bool... Booleans>
-struct cxpr_or_c : cxpr_or_impl<Booleans...> {};
-
-template<class... Booleans>
-struct cxpr_and : cxpr_and_impl<Booleans::value...> {};
-
-template<bool... Booleans>
-struct cxpr_and_c : cxpr_and_impl<Booleans...> {};
-
-}  // namespace detail
-NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/meta/type_traits.hpp

@@ -189,6 +189,37 @@ struct actual_object_comparator
 template<typename BasicJsonType>
 using actual_object_comparator_t = typename actual_object_comparator<BasicJsonType>::type;
 
+template<typename T>
+using detect_key_comp = decltype(std::declval<const T&>().key_comp());
+
+// whether ObjectType can be constructed from a pair of Iterator together with
+// a copy of its own comparator, the way std::map can: it needs a nested
+// key_compare, a const key_comp() convertible to it, and a matching
+// (Iterator, Iterator, const key_compare&) constructor.
+//
+// used to preserve a stateful comparator when a copy is built from a range
+// past the iterative deep copy's nesting bound (see copy_object_level); an
+// object type that does not satisfy this, such as nlohmann::ordered_map
+// (which has key_compare for its std::map-like interface, but no key_comp()),
+// keeps default-constructing its comparator, just as it always has
+template<typename ObjectType, typename Iterator, typename = void>
+struct is_comparator_constructible_object_type_impl : std::false_type {};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type_impl <
+    ObjectType, Iterator, enable_if_t<is_detected<detect_key_compare, ObjectType>::value >>
+{
+    using key_compare = typename ObjectType::key_compare;
+
+    static constexpr bool value =
+        is_detected_convertible<key_compare, detect_key_comp, ObjectType>::value &&
+        std::is_constructible<ObjectType, Iterator, Iterator, const key_compare&>::value;
+};
+
+template<typename ObjectType, typename Iterator>
+struct is_comparator_constructible_object_type
+    : is_comparator_constructible_object_type_impl<ObjectType, Iterator> {};
+
 /////////////////
 // char_traits //
 /////////////////
@@ -283,6 +314,13 @@ template<class B, class... Bn>
 struct conjunction<B, Bn...>
 : std::conditional<static_cast<bool>(B::value), conjunction<Bn...>, B>::type {};
 
+// https://en.cppreference.com/w/cpp/types/disjunction
+template<class...> struct disjunction : std::false_type { };
+template<class B> struct disjunction<B> : B { };
+template<class B, class... Bn>
+struct disjunction<B, Bn...>
+: std::conditional<static_cast<bool>(B::value), B, disjunction<Bn...>>::type {};
+
 // https://en.cppreference.com/w/cpp/types/negation
 template<class B> struct negation : std::integral_constant < bool, !B::value > { };
 
@@ -477,9 +515,7 @@ template<typename T> struct is_range_view_optional_type<std::optional<T>> : std:
 template<typename T> struct is_range_view_optional_type : std::false_type {};
 #endif
 
-// std::ranges does not work properly on MinGW due to incomplete C++20 support
+#if JSON_HAS_RANGE_VIEW_CONVERSION
-// see https://github.com/nlohmann/json/issues/4916
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
 
 // SafeToCheck guards against types that trigger circular constraints when
 // std::ranges::view<T> is evaluated on GCC 12 / libstdc++ 12:
@@ -518,7 +554,7 @@ struct is_compatible_array_type_impl <
 // filter_view) can match BOTH this iterator-based specialization AND the view-based one
 // below, causing ambiguity. Exclude views here so the two specializations are mutually
 // exclusive: this one handles plain iterable containers, the other handles views.
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
 && !is_compatible_range_view<CompatibleArrayType>::value
 #endif
             >>
@@ -528,7 +564,7 @@ struct is_compatible_array_type_impl <
         range_value_t<CompatibleArrayType>>::value;
 };
 
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
 template<typename BasicJsonType, typename CompatibleArrayType>
 struct is_compatible_array_type_impl <
     BasicJsonType, CompatibleArrayType,
@@ -604,7 +640,6 @@ struct is_compatible_integer_type_impl <
     std::is_integral<CompatibleNumberIntegerType>::value&&
     !std::is_same<bool, CompatibleNumberIntegerType>::value >>
 {
-    // is there an assert somewhere on overflows?
     using RealLimits = std::numeric_limits<RealIntegerType>;
     using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
 
@@ -760,6 +795,30 @@ using is_usable_as_key_type = typename std::conditional <
                               std::true_type,
                               std::false_type >::type;
 
+#ifdef JSON_HAS_CPP_17
+// type trait to check if KeyType can only be used as an object key after
+// converting it to std::string_view: it is convertible to std::string_view, the
+// object's comparator cannot compare it with object_t::key_type directly, but
+// can compare a std::string_view. JSON pointers and JSON iterators are ruled out
+// first, so that the conversion checks are never instantiated for them (a JSON
+// pointer's deprecated conversion to string_t would be named otherwise).
+template < typename BasicJsonType, typename KeyTypeCVRef, typename KeyType = uncvref_t<KeyTypeCVRef>,
+           bool = is_json_pointer<KeyType>::value || is_json_iterator_of<BasicJsonType, KeyType>::value >
+struct is_string_view_convertible_key_type : std::false_type {};
+
+template<typename BasicJsonType, typename KeyTypeCVRef, typename KeyType>
+struct is_string_view_convertible_key_type<BasicJsonType, KeyTypeCVRef, KeyType, false>
+    : std::integral_constant < bool,
+      std::is_convertible<KeyTypeCVRef, std::string_view>::value
+      && !is_usable_as_key_type<typename BasicJsonType::object_comparator_t,
+      typename BasicJsonType::object_t::key_type, KeyTypeCVRef, true, false>::value
+      && is_usable_as_key_type<typename BasicJsonType::object_comparator_t,
+      typename BasicJsonType::object_t::key_type, std::string_view, true, false>::value > {};
+#else
+template<typename BasicJsonType, typename KeyTypeCVRef>
+struct is_string_view_convertible_key_type : std::false_type {};
+#endif
+
 // type trait to check if KeyType can be used as an object key
 // true if:
 //   - KeyType is comparable with BasicJsonType::object_t::key_type
@@ -773,9 +832,7 @@ using is_usable_as_basic_json_key_type = typename std::conditional <
      typename BasicJsonType::object_t::key_type, KeyTypeCVRef,
      RequireTransparentComparator, ExcludeObjectKeyType>::value
      && !is_json_iterator_of<BasicJsonType, KeyType>::value)
-#ifdef JSON_HAS_CPP_17
+    || is_string_view_convertible_key_type<BasicJsonType, KeyTypeCVRef>::value
-    || std::is_convertible<KeyType, std::string_view>::value
-#endif
     , std::true_type,
     std::false_type >::type;
 
@@ -810,20 +867,7 @@ struct has_capacity : std::integral_constant<bool, is_detected<detect_capacity,
 // a naive helper to check if a type is an ordered_map (exploits the fact that
 // ordered_map inherits capacity() from std::vector)
 template <typename T>
-struct is_ordered_map
+struct is_ordered_map : has_capacity<T> {};
-{
-    using one = char;
-
-    struct two
-    {
-        char x[2]; // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
-    };
-
-    template <typename C> static one test( decltype(&C::capacity) ) ;
-    template <typename C> static two test(...);
-
-    enum { value = sizeof(test<T>(nullptr)) == sizeof(char) }; // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg,cppcoreguidelines-use-enum-class)
-};
 
 // to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324)
 template < typename T, typename U, enable_if_t < !std::is_same<T, U>::value, int > = 0 >
@@ -847,10 +891,8 @@ using all_signed = conjunction<std::is_signed<Types>...>;
 template<typename... Types>
 using all_unsigned = conjunction<std::is_unsigned<Types>...>;
 
-// there's a disjunction trait in another PR; replace when merged
 template<typename... Types>
-using same_sign = std::integral_constant < bool,
+using same_sign = disjunction<all_signed<Types...>, all_unsigned<Types...>>;
-      all_signed<Types...>::value || all_unsigned<Types...>::value >;
 
 template<typename OfType, typename T>
 using never_out_of_range = std::integral_constant < bool,

include/nlohmann/detail/output/binary_writer.hpp

@@ -26,6 +26,7 @@
 #include <nlohmann/detail/input/binary_reader.hpp>
 #include <nlohmann/detail/input/string_scan.hpp>
 #include <nlohmann/detail/macro_scope.hpp>
+#include <nlohmann/detail/output/error_handler.hpp>
 #include <nlohmann/detail/output/output_adapters.hpp>
 #include <nlohmann/detail/string_concat.hpp>
 #include <nlohmann/detail/string_utils.hpp>
@@ -93,8 +94,12 @@ class binary_writer
     @param[in] sink  output sink to write to (a value-type sink such as
                      output_vector_sink, or output_adapter_sink wrapping a
                      type-erased output adapter)
+    @param[in] error_handler_  how to treat a string value or object key that
+               is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON;
+               never consulted by @ref write_bon8)
     */
-    explicit binary_writer(OutputSinkType sink) : oa(std::move(sink))
+    explicit binary_writer(OutputSinkType sink, const error_handler_t error_handler_ = binary_writer_default_error_handler())
+        : oa(std::move(sink)), error_handler(error_handler_)
     {}
 
     /*!
@@ -107,14 +112,20 @@ class binary_writer
     from one.
 
     @param[in] adapter  output adapter to write to
+    @param[in] error_handler_  how to treat a string value or object key that
+               is not valid UTF-8 (CBOR, MessagePack, UBJSON, BJData, and BSON;
+               never consulted by @ref write_bon8)
     */
     template < typename SinkType = OutputSinkType,
                typename std::enable_if < std::is_constructible<SinkType, output_adapter_t<CharType>>::value, int >::type = 0 >
-    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(SinkType(std::move(adapter)))
+    explicit binary_writer(output_adapter_t<CharType> adapter, const error_handler_t error_handler_ = binary_writer_default_error_handler())
+        : oa(SinkType(std::move(adapter))), error_handler(error_handler_)
     {}
 
     /*!
     @param[in] j  JSON value to serialize
+    @throw type_error.316 if a string value or an object key is not valid
+           UTF-8
     @throw type_error.317 if @a j is not an object
     */
     void write_bson(const BasicJsonType& j)
@@ -145,6 +156,8 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
+    @throw type_error.316 if a string value or an object key is not valid
+           UTF-8
     */
     void write_cbor(const BasicJsonType& j)
     {
@@ -211,13 +224,16 @@ class binary_writer
 
             case value_t::string:
             {
+                string_t storage;
+                const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage);
+
                 // step 1: write control byte and the string length
-                write_cbor_head(0x60, j.m_data.m_value.string->size());
+                write_cbor_head(0x60, value.size());
 
                 // step 2: write the string
                 oa.write_characters(
-                      reinterpret_cast<const CharType*>(j.m_data.m_value.string->data()),
+                      reinterpret_cast<const CharType*>(value.data()),
-                      j.m_data.m_value.string->size());
+                      value.size());
                 break;
             }
 
@@ -287,6 +303,17 @@ class binary_writer
                 // step 2: write each element
                 for (const auto& el : *j.m_data.m_value.object)
                 {
+                    // el.first is checked here, against the object as
+                    // diagnostics context, because write_cbor(el.first)
+                    // converts it to a temporary basic_json that would be
+                    // used as the context instead; for error_handler_t::keep
+                    // and ::replace/::ignore the recursive write_cbor(el.first)
+                    // call below handles the key like any other string, so no
+                    // separate check is needed here for those
+                    if (error_handler == error_handler_t::strict)
+                    {
+                        check_utf8(el.first, j);
+                    }
                     write_cbor(el.first);
                     write_cbor(el.second);
                 }
@@ -434,8 +461,11 @@ class binary_writer
 
             case value_t::string:
             {
+                string_t storage;
+                const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage);
+
                 // step 1: write control byte and the string length
-                const auto N = to_msgpack_length(j.m_data.m_value.string->size(), j);
+                const auto N = to_msgpack_length(value.size(), j);
                 if (N <= 31)
                 {
                     // fixstr
@@ -462,8 +492,8 @@ class binary_writer
 
                 // step 2: write the string
                 oa.write_characters(
-                      reinterpret_cast<const CharType*>(j.m_data.m_value.string->data()),
+                      reinterpret_cast<const CharType*>(value.data()),
-                      j.m_data.m_value.string->size());
+                      value.size());
                 break;
             }
 
@@ -610,6 +640,13 @@ class binary_writer
                 // step 2: write each element
                 for (const auto& el : *j.m_data.m_value.object)
                 {
+                    // as in write_cbor, el.first is checked here against the
+                    // object as diagnostics context; the recursive call below
+                    // handles keep/replace/ignore like any other string
+                    if (error_handler == error_handler_t::strict)
+                    {
+                        check_utf8(el.first, j);
+                    }
                     write_msgpack(el.first);
                     write_msgpack(el.second);
                 }
@@ -629,6 +666,8 @@ class binary_writer
     @param[in] add_prefix  whether prefixes need to be used for this value
     @param[in] use_bjdata  whether write in BJData format, default is false
     @param[in] bjdata_version  which BJData version to use, default is draft2
+    @throw type_error.316 if a string value or an object key is not valid
+           UTF-8
     */
     void write_ubjson(const BasicJsonType& j, const bool use_count,
                       const bool use_type, const bool add_prefix = true,
@@ -678,14 +717,17 @@ class binary_writer
 
             case value_t::string:
             {
+                string_t storage;
+                const string_t& value = sanitize_utf8_for_write(*j.m_data.m_value.string, j, storage);
+
                 if (add_prefix)
                 {
                     oa.write_character(to_char_type('S'));
                 }
-                write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata);
+                write_number_with_ubjson_prefix(value.size(), true, use_bjdata);
                 oa.write_characters(
-                      reinterpret_cast<const CharType*>(j.m_data.m_value.string->data()),
+                      reinterpret_cast<const CharType*>(value.data()),
-                      j.m_data.m_value.string->size());
+                      value.size());
                 break;
             }
 
@@ -840,10 +882,12 @@ class binary_writer
 
                 for (const auto& el : *j.m_data.m_value.object)
                 {
-                    write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);
+                    string_t storage;
+                    const string_t& key = sanitize_utf8_for_write(el.first, j, storage);
+                    write_number_with_ubjson_prefix(key.size(), true, use_bjdata);
                     oa.write_characters(
-                          reinterpret_cast<const CharType*>(el.first.data()),
+                          reinterpret_cast<const CharType*>(key.data()),
-                          el.first.size());
+                          key.size());
                     write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version);
                 }
 
@@ -884,8 +928,12 @@ class binary_writer
     /*!
     @return The size of a BSON document entry header, including the id marker
             and the entry name size (and its null-terminator).
+    @throw out_of_range.409 if @a name contains U+0000, before anything is
+           written
+    @throw type_error.316 if @a name is not valid UTF-8, before anything is
+           written
     */
-    static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)
+    std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)
     {
         const auto it = name.find(static_cast<typename string_t::value_type>(0));
         if (JSON_HEDLEY_UNLIKELY(it != BasicJsonType::string_t::npos))
@@ -893,8 +941,10 @@ class binary_writer
             JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j));
         }
 
-        static_cast<void>(j);
+        string_t storage;
-        return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
+        const string_t& sanitized = sanitize_utf8_for_write(name, j, storage);
+
+        return /*id*/ 1ul + sanitized.size() + /*zero-terminator*/1u;
     }
 
     /*!
@@ -914,14 +964,28 @@ class binary_writer
 
     /*!
     @brief Writes the given @a element_type and @a name to the output adapter
+
+    @a name has already been validated (and, for @ref error_handler_t::strict,
+    found well-formed) by @ref calc_bson_entry_header_size during the earlier
+    size pass, so only @ref error_handler_t::replace / @ref
+    error_handler_t::ignore need to sanitize it again here, to actually write
+    the bytes that size was computed from.
     */
     void write_bson_entry_header(const string_t& name,
                                  const std::uint8_t element_type)
     {
         oa.write_character(to_char_type(element_type));
-        oa.write_characters(
+
-              reinterpret_cast<const CharType*>(name.data()),
+        if (error_handler == error_handler_t::keep || error_handler == error_handler_t::strict || is_valid_utf8(name))
-              name.size());
+        {
+            oa.write_characters(reinterpret_cast<const CharType*>(name.data()), name.size());
+        }
+        else
+        {
+            const string_t sanitized = sanitize_utf8(name, error_handler);
+            oa.write_characters(reinterpret_cast<const CharType*>(sanitized.data()), sanitized.size());
+        }
+
         // the terminating null byte is written explicitly rather than taken
         // from the buffer, so that string_t::data() need not be null-terminated
         oa.write_character(to_char_type(0x00));
@@ -949,24 +1013,50 @@ class binary_writer
 
     /*!
     @return The size of the BSON-encoded string in @a value
+    @throw type_error.316 if @a value is not valid UTF-8, before anything is
+           written
+
+    @note The UTF-8 check is skipped if @a value is already too long for the
+          32-bit BSON length field (@ref to_bson_length rejects it later, once
+          the size of the whole document is known); this also keeps the check
+          from reading past a StringType that reports a size larger than what
+          it actually holds.
     */
-    static std::size_t calc_bson_string_size(const string_t& value)
+    std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j)
     {
+        if (JSON_HEDLEY_LIKELY(value_in_range_of<std::int32_t>(value.size())))
+        {
+            string_t storage;
+            const string_t& sanitized = sanitize_utf8_for_write(value, j, storage);
+            return sizeof(std::int32_t) + sanitized.size() + 1ul;
+        }
         return sizeof(std::int32_t) + value.size() + 1ul;
     }
 
     /*!
     @brief Writes a BSON element with key @a name and string value @a value
+
+    @a value has already been validated (and, for @ref error_handler_t::strict,
+    found well-formed) by @ref calc_bson_string_size during the earlier size
+    pass, so only @ref error_handler_t::replace / @ref error_handler_t::ignore
+    need to sanitize it again here, to actually write the bytes that size was
+    computed from.
     */
     void write_bson_string(const string_t& name,
                            const string_t& value)
     {
         write_bson_entry_header(name, 0x02);
 
-        write_number<std::int32_t>(to_bson_length(value.size() + 1ul), true);
+        const bool sanitize = error_handler != error_handler_t::keep
+                              && error_handler != error_handler_t::strict
+                              && !is_valid_utf8(value);
+        const string_t sanitized = sanitize ? sanitize_utf8(value, error_handler) : string_t{};
+        const string_t& written = sanitize ? sanitized : value;
+
+        write_number<std::int32_t>(to_bson_length(written.size() + 1ul), true);
         oa.write_characters(
-              reinterpret_cast<const CharType*>(value.data()),
+              reinterpret_cast<const CharType*>(written.data()),
-              value.size());
+              written.size());
         // the terminating null byte is written explicitly rather than taken
         // from the buffer, so that string_t::data() need not be null-terminated
         oa.write_character(to_char_type(0x00));
@@ -1080,8 +1170,10 @@ class binary_writer
             is neither an object nor an array
     @throw out_of_range.415 if @a j is binary with a subtype that does not fit
            into a byte, before anything is written
+    @throw type_error.316 if @a j is a string that is not valid UTF-8, before
+           anything is written
     */
-    static std::size_t calc_bson_value_size(const BasicJsonType& j)
+    std::size_t calc_bson_value_size(const BasicJsonType& j)
     {
         switch (j.type())
         {
@@ -1101,7 +1193,7 @@ class binary_writer
                 return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned);
 
             case value_t::string:
-                return calc_bson_string_size(*j.m_data.m_value.string);
+                return calc_bson_string_size(*j.m_data.m_value.string, j);
 
             case value_t::null:
                 return 0ul;
@@ -1214,8 +1306,10 @@ class binary_writer
            written
     @throw out_of_range.415 if a binary value's subtype does not fit into a
            byte, before anything is written
+    @throw type_error.316 if a string value or a key is not valid UTF-8,
+           before anything is written
     */
-    static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector<std::size_t>& nested_sizes)
+    std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector<std::size_t>& nested_sizes)
     {
         // the object or array whose entries are being sized, and the ones it
         // is in; nothing is allocated unless the document nests
@@ -2092,7 +2186,7 @@ class binary_writer
     */
     void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context)
     {
-        check_bon8_utf8(s, context);
+        check_utf8(s, context);
 
         // a string that follows another string terminates it
         if (string_open)
@@ -2122,7 +2216,7 @@ class binary_writer
     @throw type_error.316 if @a s is not valid UTF-8; the message names the
            first byte of the first invalid or incomplete sequence
     */
-    static void check_bon8_utf8(const string_t& s, const BasicJsonType& context)
+    static void check_utf8(const string_t& s, const BasicJsonType& context)
     {
         static_cast<void>(context); // only used when exceptions are enabled
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());
@@ -2133,6 +2227,57 @@ class binary_writer
         }
     }
 
+    /*!
+    @brief return @a s as it should be written, honoring @ref error_handler
+
+    Used by @ref write_cbor, @ref write_msgpack, @ref write_ubjson (and so
+    @ref write_bjdata), and the BSON writing functions for string values and
+    object keys; never by @ref write_bon8, which always validates, since UTF-8
+    lead bytes are structural there.
+
+    - @ref error_handler_t::keep: @a s is returned unchanged, without even
+      checking it (the behavior of release 3.12.0 and earlier).
+    - @ref error_handler_t::strict: @ref check_utf8 is called, which throws
+      type_error.316 if @a s is not valid UTF-8.
+    - @ref error_handler_t::replace / @ref error_handler_t::ignore: @a s is
+      sanitized into @a storage with exactly the rules @ref
+      serializer::dump_escaped_impl uses, so that parsing what @ref
+      basic_json::dump produces for the same string and the same handler
+      yields the same result.
+
+    Well-formed input is never copied: this returns a reference to @a s
+    itself in every case but a sanitized `replace`/`ignore` one, so @a
+    storage must outlive the returned reference only then.
+
+    @param[in] s        the string (value or object key) to write
+    @param[in] context  the value @a s belongs to (for diagnostics)
+    @param[out] storage  backing storage for a sanitized copy
+
+    @return a reference to @a s, or to @a storage once it holds a sanitized copy
+    */
+    const string_t& sanitize_utf8_for_write(const string_t& s, const BasicJsonType& context, string_t& storage) const
+    {
+        switch (error_handler)
+        {
+            case error_handler_t::keep:
+                return s;
+
+            case error_handler_t::strict:
+                check_utf8(s, context);
+                return s;
+
+            case error_handler_t::replace:
+            case error_handler_t::ignore:
+            default:
+                if (is_valid_utf8(s))
+                {
+                    return s;
+                }
+                storage = sanitize_utf8(s, error_handler);
+                return storage;
+        }
+    }
+
     /*!
     @brief write an integer in the shortest encoding
 
@@ -2461,6 +2606,10 @@ class binary_writer
 
     /// the output
     OutputSinkType oa;
+
+    /// how to treat a string value or object key that is not valid UTF-8
+    /// (CBOR, MessagePack, UBJSON, BJData, and BSON; not BON8)
+    const error_handler_t error_handler = binary_writer_default_error_handler();
 };
 
 }  // namespace detail

include/nlohmann/detail/output/error_handler.hpp

@@ -0,0 +1,50 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <nlohmann/detail/abi_macros.hpp>
+
+NLOHMANN_JSON_NAMESPACE_BEGIN
+namespace detail
+{
+
+/// how to treat decoding errors
+///
+/// @ref basic_json::dump uses this to decide what to do with ill-formed
+/// UTF-8 while escaping a string, and the binary writers (@ref
+/// basic_json::to_cbor, @ref basic_json::to_ubjson, @ref
+/// basic_json::to_bjdata, @ref basic_json::to_bson) use it the same way for
+/// string values and object keys. The binary readers (@ref
+/// basic_json::from_cbor, @ref basic_json::from_msgpack, @ref
+/// basic_json::from_ubjson, @ref basic_json::from_bjdata, @ref
+/// basic_json::from_bson) use it to decide whether to check text strings
+/// and object keys for well-formed UTF-8 at all, since none of those
+/// formats requires a decoder to do so.
+enum class error_handler_t
+{
+    strict,  ///< throw a type_error/parse_error exception in case of invalid UTF-8
+    replace, ///< replace invalid UTF-8 sequences with U+FFFD
+    ignore,  ///< ignore invalid UTF-8 sequences
+    keep     ///< keep invalid UTF-8 sequences unchanged
+};
+
+/// the default error handler of the CBOR, UBJSON, BJData, and BSON writers:
+/// error_handler_t::strict if JSON_STRICT_BINARY_UTF8 is enabled, otherwise
+/// error_handler_t::keep (the behavior before version 3.13.0)
+constexpr error_handler_t binary_writer_default_error_handler() noexcept
+{
+#if JSON_STRICT_BINARY_UTF8
+    return error_handler_t::strict;
+#else
+    return error_handler_t::keep;
+#endif
+}
+
+}  // namespace detail
+NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/detail/output/serializer.hpp

@@ -27,6 +27,7 @@
 #include <nlohmann/detail/input/string_scan.hpp>
 #include <nlohmann/detail/macro_scope.hpp>
 #include <nlohmann/detail/meta/cpp_future.hpp>
+#include <nlohmann/detail/output/error_handler.hpp>
 #include <nlohmann/detail/output/output_adapters.hpp>
 #include <nlohmann/detail/recursion_depth_limit.hpp>
 #include <nlohmann/detail/string_concat.hpp>
@@ -41,14 +42,6 @@ namespace detail
 // serialization //
 ///////////////////
 
-/// how to treat decoding errors
-enum class error_handler_t
-{
-    strict,  ///< throw a type_error exception in case of invalid UTF-8
-    replace, ///< replace invalid UTF-8 sequences with U+FFFD
-    ignore   ///< ignore invalid UTF-8 sequences
-};
-
 template<typename BasicJsonType>
 class serializer
 {
@@ -839,6 +832,16 @@ class serializer
                             // EnsureAscii parameter is used, non-ASCII characters
                             if ((codepoint <= 0x1F) || (EnsureAscii && (codepoint >= 0x7F)))
                             {
+                                if (EnsureAscii && error_handler == error_handler_t::keep)
+                                {
+                                    // this character was buffered as raw bytes
+                                    // below in case it turned out to be part of
+                                    // an ill-formed sequence (which is kept as
+                                    // is); now that it decoded to a well-formed
+                                    // code point, undo that and \u-escape it
+                                    // like any other character instead
+                                    bytes = bytes_after_last_accept;
+                                }
                                 if (codepoint <= 0xFFFF)
                                 {
                                     write_u_escape(bytes, static_cast<std::uint16_t>(codepoint));
@@ -937,6 +940,44 @@ class serializer
                             break;
                         }
 
+                        case error_handler_t::keep:
+                        {
+                            // the bytes of this (now abandoned) ill-formed
+                            // sequence seen so far are already buffered below
+                            // and are kept unchanged in the output
+                            if (undumped_chars > 0)
+                            {
+                                // the byte that ended the sequence may be OK
+                                // for itself (e.g., a quote that must still be
+                                // escaped, or the lead byte of a well-formed
+                                // code point), so read it again
+                                --i;
+                            }
+                            else
+                            {
+                                // a byte that cannot start a sequence (e.g.,
+                                // 0xFF or a stray continuation byte) is kept
+                                // as well
+                                string_buffer[bytes++] = s[i];
+                            }
+
+                            // write buffer and reset index; there must be 13 bytes
+                            // left, as this is the maximal number of bytes to be
+                            // written ("\uxxxx\uxxxx\0") for one code point
+                            if (string_buffer.size() - bytes < 13)
+                            {
+                                put_buffer(string_buffer, bytes);
+                                bytes = 0;
+                            }
+
+                            bytes_after_last_accept = bytes;
+                            undumped_chars = 0;
+
+                            // continue processing the string
+                            state = UTF8_ACCEPT;
+                            break;
+                        }
+
                         default:            // LCOV_EXCL_LINE
                             JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
                     }
@@ -945,9 +986,12 @@ class serializer
 
                 default:  // decode found yet incomplete multibyte code point
                 {
-                    if (!EnsureAscii)
+                    if (!EnsureAscii || error_handler == error_handler_t::keep)
                     {
-                        // code point will not be escaped - copy byte to buffer
+                        // code point will not be escaped (or will be kept as
+                        // is if it turns out to be ill-formed) - copy byte to
+                        // buffer; dropped again above if it decodes to a
+                        // well-formed code point that needs \u-escaping
                         string_buffer[bytes++] = s[i];
                     }
                     ++undumped_chars;
@@ -998,6 +1042,14 @@ class serializer
                     break;
                 }
 
+                case error_handler_t::keep:
+                {
+                    // write the ill-formed trailing bytes as is; they were
+                    // buffered above regardless of EnsureAscii
+                    put_buffer(string_buffer, bytes);
+                    break;
+                }
+
                 default:            // LCOV_EXCL_LINE
                     JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE
             }

include/nlohmann/detail/string_concat.hpp

@@ -39,7 +39,6 @@ inline std::size_t concat_length(const char /*c*/, const Args& ... rest)
 template<typename... Args>
 inline std::size_t concat_length(const char* cstr, const Args& ... rest)
 {
-    // cppcheck-suppress ignoredReturnValue
     return ::strlen(cstr) + concat_length(rest...);
 }
 

include/nlohmann/detail/string_utils.hpp

@@ -13,9 +13,11 @@
 #include <cstddef> // size_t
 #include <cstdint> // uint8_t, uint32_t
 #include <string> // string, to_string
+#include <utility> // move
 
 #include <nlohmann/detail/abi_macros.hpp>
 #include <nlohmann/detail/macro_scope.hpp>
+#include <nlohmann/detail/output/error_handler.hpp>
 
 NLOHMANN_JSON_NAMESPACE_BEGIN
 namespace detail
@@ -117,13 +119,14 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally
 written by Björn Hoehrmann. See
 http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
 
-The library checks UTF-8 well-formedness (RFC 3629, section 4) in four
+The library checks UTF-8 well-formedness (RFC 3629, section 4) in three
 places, which differ in speed, diagnostics, and how they read the input:
 
-- decode() and @ref is_valid_utf8 below: the serializer (to escape and, in
+- decode() below: the serializer, to escape and, in strict mode, reject
-  strict mode, reject ill-formed UTF-8 when dumping a string) and the CBOR,
+  ill-formed UTF-8 when dumping a string. The CBOR, MessagePack, BSON,
-  MessagePack, BSON, UBJSON and BJData readers (to reject ill-formed UTF-8 in
+  UBJSON and BJData readers do not use it: none of those specs requires a
-  text strings at decode time).
+  decoder to reject ill-formed UTF-8 in text strings, so the readers keep
+  the bytes as is and leave the check to dump() and the binary writers.
 - the per-lead-byte switch in lexer::scan_string(): JSON text, with a
   diagnostic for each kind of error.
 - validate_one_utf8() and valid_utf8_prefix() in string_scan.hpp: the lexer's
@@ -179,19 +182,19 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std:
 }
 
 /*!
-@brief check whether a string consists solely of valid UTF-8
+@brief check a string for well-formed UTF-8 (RFC 3629, section 4)
 
-Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text
+Used by the binary readers (CBOR, MessagePack, UBJSON, BJData, BSON) when an
-strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the
+@ref error_handler_t other than `keep` is requested for a text string value
-MessagePack/BSON specifications all require text strings to be UTF-8), so
+or object key: none of those formats requires a decoder to reject ill-formed
-that malformed input is caught immediately instead of only surfacing later
+UTF-8 on its own, so the check is opt-in there, unlike the JSON lexer and the
-as a type_error.316 when the resulting value is dumped.
+serializer's @ref decode -based escaping, which always run it.
 
 @param[in] s      the string to check
-@param[in] first  index of the first byte to check; the bytes before it are
+@param[in] first  the index to start checking at
-                  assumed to have been validated already and to end on a
+@return whether `s.substr(first)` is well-formed UTF-8
-                  code point boundary
+
-@return whether @a s (from index @a first on) is valid UTF-8
+@sa @ref decode
 */
 template<typename StringType>
 inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept
@@ -211,5 +214,101 @@ inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noex
     return state == UTF8_ACCEPT;
 }
 
+/*!
+@brief sanitize a string with ill-formed UTF-8 for @ref error_handler_t::replace or @ref error_handler_t::ignore
+
+Replaces every maximal ill-formed subsequence with U+FFFD (`replace`) or
+drops it (`ignore`), using exactly the same boundaries @ref
+serializer::dump_escaped_impl uses while escaping a string: a byte that does
+not extend the sequence started by the previous byte(s) is reread as the
+start of a new one, instead of being swallowed along with them.
+
+@pre @a error_handler is @ref error_handler_t::replace or @ref error_handler_t::ignore
+@note Well-formed input is copied through unchanged, including bytes (e.g.
+      control characters or quotes) that @ref serializer::dump_escaped_impl
+      would itself escape; this function only concerns itself with
+      well-formedness, not with producing valid JSON text.
+
+@param[in] s              the string to sanitize
+@param[in] error_handler  @ref error_handler_t::replace or @ref error_handler_t::ignore
+
+@return @a s with every ill-formed subsequence replaced or removed
+
+@sa @ref decode
+*/
+template<typename StringType>
+inline StringType sanitize_utf8(const StringType& s, const error_handler_t error_handler)
+{
+    JSON_ASSERT(error_handler == error_handler_t::replace || error_handler == error_handler_t::ignore);
+
+    StringType result;
+    result.reserve(s.size());
+
+    std::uint32_t codepoint = 0;
+    std::uint8_t state = UTF8_ACCEPT;
+    // length of result after the last accepted code point
+    std::size_t result_len_after_last_accept = 0;
+    // whether bytes of an as yet unresolved sequence were already appended
+    bool pending = false;
+
+    for (std::size_t i = 0; i < s.size(); ++i)
+    {
+        switch (decode(state, codepoint, static_cast<std::uint8_t>(s[i])))
+        {
+            case UTF8_ACCEPT: // decode found a well-formed code point
+            {
+                result.push_back(s[i]);
+                result_len_after_last_accept = result.size();
+                pending = false;
+                break;
+            }
+
+            case UTF8_REJECT: // decode found an ill-formed byte
+            {
+                // in case we saw this byte for the first time, read it again,
+                // because it may be fine for itself, just not for the
+                // sequence that came before it
+                if (pending)
+                {
+                    --i;
+                }
+
+                // drop the bytes of the ill-formed sequence buffered below
+                result.resize(result_len_after_last_accept);
+
+                if (error_handler == error_handler_t::replace)
+                {
+                    result.append("\xEF\xBF\xBD");
+                    result_len_after_last_accept = result.size();
+                }
+
+                pending = false;
+                state = UTF8_ACCEPT;
+                break;
+            }
+
+            default: // decode found yet incomplete multibyte code point
+            {
+                result.push_back(s[i]);
+                pending = true;
+                break;
+            }
+        }
+    }
+
+    // the string ended with an incomplete sequence
+    if (state != UTF8_ACCEPT)
+    {
+        result.resize(result_len_after_last_accept);
+
+        if (error_handler == error_handler_t::replace)
+        {
+            result.append("\xEF\xBF\xBD");
+        }
+    }
+
+    return result;
+}
+
 }  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END

include/nlohmann/ordered_map.hpp

@@ -74,31 +74,59 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,
         return *this;
     }
 
-    std::pair<iterator, bool> emplace(const key_type& key, T&& t)
+private:
+    /// @brief find the entry for @a key, for either constness of @a self
+    /// @note the single place that performs the linear key search
+    template<typename Self, typename KeyType>
+    static auto find_impl(Self& self, KeyType&& key) -> decltype(self.begin())
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        for (auto it = self.begin(); it != self.end(); ++it)
         {
-            if (m_compare(it->first, key))
+            if (self.m_compare(it->first, key))
+            {
+                return it;
+            }
+        }
+        return self.end();
+    }
+
+    /// @brief remove the entry @a it points to, preserving order
+    /// @note keys are not movable, so the tail is destroyed and re-constructed in place
+    void erase_at(iterator it)
+    {
+        for (auto next = it; ++next != this->end(); ++it)
+        {
+            it->~value_type(); // Destroy but keep allocation
+            new (&*it) value_type{std::move(*next)};
+        }
+        Container::pop_back();
+    }
+
+public:
+    template<class V, detail::enable_if_t<
+                 detail::is_constructible<T, V>::value, int> = 0>
+    std::pair<iterator, bool> emplace(const key_type& key, V && t)
+    {
+        const auto it = find_impl(*this, key);
+        if (it != this->end())
         {
             return {it, false};
         }
-        }
+        append(key, std::forward<V>(t));
-        append(key, std::forward<T>(t));
         return {std::prev(this->end()), true};
     }
 
-    template<class KeyType, detail::enable_if_t<
+    template<class KeyType, class V, detail::enable_if_t<
-                 detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
+                 detail::conjunction<detail::is_usable_as_key_type<key_compare, key_type, KeyType>,
-    std::pair<iterator, bool> emplace(KeyType && key, T && t)
+                                     detail::is_constructible<T, V>>::value, int> = 0>
+    std::pair<iterator, bool> emplace(KeyType && key, V && t)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
-        {
+        if (it != this->end())
-            if (m_compare(it->first, key))
         {
             return {it, false};
         }
-        }
+        append(std::forward<KeyType>(key), std::forward<V>(t));
-        append(std::forward<KeyType>(key), std::forward<T>(t));
         return {std::prev(this->end()), true};
     }
 
@@ -128,76 +156,56 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,
 
     T& at(const key_type& key)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it == this->end())
         {
-            if (m_compare(it->first, key))
-            {
-                return it->second;
-            }
-        }
-
             JSON_THROW(std::out_of_range("key not found"));
         }
+        return it->second;
+    }
 
     template<class KeyType, detail::enable_if_t<
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     T & at(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it == this->end())
         {
-            if (m_compare(it->first, key))
-            {
-                return it->second;
-            }
-        }
-
             JSON_THROW(std::out_of_range("key not found"));
         }
+        return it->second;
+    }
 
     const T& at(const key_type& key) const
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it == this->end())
         {
-            if (m_compare(it->first, key))
-            {
-                return it->second;
-            }
-        }
-
             JSON_THROW(std::out_of_range("key not found"));
         }
+        return it->second;
+    }
 
     template<class KeyType, detail::enable_if_t<
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     const T & at(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it == this->end())
         {
-            if (m_compare(it->first, key))
-            {
-                return it->second;
-            }
-        }
-
             JSON_THROW(std::out_of_range("key not found"));
         }
+        return it->second;
+    }
 
     size_type erase(const key_type& key)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it != this->end())
         {
-            if (m_compare(it->first, key))
+            erase_at(it);
-            {
-                // Since we cannot move const Keys, re-construct them in place
-                for (auto next = it; ++next != this->end(); ++it)
-                {
-                    it->~value_type(); // Destroy but keep allocation
-                    new (&*it) value_type{std::move(*next)};
-                }
-                Container::pop_back();
             return 1;
         }
-        }
         return 0;
     }
 
@@ -205,20 +213,12 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     size_type erase(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, key);
+        if (it != this->end())
         {
-            if (m_compare(it->first, key))
+            erase_at(it);
-            {
-                // Since we cannot move const Keys, re-construct them in place
-                for (auto next = it; ++next != this->end(); ++it)
-                {
-                    it->~value_type(); // Destroy but keep allocation
-                    new (&*it) value_type{std::move(*next)};
-                }
-                Container::pop_back();
             return 1;
         }
-        }
         return 0;
     }
 
@@ -282,80 +282,38 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,
 
     size_type count(const key_type& key) const
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key) != this->end() ? 1 : 0;
-        {
-            if (m_compare(it->first, key))
-            {
-                return 1;
-            }
-        }
-        return 0;
     }
 
     template<class KeyType, detail::enable_if_t<
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     size_type count(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key) != this->end() ? 1 : 0;
-        {
-            if (m_compare(it->first, key))
-            {
-                return 1;
-            }
-        }
-        return 0;
     }
 
     iterator find(const key_type& key)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key);
-        {
-            if (m_compare(it->first, key))
-            {
-                return it;
-            }
-        }
-        return Container::end();
     }
 
     template<class KeyType, detail::enable_if_t<
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     iterator find(KeyType && key) // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key);
-        {
-            if (m_compare(it->first, key))
-            {
-                return it;
-            }
-        }
-        return Container::end();
     }
 
     const_iterator find(const key_type& key) const
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key);
-        {
-            if (m_compare(it->first, key))
-            {
-                return it;
-            }
-        }
-        return Container::end();
     }
 
     template<class KeyType, detail::enable_if_t<
                  detail::is_usable_as_key_type<key_compare, key_type, KeyType>::value, int> = 0>
     const_iterator find(KeyType && key) const // NOLINT(cppcoreguidelines-missing-std-forward)
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        return find_impl(*this, key);
-        {
-            if (m_compare(it->first, key))
-            {
-                return it;
-            }
-        }
-        return Container::end();
     }
 
     std::pair<iterator, bool> insert( value_type&& value )
@@ -365,13 +323,11 @@ template <class Key, class T, class IgnoredLess = std::less<Key>,
 
     std::pair<iterator, bool> insert( const value_type& value )
     {
-        for (auto it = this->begin(); it != this->end(); ++it)
+        const auto it = find_impl(*this, value.first);
-        {
+        if (it != this->end())
-            if (m_compare(it->first, value.first))
         {
             return {it, false};
         }
-        }
         append(value);
         return {--this->end(), true};
     }

single_include/nlohmann/json_fwd.hpp

@@ -63,6 +63,10 @@
     #define JSON_STRICT_NUL_HANDLING 0
 #endif
 
+#ifndef JSON_STRICT_BINARY_UTF8
+    #define JSON_STRICT_BINARY_UTF8 0
+#endif
+
 #if JSON_DIAGNOSTICS
     #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag
 #else
@@ -99,14 +103,20 @@
     #define NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING
 #endif
 
+#if JSON_STRICT_BINARY_UTF8
+    #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8 _sbu8
+#else
+    #define NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8
+#endif
+
 #ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION
     #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0
 #endif
 
 // Construct the namespace ABI tags component
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f) json_abi ## a ## b ## c ## d ## e ## f
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g) json_abi ## a ## b ## c ## d ## e ## f ## g
-#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f) \
+#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b, c, d, e, f, g) \
-    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f)
+    NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b, c, d, e, f, g)
 
 #define NLOHMANN_JSON_ABI_TAGS                                       \
     NLOHMANN_JSON_ABI_TAGS_CONCAT(                                   \
@@ -115,7 +125,8 @@
             NLOHMANN_JSON_ABI_TAG_DIAGNOSTIC_POSITIONS,              \
             NLOHMANN_JSON_ABI_TAG_BRACE_INIT_COPY_SEMANTICS,         \
             NLOHMANN_JSON_ABI_TAG_PRECISE_STREAM_POSITION,           \
-            NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING)
+            NLOHMANN_JSON_ABI_TAG_STRICT_NUL_HANDLING,               \
+            NLOHMANN_JSON_ABI_TAG_STRICT_BINARY_UTF8)
 
 // Construct the namespace version component
 #define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \

src/modules/json.cppm

@@ -47,6 +47,7 @@ inline namespace json_literals
 namespace detail
 {
     using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_callback_parser;
+    using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_parser;
     using NLOHMANN_JSON_NAMESPACE::detail::unknown_size;
 } // namespace detail