Merge remote-tracking branch 'origin/develop' into claude/fix-issue-3989-db7e45

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/CONTRIBUTING.md

@@ -205,6 +205,9 @@ API of the 3.x.y version is broken. This includes:
 - Changing access specifiers.
 - Changing default arguments.
 
+What is and is not covered by this guarantee is described in the
+[roadmap](https://json.nlohmann.me/community/roadmap/#api-stability).
+
 Although these guidelines may seem restrictive, they are essential for maintaining the library’s utility.
 
 Breaking changes may be introduced when they are guarded with a feature macro such as

.github/external_ci/appveyor.yml

@@ -87,9 +87,4 @@ build_script:
   - cmake --build . --config "%configuration%" --parallel 2
 
 test_script:
-  - if "%configuration%"=="Release" ctest -C "%configuration%" --parallel 2 --output-on-failure
-  # On Debug builds, skip test-unicode_all
-  # as it is extremely slow to run and cause
-  # occasional timeouts on AppVeyor.
-  # More info: https://github.com/nlohmann/json/pull/1570
-  - if "%configuration%"=="Debug" ctest --exclude-regex "test-unicode" -C "%configuration%" --parallel 2 --output-on-failure
+  - ctest -C "%configuration%" --parallel 2 --output-on-failure

.github/workflows/windows.yml

@@ -178,7 +178,7 @@ jobs:
       - name: Build
         run: cmake --build build --parallel 10
       - name: Test
-        run: cd build ; ctest -j 10 -C Debug --exclude-regex "test-unicode" --output-on-failure
+        run: cd build ; ctest -j 10 -C Debug --output-on-failure
 
   clang-cl-12:
     runs-on: windows-2022
@@ -195,7 +195,7 @@ jobs:
       - name: Build
         run: cmake --build build --config Debug --parallel 10
       - name: Test
-        run: cd build ; ctest -j 10 -C Debug --exclude-regex "test-unicode" --output-on-failure
+        run: cd build ; ctest -j 10 -C Debug --output-on-failure
 
   ci_module_cpp20:
     runs-on: windows-2022

README.md

@@ -7,7 +7,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/nlohmann/json/badge.svg?branch=develop)](https://coveralls.io/github/nlohmann/json?branch=develop)
 [![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/e0d1a9d5d6fd46fcb655c4cb930bb3e8)](https://app.codacy.com/gh/nlohmann/json/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
-[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/json.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:json)
+[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/json.svg)](https://issues.oss-fuzz.com/issues?q=project:json)
 [![Try online](https://img.shields.io/badge/try-online-blue.svg)](https://wandbox.org/permlink/1mp10JbaANo6FUc7)
 [![Documentation](https://img.shields.io/badge/docs-mkdocs-blue.svg)](https://json.nlohmann.me)
 [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/develop/LICENSE.MIT)
@@ -494,7 +494,7 @@ bool key(string_t& val);
 bool parse_error(std::size_t position, const std::string& last_token, const detail::exception& ex);
 ```
 
-The return value of each function determines whether parsing should proceed.
+The return value of each function determines whether parsing should proceed. For `parse_error`, returning `true` [recovers from the error](https://json.nlohmann.me/features/parsing/error_recovery/): the parser repairs the input and continues.
 
 To implement your own SAX handler, proceed as follows:
 
@@ -502,7 +502,7 @@ To implement your own SAX handler, proceed as follows:
 2. Create an object of your SAX interface class, e.g. `my_sax`.
 3. Call `bool json::sax_parse(input, &my_sax)`; where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface.
 
-Note the `sax_parse` function only returns a `bool` indicating the result of the last executed SAX event. It does not return a  `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp).
+Note the `sax_parse` function only returns a `bool` indicating whether the input was parsed without errors and no SAX event returned `false`. It does not return a  `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error -- it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file [`json_sax.hpp`](https://github.com/nlohmann/json/blob/develop/include/nlohmann/detail/input/json_sax.hpp).
 
 ### STL-like access
 

cmake/ci.cmake

@@ -456,13 +456,14 @@ add_custom_target(ci_test_single_header
 # Valgrind.
 ###############################################################################
 
+# The Unicode test (~17M assertions) is too slow under Valgrind.
 add_custom_target(ci_test_valgrind
     COMMAND CXX=${GCC_TOOL} ${CMAKE_COMMAND}
         -DCMAKE_BUILD_TYPE=Debug -GNinja
         -DJSON_BuildTests=ON -DJSON_Valgrind=ON
         -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_valgrind
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_valgrind
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --parallel ${N} --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_valgrind && ${CMAKE_CTEST_COMMAND} -L valgrind --exclude-regex "test-unicode" --parallel ${N} --output-on-failure
     COMMENT "Compile and test with Valgrind"
 )
 
@@ -787,7 +788,7 @@ foreach(COMPILER g++-4.8 g++-4.9 g++-5 g++-6 g++-7 g++-8 g++-9 g++-10 g++-11 cla
                 -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_compiler_${COMPILER}
                 ${ADDITIONAL_FLAGS}
             COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER}
-            COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure
+            COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_${COMPILER} && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure
             COMMENT "Compile and test with ${COMPILER}"
         )
     endif()
@@ -801,7 +802,7 @@ add_custom_target(ci_test_compiler_default
         -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_compiler_default
         ${ADDITIONAL_FLAGS}
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_compiler_default --parallel ${N}
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_default && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" -LE git_required --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_compiler_default && ${CMAKE_CTEST_COMMAND} --parallel ${N} -LE git_required --output-on-failure
     COMMENT "Compile and test with default C++ compiler"
 )
 
@@ -839,7 +840,7 @@ add_custom_target(ci_icpc
         -DJSON_BuildTests=ON -DJSON_FastTests=ON
         -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_icpc
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_icpc
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_icpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_icpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure
     COMMENT "Compile and test with ICPC"
 )
 
@@ -850,7 +851,7 @@ add_custom_target(ci_icpx
         -DJSON_BuildTests=ON -DJSON_FastTests=ON
         -S${PROJECT_SOURCE_DIR} -B${PROJECT_BINARY_DIR}/build_icpx
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_icpx
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_icpx && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode" --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_icpx && ${CMAKE_CTEST_COMMAND} --parallel ${N} --output-on-failure
     COMMENT "Compile and test with ICPX (Intel oneAPI DPC++/C++)"
 )
 
@@ -886,7 +887,7 @@ add_custom_target(ci_nvhpc
     COMMAND ${CMAKE_COMMAND} --build ${PROJECT_BINARY_DIR}/build_nvhpc
     # the pipes are escaped so the surrounding shell passes them to ctest verbatim
     # instead of treating them as shell pipe operators
-    COMMAND cd ${PROJECT_BINARY_DIR}/build_nvhpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-unicode\\|test-comparison_cpp20\\|test-comparison_legacy_cpp20\\|test-constructor1_cpp11\\|test-deserialization_cpp20" --output-on-failure
+    COMMAND cd ${PROJECT_BINARY_DIR}/build_nvhpc && ${CMAKE_CTEST_COMMAND} --parallel ${N} --exclude-regex "test-comparison_cpp20\\|test-comparison_legacy_cpp20\\|test-constructor1_cpp11\\|test-deserialization_cpp20" --output-on-failure
     COMMENT "Compile and test with NVIDIA HPC SDK (nvc++)"
 )
 

docs/mkdocs/docs/api/basic_json/cbor_tag_handler_t.md

@@ -9,10 +9,10 @@ enum class cbor_tag_handler_t
 };
 ```
 
-This enumeration is used in the [`from_cbor`](from_cbor.md) function to choose how to treat tags:
+This enumeration is used in [`from_cbor`](from_cbor.md) and [`sax_parse`](sax_parse.md) to choose how to treat tags:
 
 error
-:   throw a `parse_error` exception in case of a tag
+:   report a parse error in case of a tag (the `from_cbor` overloads throw a `parse_error` exception by default)
 
 ignore
 :   ignore tags

docs/mkdocs/docs/api/basic_json/sax_parse.md

@@ -8,7 +8,8 @@ static bool sax_parse(InputType&& i,
                       input_format_t format = input_format_t::json,
                       const bool strict = true,
                       const bool ignore_comments = false,
-                      const bool ignore_trailing_commas = false);
+                      const bool ignore_trailing_commas = false,
+                      const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error);
 
 // (2)
 template<class IteratorType, class SAX, class SentinelType = IteratorType>
@@ -17,13 +18,14 @@ static bool sax_parse(IteratorType first, SentinelType last,
                       input_format_t format = input_format_t::json,
                       const bool strict = true,
                       const bool ignore_comments = false,
-                      const bool ignore_trailing_commas = false);
+                      const bool ignore_trailing_commas = false,
+                      const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error);
 ```
 
 Read from input and generate SAX events
 
 1. Read from a compatible input.
-2. Read from a pair of character iterators, or an iterator and a sentinel of a different type (C++20 ranges support)
+2. Read from a pair of character iterators, or an iterator and a sentinel of a different type (C++20 ranges support).
     
     The value_type of the iterator must be an integral type with a size of 1, 2, or 4 bytes, which will be interpreted
     respectively as UTF-8, UTF-16, and UTF-32. If `SentinelType` differs from `IteratorType`, it must be comparable to
@@ -82,6 +84,10 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index
 :   whether trailing commas in arrays or objects should be ignored and treated like whitespace (`#!cpp true`) or yield a parse error
     (`#!cpp false`); (optional, `#!cpp false` by default)
 
+`tag_handler` (in)
+:   how to handle CBOR tags; see [`cbor_tag_handler_t`](cbor_tag_handler_t.md). Ignored for formats other than CBOR
+    (optional, `cbor_tag_handler_t::error` by default).
+
 `first` (in)
 :   iterator to the start of a character range
 
@@ -90,7 +96,9 @@ The SAX event lister must follow the interface of [`json_sax`](../json_sax/index
 
 ## Return value
 
-return value of the last processed SAX event
+`#!cpp true` if the input was parsed without errors and no SAX event returned `#!cpp false`; `#!cpp false` otherwise.
+In particular, the result is `#!cpp false` for input with errors, even if the SAX parser recovered from all of them
+(see [error recovery](../../features/parsing/error_recovery.md)).
 
 ## Exception safety
 
@@ -137,7 +145,9 @@ A UTF-8 byte order mark is silently ignored.
 - Added in version 3.2.0.
 - Ignoring comments via `ignore_comments` added in version 3.9.0.
 - Added `ignore_trailing_commas` in version 3.13.0.
+- Added `tag_handler` in version 3.13.0.
 - Extended container support (1) to include types with lvalue-only ADL `begin`/`end` (matching `std::begin`/`std::end` semantics) in version 3.13.0.
+- Recovering from parse errors (see [`parse_error`](../json_sax/parse_error.md)) added in version 3.13.0.
 - Extended overload (2) to accept heterogeneous iterator+sentinel pairs (C++20 ranges support) in version 3.13.0.
 - `JSON_PRECISE_STREAM_POSITION` added in version 3.13.0 to optionally leave a `#!cpp std::istream` positioned right
   after the parsed value when `strict` is `#!cpp false`.

docs/mkdocs/docs/api/json_sax/index.md

@@ -7,7 +7,8 @@ struct json_sax;
 
 This class describes the SAX interface used by [sax_parse](../basic_json/sax_parse.md). Each function is called in
 different situations while the input is parsed. The boolean return value informs the parser whether to continue
-processing the input.
+processing the input; for [`parse_error`](parse_error.md), it decides whether to
+[recover from the error](../../features/parsing/error_recovery.md).
 
 For instance, parsing the JSON text `{"a": [1, true]}` triggers the following callbacks, in order:
 

docs/mkdocs/docs/api/json_sax/parse_error.md

@@ -21,11 +21,18 @@ A parse error occurred.
 
 ## Return value
 
-Whether parsing should proceed (**must return `#!cpp false`**).
+Whether to recover from the error:
+
+- `#!cpp false` stops parsing.
+- `#!cpp true` recovers from the error: the error is repaired and parsing continues. If that is not possible, which
+  happens in the binary formats when the end of the item with the error is unknown, the value read so far is completed
+  and parsing stops. See [error recovery](../../features/parsing/error_recovery.md) for how errors are repaired.
+
+Either way, [`sax_parse`](../basic_json/sax_parse.md) returns `#!cpp false`.
 
 ## Examples
 
-??? example
+??? example "Example: (1) the SAX interface"
 
     The example below shows how the SAX interface is used.
 
@@ -39,12 +46,29 @@ Whether parsing should proceed (**must return `#!cpp false`**).
     --8<-- "examples/sax_parse.output"
     ```
 
+??? example "Example: (2) recovering from errors"
+
+    The example below shows how a SAX parser recovers from errors.
+
+    ```cpp
+    --8<-- "examples/sax_parse__error_recovery.cpp"
+    ```
+    
+    Output:
+    
+    ```
+    --8<-- "examples/sax_parse__error_recovery.output"
+    ```
+
 ## See also
 
 - [sax_parse](../basic_json/sax_parse.md) - SAX parser
 - [Parsing and Exceptions](../../features/parsing/parse_exceptions.md) - the article on handling parse errors without
   exceptions
+- [Error Recovery](../../features/parsing/error_recovery.md) - the article on recovering from parse errors
 
 ## Version history
 
 - Added in version 3.2.0.
+- Returning `#!cpp true` recovers from the error since version 3.13.0; before, parsing stopped, but the result of
+  [`sax_parse`](../basic_json/sax_parse.md) could be wrong.

docs/mkdocs/docs/api/macros/json_brace_init_copy_semantics.md

@@ -50,9 +50,11 @@ The default value is `0` (disabled — existing behavior is preserved).
     ```
 
     Code that relies on these producing arrays must use `json::array()` instead (see below). Lists with more than one
-    element, and a single `[string, value]` pair such as `{{"key", "value"}}`, which still creates an object, are not
-    affected. The library's own conversions are not affected either: for example, `std::tuple<int>{5}` still becomes
-    `[5]`.
+    element, and a single `[string, value]` pair *written as a braced list*, such as `{{"key", "value"}}`, which still
+    creates an object, are not affected. This exception is based on how the pair is written, not on the shape of its
+    value: an existing JSON value that happens to be a two-element array with a string as its first element, such as
+    `json arr = {"key", 42};`, is still copied by `json j{arr};` rather than turned into an object. The library's own
+    conversions are not affected either: for example, `std::tuple<int>{5}` still becomes `[5]`.
 
 !!! note "ABI compatibility"
 

docs/mkdocs/docs/community/roadmap.md

@@ -25,9 +25,7 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js
 
 ## What the project will not do
 
-- **Break the public API of version 3.x.** See the
-  [contribution guidelines](https://github.com/nlohmann/json/blob/develop/.github/CONTRIBUTING.md#break-the-public-api)
-  for what counts as a breaking change.
+- **Break the public API of version 3.x.** See [API stability](#api-stability) for what this covers.
 - **Require a newer C++ standard than C++11.**
 - **Break JSON conformance** or enable non-standard extensions by default.
 - **Add dependencies** or require a build step. The library remains header-only, and the single header
@@ -35,6 +33,32 @@ work items are tracked in the [GitHub milestones](https://github.com/nlohmann/js
 - **Trade simplicity for speed or memory efficiency.** Performance improvements are welcome, but the library is not
   meant to compete with the fastest JSON libraries, see [Design goals](../home/design_goals.md).
 
+## API stability
+
+Releases follow [semantic versioning](https://semver.org): a minor or patch release of version 3.x does not break code
+that uses the public API. In particular, a 3.x release does not:
+
+- change the signature of a function (its parameter types, return type, number of parameters, or the const-ness of a
+  member function);
+- remove or rename a function or class;
+- change which exceptions a function throws, or the [exception ids](../home/exceptions.md);
+- change access specifiers or default arguments.
+
+Exceptions to these rules, for instance when fixing a bug requires changing the exception a function throws, are
+documented in the [release notes](../home/releases.md).
+
+The following are **not** part of the public API and may change in any release, including patch releases:
+
+- The text of exception messages returned by `what()`. Use the [exception id](../home/exceptions.md) to tell errors
+  apart.
+- The ABI, including `sizeof(basic_json)` and the memory layout of its values. Recompile your code when you upgrade the
+  library. The [versioned inline namespace](../features/namespace.md) turns mixing versions into a link error.
+- Everything in namespace `nlohmann::detail`, and macros and type traits that are not documented in the
+  [API reference](../api/basic_json/index.md).
+
+Changes that would break the public API are only added behind a macro whose default keeps the 3.x behavior, see
+[Version 4.0](#version-40).
+
 ## Version 4.0
 
 There is no release date for version 4.0 yet. Proposals that need a major version, for instance stricter type

docs/mkdocs/docs/examples/sax_parse__error_recovery.cpp

@@ -0,0 +1,43 @@
+#include <iostream>
+#include <iomanip>
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+// a SAX parser that creates a JSON value like json::parse does, but that
+// recovers from parse errors instead of stopping at the first one
+class recovering_parser : public nlohmann::detail::json_sax_dom_parser<json>
+{
+  public:
+    explicit recovering_parser(json& result)
+        : nlohmann::detail::json_sax_dom_parser<json>(result, false)
+    {}
+
+    bool parse_error(std::size_t position,
+                     const std::string& /*last_token*/,
+                     const json::exception& ex)
+    {
+        std::cout << "byte " << position << ": " << ex.what() << '\n';
+
+        // repair the input and continue
+        return true;
+    }
+};
+
+int main()
+{
+    // JSON text with several mistakes that ends too early
+    const std::string text = R"({
+    "name": "Hello World",
+    "tags": ["a" "b",],
+    "valid": tru,
+    "size": 1.,
+    "nested": {"x": 1)";
+
+    json result;
+    recovering_parser sax(result);
+    const bool valid = json::sax_parse(text, &sax);
+
+    std::cout << "\nvalid JSON: " << std::boolalpha << valid << '\n'
+              << std::setw(4) << result << std::endl;
+}

docs/mkdocs/docs/examples/sax_parse__error_recovery.output

@@ -0,0 +1,19 @@
+byte 49: [json.exception.parse_error.101] parse error at line 3, column 20: syntax error while parsing array - unexpected string literal; expected ']'
+byte 51: [json.exception.parse_error.101] parse error at line 3, column 22: syntax error while parsing value - unexpected ']'; expected '[', '{', or a literal
+byte 70: [json.exception.parse_error.101] parse error at line 4, column 17: syntax error while parsing value - invalid literal; last read: '"valid": tru,'
+byte 86: [json.exception.parse_error.101] parse error at line 5, column 15: syntax error while parsing value - invalid number; expected digit after '.'; last read: '1.,'
+byte 109: [json.exception.parse_error.101] parse error at line 6, column 22: syntax error while parsing object - unexpected end of input; expected '}'
+
+valid JSON: false
+{
+    "name": "Hello World",
+    "nested": {
+        "x": 1
+    },
+    "size": 1,
+    "tags": [
+        "a",
+        "b"
+    ],
+    "valid": null
+}

docs/mkdocs/docs/features/binary_formats/cbor.md

@@ -131,6 +131,7 @@ The library maps CBOR types to JSON value types as follows:
 | Byte string            | binary          | 0x59       |
 | Byte string            | binary          | 0x5A       |
 | Byte string            | binary          | 0x5B       |
+| Byte string            | binary          | 0x5F       |
 | UTF-8 string           | string          | 0x60..0x77 |
 | UTF-8 string           | string          | 0x78       |
 | UTF-8 string           | string          | 0x79       |
@@ -156,6 +157,9 @@ The library maps CBOR types to JSON value types as follows:
 | Single-Precision Float | number_float    | 0xFA       |
 | Double-Precision Float | number_float    | 0xFB       |
 
+Indefinite-length UTF-8 strings (0x7F) and byte strings (0x5F) are supported. Each chunk must be a definite-length
+string of the same major type, as required by [RFC 8949, Section 3.2.3](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.2.3).
+
 !!! warning "Incomplete mapping"
 
     The mapping is **incomplete** in the sense that not all CBOR types can be converted to a JSON value. The following CBOR types are not supported and will yield parse errors:

docs/mkdocs/docs/features/parsing/error_recovery.md

@@ -0,0 +1,122 @@
+# Error Recovery
+
+By default, parsing stops at the first error. With the [SAX interface](sax_interface.md), you can instead ask the
+parser to *recover*: to repair the error and continue, so that you get as much as possible out of malformed input, for
+instance a file that was cut off, JSON edited by hand, or the output of a language model.
+
+## Recovering from errors
+
+The SAX parser's [`parse_error`](../../api/json_sax/parse_error.md) function is called for every error. Its return value
+decides what happens next:
+
+- `#!cpp false` stops parsing. This is what the SAX parsers of the library do, so [`parse`](../../api/basic_json/parse.md)
+  and [`accept`](../../api/basic_json/accept.md) never recover.
+- `#!cpp true` repairs the error and continues parsing.
+
+When recovering, the SAX parser still receives well-formed events: every `start_object` or `start_array` is followed by
+the matching `end_object` or `end_array`, and every `key` is followed by exactly one value. A SAX parser that creates a
+JSON value, such as the one in the example below, therefore gets a complete value. Parsing always ends, and
+[`sax_parse`](../../api/basic_json/sax_parse.md) returns `#!cpp false` for input that is not valid JSON, even if every
+error was repaired. Each token is reported at most once, and the SAX parser can stop at any error by returning
+`#!cpp false`.
+
+!!! example
+
+    The example below derives a SAX parser from the library's parser for `json` values (`json_sax_dom_parser`),
+    and recovers from all errors.
+
+    ```cpp
+    --8<-- "examples/sax_parse__error_recovery.cpp"
+    ```
+    
+    Output:
+    
+    ```
+    --8<-- "examples/sax_parse__error_recovery.output"
+    ```
+
+## How errors are repaired
+
+Each error is repaired with the smallest local edit: a missing separator is inserted, a stray token is removed, what can
+be read of a broken string or number is kept, and a value that cannot be read at all becomes `#!json null`.
+
+| Mistake                   | Repair                                                                         | Example                                  | Result                     |
+|---------------------------|--------------------------------------------------------------------------------|------------------------------------------|----------------------------|
+| missing `,` or `:`        | inserted                                                                       | `#!json [1 2]`, `#!json {"a" 1}`         | `[1,2]`, `{"a":1}`         |
+| missing value             | `#!json null` for an object key or between commas in an array                  | `#!json {"a":}`, `#!json [1,,2]`         | `{"a":null}`, `[1,null,2]` |
+| trailing comma            | removed                                                                        | `#!json [1,2,]`                          | `[1,2]`                    |
+| broken string             | invalid escapes and bytes are replaced (see below); a line break ends the string | `#!json ["a\qb"]`                        | `["aqb"]`                  |
+| broken number             | the longest valid beginning is kept                                            | `#!json [1., 2e+]`                       | `[1,2]`                    |
+| unreadable value          | `#!json null`                                                                  | `#!json [1, NaN, tru]`                   | `[1,null,null]`            |
+| number too large          | passed as infinity, together with its text                                     | `#!json [1e999]`                         | infinity (see below)       |
+| stray `:`                 | removed                                                                        | `#!json ["a":1]`                         | `["a",1]`                  |
+| member without a key      | skipped up to the next `,` or `}`                                              | `#!json {1:2, "b":3}`                    | `{"b":3}`                  |
+| wrong closing bracket     | closes the innermost array or object                                           | `#!json {"a":[1,2}, "b":3}`              | `{"a":[1,2],"b":3}`        |
+| input ends too early      | all open arrays and objects are closed                                         | `#!json {"a":[1,2`                       | `{"a":[1,2]}`              |
+| text before the value     | skipped                                                                        | `#!json )]}'{"a":1}`                     | `{"a":1}`                  |
+
+In a string, an unknown escape like `\q` stands for the escaped character (`q`), as in JavaScript. An invalid `\u`
+escape, a lone surrogate, and ill-formed UTF-8 are each replaced by U+FFFD (REPLACEMENT CHARACTER), and control
+characters are kept. A string without its closing quote ends at the next line break or at the end of the input.
+
+The input after the top-level value is not repaired: as without recovery, it is reported as an error, and parsing stops.
+
+## Binary formats
+
+The binary formats ([BJData](../binary_formats/bjdata.md), [BON8](../binary_formats/bon8.md),
+[BSON](../binary_formats/bson.md), [CBOR](../binary_formats/cbor.md), [MessagePack](../binary_formats/messagepack.md),
+and [UBJSON](../binary_formats/ubjson.md)) have no delimiters to find the next value by. So what can be repaired depends
+on whether the end of the item with the error is known, a distinction that
+[RFC 8949, Section 5.3](https://www.rfc-editor.org/rfc/rfc8949.html#section-5.3) makes for CBOR, too.
+
+If the item is complete, but cannot be passed on as it is, it is replaced, and parsing continues after it:
+
+| Mistake                                                             | Formats                 | Repair                                                                  |
+|---------------------------------------------------------------------|-------------------------|-------------------------------------------------------------------------|
+| tag, if `tag_handler` is `cbor_tag_handler_t::error` (the default)  | CBOR                    | ignored                                                                 |
+| simple value other than `false`, `true`, and `null`, like undefined | CBOR                    | `#!json null`                                                           |
+| number too large for a custom `number_float_t`, like `float`        | all                     | infinity                                                                |
+| character (`C`) that is not ASCII                                   | BJData, UBJSON          | U+FFFD                                                                  |
+| invalid high-precision number (`H`)                                 | BJData, UBJSON          | the longest valid beginning is kept, as for JSON text, or `#!json null` |
+| high-precision number too large                                     | BJData, UBJSON          | passed as infinity, together with its text                              |
+| object key that is not a string                                     | BON8, CBOR, MessagePack | the member is skipped                                                   |
+| element of a type the library does not read, like ObjectId or date  | BSON                    | `#!json null`                                                           |
+| string without its terminator                                       | BSON                    | kept                                                                    |
+| document whose size does not match its content                      | BSON                    | kept                                                                    |
+
+CBOR tags and simple values are repaired as [RFC 8949, Section 6.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-6.1)
+suggests for converting CBOR to JSON. By default, [`sax_parse`](../../api/basic_json/sax_parse.md) reports every tag as
+an error; when recovering, tags are then ignored like with
+[`cbor_tag_handler_t::ignore`](../../api/basic_json/cbor_tag_handler_t.md). Strings that are not valid UTF-8 are no
+error: like [`from_cbor`](../../api/basic_json/from_cbor.md) and the other functions by default, `sax_parse` passes
+them on as they are.
+
+After any other error, the end of the item is unknown: the input ended, a byte is not a valid type marker, or a size
+cannot be right. Parsing then stops, and the value read so far is completed: a key that waits for its value gets
+`#!json null`, and all open arrays and objects are closed. This keeps everything before the error of an input that was
+cut off. The exception is BSON, which stores the size of every document: an element whose end is unknown gets
+`#!json null`, the rest of its document is skipped, and parsing continues after the document.
+
+## Limitations
+
+- A repair is a guess. For example, `#!json {"a" "b": 1}` could be meant as `#!json {"a": "b"}` or as
+  `#!json {"a": null, "b": 1}`; it is repaired to the former. Treat recovered values as a best effort, and check the
+  reported errors.
+- A closing bracket always closes the innermost array or object. If a bracket is missing rather than wrong, the
+  repair differs from the intention: `#!json {"a": {"b": [1, 2}, "c": 3}` is repaired to
+  `#!json {"a": {"b": [1, 2], "c": 3}}`, although `#!json {"a": {"b": [1, 2]}, "c": 3}` may have been meant.
+- Keys without quotes, and strings in single quotes, are not supported; such members are skipped.
+- In the binary formats, a member that is skipped because its key is not a string is lost, and so are the elements of a
+  BSON document after one whose end is unknown.
+- A number that is too large for `number_float_t` is passed as positive or negative infinity. The SAX parser's
+  `number_float` also gets the number's text, but a JSON value cannot store it, and
+  [`dump`](../../api/basic_json/dump.md) serializes infinity as `#!json null`.
+- When parsing is not strict (see [`sax_parse`](../../api/basic_json/sax_parse.md)), a repair may read parts of the
+  input after the value, for instance of the next value in a stream of concatenated values.
+
+## See also
+
+- [SAX interface](sax_interface.md) - implement a custom SAX handler
+- [`parse_error`](../../api/json_sax/parse_error.md) - the SAX event for parse errors
+- [`sax_parse`](../../api/basic_json/sax_parse.md) - generate SAX events
+- [parsing and exceptions](parse_exceptions.md) - control error handling

docs/mkdocs/docs/features/parsing/index.md

@@ -75,7 +75,7 @@ You can influence a DOM parse without switching to the SAX interface by passing
 When the input is not valid JSON, the `parse` function throws an exception by default. If exceptions are undesired or
 unavailable, the parser can instead return a discarded value, or [`accept`](../../api/basic_json/accept.md) can be used
 to only check whether an input is valid JSON. See [parsing and exceptions](parse_exceptions.md) for the available
-options.
+options. To get as much as possible out of malformed input, a SAX parser can [recover from errors](error_recovery.md).
 
 ## See also
 
@@ -86,4 +86,5 @@ options.
 - [parser callbacks](parser_callbacks.md) - influence the parsing by a callback function
 - [SAX interface](sax_interface.md) - implement a custom SAX handler
 - [parsing and exceptions](parse_exceptions.md) - control error handling
+- [error recovery](error_recovery.md) - get as much as possible out of malformed input
 - [parsing untrusted input](untrusted_input.md) - what to consider when parsing input from untrusted sources

docs/mkdocs/docs/features/parsing/parse_exceptions.md

@@ -64,7 +64,8 @@ bool parse_error(std::size_t position,
                  const json::exception& ex);
 ```
 
-The return value indicates whether the parsing should continue, so the function should usually return `#!cpp false`.
+The return value decides whether to stop parsing (`#!cpp false`) or to repair the error and continue
+(`#!cpp true`); see [error recovery](error_recovery.md) for the latter.
 
 ??? example "Example: report parse errors without exceptions"
 

docs/mkdocs/docs/features/parsing/sax_interface.md

@@ -60,7 +60,8 @@ bool key(string_t& val);
 bool parse_error(std::size_t position, const std::string& last_token, const json::exception& ex);
 ```
 
-The return value of each function determines whether parsing should proceed.
+The return value of each function determines whether parsing should proceed. For `parse_error`, returning
+`#!cpp true` [recovers from the error](error_recovery.md).
 
 To implement your own SAX handler, proceed as follows:
 
@@ -68,7 +69,7 @@ To implement your own SAX handler, proceed as follows:
 2. Create an object of your SAX interface class, e.g. `my_sax`.
 3. Call `#!cpp bool json::sax_parse(input, &my_sax);` where the first parameter can be any input like a string or an input stream and the second parameter is a pointer to your SAX interface.
 
-Note the `sax_parse` function only returns a `#!cpp bool` indicating the result of the last executed SAX event. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`.
+Note the `sax_parse` function only returns a `#!cpp bool` indicating whether the input was parsed without errors and no SAX event returned `#!cpp false`. It does not return `json` value - it is up to you to decide what to do with the SAX events. Furthermore, no exceptions are thrown in case of a parse error - it is up to you what to do with the exception object passed to your `parse_error` implementation. Internally, the SAX interface is used for the DOM parser (class `json_sax_dom_parser`) as well as the acceptor (`json_sax_acceptor`), see file `json_sax.hpp`.
 
 ## See also
 

docs/mkdocs/mkdocs.yml

@@ -88,6 +88,7 @@ nav:
       - features/performance.md
       - Parsing:
           - features/parsing/index.md
+          - features/parsing/error_recovery.md
           - features/parsing/json_lines.md
           - features/parsing/parse_exceptions.md
           - features/parsing/parser_callbacks.md

include/nlohmann/detail/conversions/to_json.hpp

@@ -13,7 +13,6 @@
     #include <optional> // optional
 #endif
 
-#include <algorithm> // copy
 #include <iterator> // begin, end
 #include <memory> // allocator_traits
 #include <string> //  basic_string, char_traits
@@ -39,10 +38,14 @@ namespace detail
 //////////////////
 
 /*
- * Note all external_constructor<>::construct functions need to call
- * j.m_data.m_value.destroy(j.m_data.m_type) to avoid a memory leak in case j contains an
- * allocated value (e.g., a string). See bug issue
+ * Note all external_constructor<>::construct functions need to store the new
+ * value with j.replace_value(), which destroys the old one to avoid a memory
+ * leak in case j contains an allocated value (e.g., a string). See bug issue
  * https://github.com/nlohmann/json/issues/2865 for more information.
+ *
+ * A value that has to be allocated is created before the old one is destroyed:
+ * were it the other way around, an exception while creating the new value would
+ * leave j with the type of the new value, but the pointer to the destroyed old one.
  */
 
 template<value_t> struct external_constructor;
@@ -53,10 +56,7 @@ struct external_constructor<value_t::boolean>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::boolean;
-        j.m_data.m_value = b;
-        j.assert_invariant();
+        j.replace_value(value_t::boolean, b);
     }
 };
 
@@ -66,19 +66,15 @@ struct external_constructor<value_t::string>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::string;
-        j.m_data.m_value = s;
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(s);
+        j.replace_value(value_t::string, value);
     }
 
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::string;
-        j.m_data.m_value = std::move(s);
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(std::move(s));
+        j.replace_value(value_t::string, value);
     }
 
     template < typename BasicJsonType, typename CompatibleStringType,
@@ -86,10 +82,8 @@ struct external_constructor<value_t::string>
                              int > = 0 >
     static void construct(BasicJsonType& j, const CompatibleStringType& str)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::string;
-        j.m_data.m_value.string = j.template create<typename BasicJsonType::string_t>(str);
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(j.template create<typename BasicJsonType::string_t>(str));
+        j.replace_value(value_t::string, value);
     }
 };
 
@@ -99,19 +93,15 @@ struct external_constructor<value_t::binary>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::binary;
-        j.m_data.m_value = typename BasicJsonType::binary_t(b);
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(b);
+        j.replace_value(value_t::binary, value);
     }
 
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::binary;
-        j.m_data.m_value = typename BasicJsonType::binary_t(std::move(b));
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(std::move(b));
+        j.replace_value(value_t::binary, value);
     }
 };
 
@@ -121,10 +111,7 @@ struct external_constructor<value_t::number_float>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::number_float;
-        j.m_data.m_value = val;
-        j.assert_invariant();
+        j.replace_value(value_t::number_float, val);
     }
 };
 
@@ -134,10 +121,7 @@ struct external_constructor<value_t::number_unsigned>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::number_unsigned;
-        j.m_data.m_value = val;
-        j.assert_invariant();
+        j.replace_value(value_t::number_unsigned, val);
     }
 };
 
@@ -147,10 +131,7 @@ struct external_constructor<value_t::number_integer>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::number_integer;
-        j.m_data.m_value = val;
-        j.assert_invariant();
+        j.replace_value(value_t::number_integer, val);
     }
 };
 
@@ -160,21 +141,15 @@ struct external_constructor<value_t::array>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value = arr;
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(arr);
+        j.replace_value(value_t::array, value);
     }
 
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value = std::move(arr);
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(std::move(arr));
+        j.replace_value(value_t::array, value);
     }
 
     template < typename BasicJsonType, typename CompatibleArrayType,
@@ -188,39 +163,23 @@ struct external_constructor<value_t::array>
         using std::begin;
         using std::end;
 
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr)));
+        j.replace_value(value_t::array, value);
     }
 
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, const std::vector<bool>& arr)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value = value_t::array;
-        j.m_data.m_value.array->reserve(arr.size());
-        for (const bool x : arr)
-        {
-            j.m_data.m_value.array->push_back(x);
-            j.set_parent(j.m_data.m_value.array->back());
-        }
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(j.template create<typename BasicJsonType::array_t>(arr.begin(), arr.end()));
+        j.replace_value(value_t::array, value);
     }
 
     template<typename BasicJsonType, typename T,
              enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
     static void construct(BasicJsonType& j, const std::valarray<T>& arr)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value = value_t::array;
-        j.m_data.m_value.array->resize(arr.size());
-        std::copy(std::begin(arr), std::end(arr), j.m_data.m_value.array->begin());
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(j.template create<typename BasicJsonType::array_t>(std::begin(arr), std::end(arr)));
+        j.replace_value(value_t::array, value);
     }
 
 #if JSON_HAS_RANGE_VIEW_CONVERSION
@@ -228,18 +187,15 @@ struct external_constructor<value_t::array>
              enable_if_t<is_compatible_range_view<std::remove_cvref_t<CompatibleArrayType>>::value, int> = 0>
     static void construct(BasicJsonType& j, CompatibleArrayType && arr)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::array;
-        j.m_data.m_value = value_t::array;
+        // no range constructor: a view's begin() and end() may have different
+        // types, and the view may only be iterable once
+        typename BasicJsonType::array_t elements;
         for (auto&& x : std::forward<CompatibleArrayType>(arr))
         {
-            j.m_data.m_value.array->push_back(x);
+            elements.push_back(x);
         }
-        // set the parents only once all elements are in place: a push_back
-        // that reallocates moves the earlier elements, which does not keep
-        // their parent pointers
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(std::move(elements));
+        j.replace_value(value_t::array, value);
     }
 #endif
 };
@@ -250,21 +206,15 @@ struct external_constructor<value_t::object>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::object;
-        j.m_data.m_value = obj;
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(obj);
+        j.replace_value(value_t::object, value);
     }
 
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
     {
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::object;
-        j.m_data.m_value = std::move(obj);
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(std::move(obj));
+        j.replace_value(value_t::object, value);
     }
 
     template < typename BasicJsonType, typename CompatibleObjectType,
@@ -274,11 +224,8 @@ struct external_constructor<value_t::object>
         using std::begin;
         using std::end;
 
-        j.m_data.m_value.destroy(j.m_data.m_type);
-        j.m_data.m_type = value_t::object;
-        j.m_data.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
-        j.set_parents();
-        j.assert_invariant();
+        const typename BasicJsonType::json_value value(j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj)));
+        j.replace_value(value_t::object, value);
     }
 };
 

include/nlohmann/detail/input/json_sax.hpp

@@ -132,7 +132,9 @@ struct json_sax
     @param[in] position    the position in the input where the error occurs
     @param[in] last_token  the last read token
     @param[in] ex          an exception object describing the error
-    @return whether parsing should proceed (must return false)
+    @return whether to recover from the error: false stops parsing; true
+            repairs the error and continues, or, if that is not possible,
+            stops after completing the value read so far
     */
     virtual bool parse_error(std::size_t position,
                              const std::string& last_token,
@@ -269,9 +271,12 @@ a pointer to the respective array or object for each recursion depth.
 After successful parsing, the value that is passed by reference to the
 constructor contains the parsed value.
 
-@tparam BasicJsonType  the JSON type
+@tparam BasicJsonType     the JSON type
+@tparam InputAdapterType  the input adapter of the lexer that can be passed to
+                          the constructor to record diagnostic positions; it
+                          does not matter if no lexer is passed
 */
-template<typename BasicJsonType, typename InputAdapterType>
+template<typename BasicJsonType, typename InputAdapterType = string_input_adapter_type>
 class json_sax_dom_parser
 {
   public:
@@ -518,7 +523,7 @@ class json_sax_dom_parser
     lexer_t* m_lexer_ref = nullptr;
 };
 
-template<typename BasicJsonType, typename InputAdapterType>
+template<typename BasicJsonType, typename InputAdapterType = string_input_adapter_type>
 class json_sax_dom_callback_parser
 {
   public:

include/nlohmann/detail/input/lexer.hpp

@@ -10,7 +10,7 @@
 
 #include <array> // array
 #include <cstddef> // size_t
-#include <cstdint> // uint32_t
+#include <cstdint> // uint8_t, uint32_t
 #include <cstdio> // snprintf
 #include <initializer_list> // initializer_list
 #include <string> // char_traits, string
@@ -439,8 +439,16 @@ class lexer : public lexer_base<BasicJsonType>
                             if (0xD800 <= codepoint1 && codepoint1 <= 0xDBFF)
                             {
                                 // expect next \uxxxx entry
-                                if (JSON_HEDLEY_LIKELY(get() == '\\' && get() == 'u'))
+                                if (JSON_HEDLEY_LIKELY(get() == '\\'))
                                 {
+                                    if (JSON_HEDLEY_UNLIKELY(get() != 'u'))
+                                    {
+                                        // current is the character escaped by the backslash
+                                        error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+                                        string_error_resume = resume_kind::escaped_character;
+                                        return token_type::parse_error;
+                                    }
+
                                     const int codepoint2 = get_codepoint();
 
                                     if (JSON_HEDLEY_UNLIKELY(codepoint2 == -1))
@@ -465,7 +473,11 @@ class lexer : public lexer_base<BasicJsonType>
                                     }
                                     else
                                     {
+                                        // the second escape was read completely and is a
+                                        // code point of its own
                                         error_message = "invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF";
+                                        string_error_resume = resume_kind::after_escape;
+                                        string_error_codepoint = codepoint2;
                                         return token_type::parse_error;
                                     }
                                 }
@@ -479,7 +491,9 @@ class lexer : public lexer_base<BasicJsonType>
                             {
                                 if (JSON_HEDLEY_UNLIKELY(0xDC00 <= codepoint1 && codepoint1 <= 0xDFFF))
                                 {
+                                    // the escape was read completely
                                     error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    string_error_resume = resume_kind::after_escape;
                                     return token_type::parse_error;
                                 }
                             }
@@ -2133,6 +2147,574 @@ scan_number_done:
         }
     }
 
+  public:
+    /////////////////////
+    // error recovery
+    /////////////////////
+
+    /*!
+    @brief make the best of the token that scan() rejected
+
+    Called by the parser after scan() returned token_type::parse_error and the
+    SAX parser asked to recover from the error (see #3989). Keeps what can be
+    read of the token and skips the rest:
+
+    - A string keeps its characters. An unknown escape stands for the escaped
+      character itself (as in JavaScript), an invalid Unicode escape and ill-formed
+      UTF-8 become U+FFFD, and a control character is kept. A line break or the
+      end of the input ends a string that lacks its closing quote.
+    - A number keeps its longest valid prefix, e.g. `1` for `1.` or `1e+`.
+    - A block comment that is not closed runs to the end of the input.
+    - Anything else is skipped.
+
+    The rest of an invalid token is skipped up to the next delimiter
+    (whitespace, a structural character, or a quote). A delimiter that the
+    invalid token consumed is returned to the input, so that the next scan()
+    reads it.
+
+    @return token_type::value_string or a number token type if a string or a
+            number could be read, token_type::end_of_input for a block comment
+            that is not closed, token_type::uninitialized otherwise
+    */
+    token_type recover_token()
+    {
+        const resume_kind resume = string_error_resume;
+        const int codepoint = string_error_codepoint;
+        string_error_resume = resume_kind::character;
+        string_error_codepoint = -1;
+
+        if (error_message_starts_with("invalid string"))
+        {
+            return recover_string(resume, codepoint);
+        }
+
+        if (error_message_starts_with("invalid number"))
+        {
+            return recover_number();
+        }
+
+        if (error_message_starts_with("invalid comment; missing"))
+        {
+            // the comment runs to the end of the input
+            return token_type::end_of_input;
+        }
+
+        skip_to_delimiter();
+        return token_type::uninitialized;
+    }
+
+    /*!
+    @brief return the token that scan() read last to the input, so that the
+           next scan() reads it again
+
+    Called by the parser when recovering from an error. The token must be a
+    single character (',', ':', '[', ']', '{', or '}') or the end of the
+    input, and scan() must have read it last.
+    */
+    void unget_token()
+    {
+        JSON_ASSERT(!next_unget);
+        unget();
+    }
+
+    /*!
+    @brief let the token string for the next error begin at the current character
+
+    The token string of an error reaches back to the beginning of the last
+    string or number. After an error, the parser calls this function so that
+    the next error does not report (and, with many errors, copy) everything
+    read since then.
+    */
+    void restart_token_string()
+    {
+        restart_token_string_impl(std::integral_constant<bool, lazy_token_string> {});
+    }
+
+  private:
+    /// how recover_string() continues after the error scan_string() reported
+    enum class resume_kind : std::uint8_t
+    {
+        /// current is the next character of the string (or the end of input)
+        character,
+        /// current is the character escaped by the preceding backslash
+        escaped_character,
+        /// current is the last character of a complete escape
+        after_escape
+    };
+
+    /// whether error_message begins with @a prefix
+    bool error_message_starts_with(const char* prefix) const noexcept
+    {
+        const char* message = error_message;
+        while (*prefix != '\0')
+        {
+            if (*message++ != *prefix++)
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /// whether current ends an invalid token (see recover_token())
+    bool current_is_delimiter() const noexcept
+    {
+        switch (current)
+        {
+            case ' ':
+            case '\t':
+            case '\n':
+            case '\r':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case ',':
+            case ':':
+            case '\"':
+#if !JSON_STRICT_NUL_HANDLING
+            case '\0':
+#endif
+            case char_traits<char_type>::eof():
+                return true;
+
+            case '/':
+                return ignore_comments;
+
+            default:
+                return false;
+        }
+    }
+
+    /// skip the rest of an invalid token and return its delimiter to the input
+    void skip_to_delimiter()
+    {
+        while (!current_is_delimiter())
+        {
+            get();
+        }
+
+        if (current != char_traits<char_type>::eof())
+        {
+            unget();
+        }
+    }
+
+    /// append U+FFFD REPLACEMENT CHARACTER to token_buffer
+    void add_replacement_character()
+    {
+        add(0xEF);
+        add(0xBF);
+        add(0xBD);
+    }
+
+    /// append the UTF-8 encoding of @a codepoint (not a surrogate) to token_buffer
+    void add_codepoint(const int codepoint)
+    {
+        JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
+        const auto cp = static_cast<unsigned int>(codepoint);
+        if (cp < 0x80)
+        {
+            add(static_cast<char_int_type>(cp));
+        }
+        else if (cp <= 0x7FF)
+        {
+            add(static_cast<char_int_type>(0xC0u | (cp >> 6u)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+        else if (cp <= 0xFFFF)
+        {
+            add(static_cast<char_int_type>(0xE0u | (cp >> 12u)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 6u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+        else
+        {
+            add(static_cast<char_int_type>(0xF0u | (cp >> 18u)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 12u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | ((cp >> 6u) & 0x3Fu)));
+            add(static_cast<char_int_type>(0x80u | (cp & 0x3Fu)));
+        }
+    }
+
+    /// append a code point read from a Unicode escape; a surrogate becomes U+FFFD
+    void add_escaped_codepoint(const int codepoint)
+    {
+        if (0xD800 <= codepoint && codepoint <= 0xDFFF)
+        {
+            add_replacement_character();
+        }
+        else
+        {
+            add_codepoint(codepoint);
+        }
+    }
+
+    /*!
+    @brief remove an incomplete UTF-8 sequence from the end of token_buffer
+
+    next_byte_in_range() adds the bytes of a sequence as it checks them, so
+    when it rejects a byte, the beginning of the sequence is already in
+    token_buffer, which otherwise holds only complete sequences.
+
+    @return whether an incomplete sequence was removed
+    */
+    bool remove_incomplete_utf8_sequence()
+    {
+        std::size_t lead = token_buffer.size();
+        std::size_t continuation_bytes = 0;
+        while (lead > 0 && continuation_bytes < 3
+                && (static_cast<unsigned char>(token_buffer[lead - 1]) & 0xC0u) == 0x80u)
+        {
+            --lead;
+            ++continuation_bytes;
+        }
+        if (lead == 0)
+        {
+            return false;
+        }
+
+        const auto lead_byte = static_cast<unsigned char>(token_buffer[lead - 1]);
+        std::size_t expected = 0;
+        if (lead_byte >= 0xF0)
+        {
+            expected = 3;
+        }
+        else if (lead_byte >= 0xE0)
+        {
+            expected = 2;
+        }
+        else if (lead_byte >= 0xC0)
+        {
+            expected = 1;
+        }
+        if (continuation_bytes >= expected)
+        {
+            return false;
+        }
+
+        token_buffer.resize(lead - 1);
+        return true;
+    }
+
+    /*!
+    @brief read the UTF-8 sequence that begins with current, which is not ASCII
+    @return whether the next character must be read; false if current still
+            needs to be handled, because it does not belong to the sequence
+    */
+    bool recover_utf8_sequence()
+    {
+        // the number of continuation bytes and the range of the first one;
+        // see the ranges in scan_string()
+        std::size_t count = 0;
+        char_int_type low = 0x80;
+        char_int_type high = 0xBF;
+        if (current >= 0xC2 && current <= 0xDF)
+        {
+            count = 1;
+        }
+        else if (current >= 0xE0 && current <= 0xEF)
+        {
+            count = 2;
+            low = (current == 0xE0) ? 0xA0 : 0x80;
+            high = (current == 0xED) ? 0x9F : 0xBF;
+        }
+        else if (current >= 0xF0 && current <= 0xF4)
+        {
+            count = 3;
+            low = (current == 0xF0) ? 0x90 : 0x80;
+            high = (current == 0xF4) ? 0x8F : 0xBF;
+        }
+        else
+        {
+            // an ill-formed byte
+            add_replacement_character();
+            return true;
+        }
+
+        const std::size_t start = token_buffer.size();
+        add(current);
+        for (std::size_t i = 0; i < count; ++i)
+        {
+            get();
+            if (current < low || current > high)
+            {
+                token_buffer.resize(start);
+                add_replacement_character();
+                return false;
+            }
+            add(current);
+            low = 0x80;
+            high = 0xBF;
+        }
+        return true;
+    }
+
+    /*!
+    @brief read the low surrogate that must follow the high surrogate @a high
+    @return whether the next character must be read; false if current still
+            needs to be handled
+    */
+    bool recover_low_surrogate(int high)
+    {
+        while (true)
+        {
+            if (get() != '\\')
+            {
+                add_replacement_character();
+                return false;
+            }
+            if (get() != 'u')
+            {
+                add_replacement_character();
+                // not 'u', so this does not come back here
+                return recover_escape();
+            }
+
+            const int low = get_codepoint();
+            if (low == -1)
+            {
+                add_replacement_character();
+                return false;
+            }
+            if (0xDC00 <= low && low <= 0xDFFF)
+            {
+                add_codepoint(static_cast<int>((static_cast<unsigned int>(high) << 10u)
+                                               + static_cast<unsigned int>(low) - 0x35FDC00u));
+                return true;
+            }
+
+            // high has no low surrogate
+            add_replacement_character();
+            if (low < 0xD800 || low > 0xDBFF)
+            {
+                add_codepoint(low);
+                return true;
+            }
+            // another high surrogate
+            high = low;
+        }
+    }
+
+    /*!
+    @brief read the escape whose backslash was read; current is the escaped character
+    @return whether the next character must be read; false if current still
+            needs to be handled
+    */
+    bool recover_escape()
+    {
+        switch (current)
+        {
+            case '\"':
+                add('\"');
+                return true;
+            case '\\':
+                add('\\');
+                return true;
+            case '/':
+                add('/');
+                return true;
+            case 'b':
+                add('\b');
+                return true;
+            case 'f':
+                add('\f');
+                return true;
+            case 'n':
+                add('\n');
+                return true;
+            case 'r':
+                add('\r');
+                return true;
+            case 't':
+                add('\t');
+                return true;
+
+            case 'u':
+            {
+                const int codepoint = get_codepoint();
+                if (codepoint == -1)
+                {
+                    add_replacement_character();
+                    return false;
+                }
+                if (0xD800 <= codepoint && codepoint <= 0xDBFF)
+                {
+                    return recover_low_surrogate(codepoint);
+                }
+                add_escaped_codepoint(codepoint);
+                return true;
+            }
+
+            // an unknown escape stands for the escaped character
+            default:
+                return false;
+        }
+    }
+
+    /*!
+    @brief read the rest of a string after scan_string() rejected it
+
+    token_buffer holds what scan_string() read before the error. See
+    recover_token() for how errors are repaired.
+
+    @param[in] resume     how to continue, see resume_kind
+    @param[in] codepoint  for a high surrogate followed by an escape of another
+                          code point: that code point; -1 otherwise
+    */
+    token_type recover_string(const resume_kind resume, const int codepoint)
+    {
+        // whether the next character must be read before it can be handled
+        bool fetch = false;
+
+        if (error_message_starts_with("invalid string: surrogate")
+                || error_message_starts_with("invalid string: '\\u'")
+                || (error_message_starts_with("invalid string: ill-formed UTF-8")
+                    && remove_incomplete_utf8_sequence()))
+        {
+            add_replacement_character();
+        }
+
+        switch (resume)
+        {
+            case resume_kind::escaped_character:
+                fetch = recover_escape();
+                break;
+            case resume_kind::after_escape:
+                if (0xD800 <= codepoint && codepoint <= 0xDBFF)
+                {
+                    fetch = recover_low_surrogate(codepoint);
+                }
+                else
+                {
+                    if (codepoint != -1)
+                    {
+                        add_escaped_codepoint(codepoint);
+                    }
+                    fetch = true;
+                }
+                break;
+            case resume_kind::character:
+            default:
+                break;
+        }
+
+        while (true)
+        {
+            if (fetch)
+            {
+                get();
+            }
+            fetch = true;
+
+            switch (current)
+            {
+                case '\"':
+                // a line break or the end of the input ends a string that
+                // lacks its closing quote
+                case '\n':
+                case '\r':
+                case char_traits<char_type>::eof():
+                    return token_type::value_string;
+
+#if !JSON_STRICT_NUL_HANDLING
+                case '\0':
+                    // the end of the input, see scan()
+                    unget();
+                    return token_type::value_string;
+#endif
+
+                case '\\':
+                    get();
+                    fetch = recover_escape();
+                    break;
+
+                default:
+                    if (current < 0x80)
+                    {
+                        // including control characters
+                        add(current);
+                    }
+                    else
+                    {
+                        fetch = recover_utf8_sequence();
+                    }
+                    break;
+            }
+        }
+    }
+
+    /*!
+    @brief keep the longest valid prefix of a number that scan_number() rejected
+
+    token_buffer holds the characters scan_number() accepted before the error,
+    so the prefix ends at its last digit.
+    */
+    token_type recover_number()
+    {
+        // only size(), operator[], and resize() are used, which every string
+        // type the library supports provides
+        std::size_t length = token_buffer.size();
+        while (length != 0 && (token_buffer[length - 1] < '0' || token_buffer[length - 1] > '9'))
+        {
+            --length;
+        }
+        token_buffer.resize(length);
+
+        if (length == 0)
+        {
+            skip_to_delimiter();
+            return token_type::uninitialized;
+        }
+
+        if (decimal_point_position >= length)
+        {
+            decimal_point_position = std::string::npos;
+        }
+
+        std::size_t exponent = std::string::npos;
+        for (std::size_t i = 0; i < length; ++i)
+        {
+            if (token_buffer[i] == 'e' || token_buffer[i] == 'E')
+            {
+                exponent = i;
+                break;
+            }
+        }
+        const std::size_t mantissa_end = (exponent == std::string::npos) ? length : exponent;
+        token_type number_type = token_type::value_unsigned;
+        if (decimal_point_position != std::string::npos || exponent != std::string::npos)
+        {
+            number_type = token_type::value_float;
+        }
+        else if (token_buffer[0] == '-')
+        {
+            number_type = token_type::value_integer;
+        }
+
+        const token_type result = convert_number(number_type, mantissa_end);
+        skip_to_delimiter();
+        return result;
+    }
+
+    /// seekable adapter: the token string begins at current, which was consumed
+    void restart_token_string_impl(std::true_type /*lazy*/) noexcept
+    {
+        const std::size_t consumed = ia.get_consumed_count();
+        token_string_start = (consumed > 0 && current != char_traits<char_type>::eof()) ? consumed - 1 : consumed;
+    }
+
+    /// streaming adapter: the token string begins at current; a character
+    /// that was put back is copied again when it is read again
+    void restart_token_string_impl(std::false_type /*lazy*/)
+    {
+        token_string.clear();
+        if (!next_unget && current != char_traits<char_type>::eof())
+        {
+            token_string.push_back(char_traits<char_type>::to_char_type(current));
+        }
+    }
+
     /// input adapter
     InputAdapterType ia;
 
@@ -2172,6 +2754,13 @@ scan_number_done:
     /// a description of occurred lexer errors
     const char* error_message = "";
 
+    /// how recover_token() continues a string that scan_string() rejected;
+    /// set only on the error paths that need more than error_message
+    resume_kind string_error_resume = resume_kind::character;
+    /// the code point of the second escape when a high surrogate is followed
+    /// by an escape that is not a low surrogate; -1 otherwise
+    int string_error_codepoint = -1;
+
     // number values
     number_integer_t value_integer = 0;
     number_unsigned_t value_unsigned = 0;

include/nlohmann/detail/input/parser.hpp

@@ -139,26 +139,59 @@ class parser
     bool accept(const bool strict = true)
     {
         json_sax_acceptor<BasicJsonType> sax_acceptor;
-        return sax_parse(&sax_acceptor, strict);
+        return sax_parse_impl<false>(&sax_acceptor, strict);
     }
 
+    /*!
+    @brief public SAX interface
+
+    If the SAX parser's parse_error() returns true, the parser recovers from
+    the error: it repairs the input and continues (see #3989).
+
+    @param[in] sax     the SAX parser
+    @param[in] strict  whether to expect the last token to be EOF
+    @return whether the input was parsed without errors and no SAX event
+            returned false
+    */
     template<typename SAX>
     JSON_HEDLEY_NON_NULL(2)
     bool sax_parse(SAX* sax, const bool strict = true)
+    {
+        return sax_parse_impl<true>(sax, strict);
+    }
+
+  private:
+    /// what sax_parse_internal() does after an object key was expected
+    enum class next_step : std::uint8_t
+    {
+        /// stop parsing
+        stop,
+        /// parse a value that begins with last_token
+        parse_value,
+        /// evaluate the state of the innermost container, which reads
+        /// last_token again
+        evaluate_state
+    };
+
+    template<bool AllowRecovery, typename SAX>
+    JSON_HEDLEY_NON_NULL(2)
+    bool sax_parse_impl(SAX* sax, const bool strict)
     {
         (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {};
-        const bool result = sax_parse_internal(sax);
+        const bool result = sax_parse_internal<AllowRecovery>(sax);
 
         if (result)
         {
             if (strict)
             {
-                // strict mode: next byte must be EOF
-                if (get_token() != token_type::end_of_input)
+                // strict mode: next byte must be EOF; after recovering from an
+                // error, the end of the input may already have been read
+                if (last_token != token_type::end_of_input && get_token() != token_type::end_of_input)
                 {
-                    return sax->parse_error(m_lexer.get_position(),
-                                            m_lexer.get_token_string(),
-                                            parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr));
+                    // the value is complete, so there is nothing to recover
+                    static_cast<void>(report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input, "value"), nullptr),
+                                                   std::integral_constant<bool, AllowRecovery> {}));
+                    return false;
                 }
             }
             else
@@ -169,10 +202,9 @@ class parser
             }
         }
 
-        return result;
+        return result && !error_reported;
     }
 
-  private:
     /*!
     @brief run a DOM SAX parser to completion and position the lexer
 
@@ -190,7 +222,7 @@ class parser
     template<typename DomSax>
     bool parse_dom(DomSax& sdp, const bool strict)
     {
-        sax_parse_internal(&sdp);
+        sax_parse_internal<false>(&sdp);
 
         if (strict)
         {
@@ -213,10 +245,20 @@ class parser
         return !sdp.is_errored();
     }
 
-    template<typename SAX>
+    /*!
+    @brief parse a JSON value and pass it to a SAX parser
+
+    @tparam AllowRecovery  whether to recover from an error if the SAX parser's
+                           parse_error() returns true; false for the SAX parsers
+                           of parse() and accept(), which never do, so that no
+                           code for recovering is generated for them
+    */
+    template<bool AllowRecovery, typename SAX>
     JSON_HEDLEY_NON_NULL(2)
     bool sax_parse_internal(SAX* sax)
     {
+        const std::integral_constant<bool, AllowRecovery> allow_recovery{};
+
         // stack to remember the hierarchy of structured values we are parsing
         // true = array; false = object
         std::vector<bool> states;
@@ -247,12 +289,18 @@ class parser
                             break;
                         }
 
-                        // parse key
+                        // remember we are now inside an object
+                        states.push_back(false);
+
+                        // parse key (the steps of parse_key(), which are
+                        // repeated here and below for speed)
                         if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+                            if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation))
+                            {
+                                return false;
+                            }
+                            continue;
                         }
                         if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
                         {
@@ -262,14 +310,13 @@ class parser
                         // parse separator (:)
                         if (JSON_HEDLEY_UNLIKELY(!get_token_expecting(token_type::name_separator)))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+                            if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation))
+                            {
+                                return false;
+                            }
+                            continue;
                         }
 
-                        // remember we are now inside an object
-                        states.push_back(false);
-
                         // parse values
                         get_token();
                         continue;
@@ -305,9 +352,11 @@ class parser
 
                         if (JSON_HEDLEY_UNLIKELY(!std::isfinite(res)))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr));
+                            if (!overflow_error(sax, res, allow_recovery))
+                            {
+                                return false;
+                            }
+                            break;
                         }
 
                         if (JSON_HEDLEY_UNLIKELY(!sax->number_float(res, m_lexer.get_string())))
@@ -375,23 +424,63 @@ class parser
                     case token_type::parse_error:
                     {
                         // using "uninitialized" to avoid an "expected" message
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr));
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::uninitialized, "value"), nullptr), allow_recovery))
+                        {
+                            return false;
+                        }
+
+                        // recover: keep what can be read of the token
+                        recover_token();
+                        if (last_token != token_type::uninitialized)
+                        {
+                            // a string or a number
+                            continue;
+                        }
+                        if (states.empty())
+                        {
+                            // look for the value after the garbage
+                            if (!skip_to_value())
+                            {
+                                return false;
+                            }
+                            continue;
+                        }
+                        // nothing could be read
+                        if (JSON_HEDLEY_UNLIKELY(!sax->null()))
+                        {
+                            return false;
+                        }
+                        break;
                     }
                     case token_type::end_of_input:
                     {
                         if (JSON_HEDLEY_UNLIKELY(m_lexer.get_position().chars_read_total == 1))
                         {
-                            return sax->parse_error(m_lexer.get_position(),
-                                                    m_lexer.get_token_string(),
-                                                    parse_error::create(101, m_lexer.get_position(),
-                                                            "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr));
+                            // there is nothing to recover
+                            static_cast<void>(report_error(sax, parse_error::create(101, m_lexer.get_position(),
+                                                           "attempting to parse an empty input; check that your input string or stream contains the expected JSON", nullptr), allow_recovery));
+                            return false;
                         }
 
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery))
+                        {
+                            return false;
+                        }
+
+                        // recover: the input ends where a value is missing
+                        if (states.empty())
+                        {
+                            // there is no value
+                            return false;
+                        }
+                        if (!recover_missing_value(sax, states))
+                        {
+                            return false;
+                        }
+                        // the state evaluation reads the token again
+                        m_lexer.unget_token();
+                        skip_to_state_evaluation = true;
+                        continue;
                     }
                     case token_type::uninitialized:
                     case token_type::end_array:
@@ -401,9 +490,35 @@ class parser
                     case token_type::literal_or_value:
                     default: // the last token was unexpected
                     {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr));
+                        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::literal_or_value, "value"), nullptr), allow_recovery))
+                        {
+                            return false;
+                        }
+
+                        // recover
+                        if (states.empty())
+                        {
+                            // look for the value after the garbage
+                            if (!skip_to_value())
+                            {
+                                return false;
+                            }
+                            continue;
+                        }
+                        if (last_token == token_type::name_separator)
+                        {
+                            // a stray ':'; the value may follow
+                            get_token();
+                            continue;
+                        }
+                        if (!recover_missing_value(sax, states))
+                        {
+                            return false;
+                        }
+                        // the state evaluation reads the token again
+                        m_lexer.unget_token();
+                        skip_to_state_evaluation = true;
+                        continue;
                     }
                 }
             }
@@ -454,9 +569,30 @@ class parser
                     continue;
                 }
 
-                return sax->parse_error(m_lexer.get_position(),
-                                        m_lexer.get_token_string(),
-                                        parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr));
+                if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_array, "array"), nullptr), allow_recovery))
+                {
+                    return false;
+                }
+
+                // recover
+                if (last_token == token_type::end_of_input)
+                {
+                    // the input ends inside the array
+                    return close_containers(sax, states);
+                }
+                if (last_token == token_type::end_object)
+                {
+                    // a wrong closing bracket closes the innermost container
+                    if (JSON_HEDLEY_UNLIKELY(!sax->end_array()))
+                    {
+                        return false;
+                    }
+                    states.pop_back();
+                    skip_to_state_evaluation = true;
+                }
+                // otherwise, a missing ',' (or a stray ':', which value
+                // parsing drops): the next value begins here
+                continue;
             }
 
             // states.back() is false -> object
@@ -473,11 +609,12 @@ class parser
                     // parse key
                     if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
                     {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr));
+                        if (!continue_after(key_error(sax, allow_recovery, false), skip_to_state_evaluation))
+                        {
+                            return false;
+                        }
+                        continue;
                     }
-
                     if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
                     {
                         return false;
@@ -486,9 +623,11 @@ class parser
                     // parse separator (:)
                     if (JSON_HEDLEY_UNLIKELY(!get_token_expecting(token_type::name_separator)))
                     {
-                        return sax->parse_error(m_lexer.get_position(),
-                                                m_lexer.get_token_string(),
-                                                parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr));
+                        if (!continue_after(key_error(sax, allow_recovery, true), skip_to_state_evaluation))
+                        {
+                            return false;
+                        }
+                        continue;
                     }
 
                     // parse values
@@ -516,12 +655,479 @@ class parser
                 continue;
             }
 
-            return sax->parse_error(m_lexer.get_position(),
-                                    m_lexer.get_token_string(),
-                                    parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr));
+            if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_object, "object"), nullptr), allow_recovery))
+            {
+                return false;
+            }
+
+            // recover
+            if (last_token == token_type::end_of_input)
+            {
+                // the input ends inside the object
+                return close_containers(sax, states);
+            }
+            if (last_token == token_type::end_array)
+            {
+                // a wrong closing bracket closes the innermost container
+                if (JSON_HEDLEY_UNLIKELY(!sax->end_object()))
+                {
+                    return false;
+                }
+                states.pop_back();
+                skip_to_state_evaluation = true;
+                continue;
+            }
+            if (!continue_after(recover_member(sax, allow_recovery), skip_to_state_evaluation))
+            {
+                return false;
+            }
         }
     }
 
+    /*!
+    @brief continue sax_parse_internal() after a recovery
+    @return whether to continue parsing
+    */
+    bool continue_after(const next_step step, bool& skip_to_state_evaluation)
+    {
+        if (step == next_step::evaluate_state)
+        {
+            // the state evaluation reads the token again
+            m_lexer.unget_token();
+            skip_to_state_evaluation = true;
+        }
+        return step != next_step::stop;
+    }
+
+    /// the parser for parse() and accept() never recovers: stop parsing
+    static std::false_type continue_after(std::false_type /*step*/, bool& /*skip_to_state_evaluation*/) noexcept
+    {
+        return {};
+    }
+
+    /*!
+    @brief parse an object key and the name separator (:) after it
+
+    last_token is the token where the key is expected. sax_parse_internal()
+    repeats these steps rather than calling this function, which is used
+    when recovering from an error.
+
+    @return next_step::parse_value if the value follows, with last_token its
+            first token; next_step::evaluate_state if the object's state is
+            to be evaluated after recovering from an error; next_step::stop
+            to stop parsing
+    */
+    template<typename SAX>
+    next_step parse_key(SAX* sax)
+    {
+        const std::true_type allow_recovery{};
+
+        if (JSON_HEDLEY_UNLIKELY(last_token != token_type::value_string))
+        {
+            return key_error(sax, allow_recovery, false);
+        }
+
+        if (JSON_HEDLEY_UNLIKELY(!sax->key(m_lexer.get_string())))
+        {
+            return next_step::stop;
+        }
+
+        // parse separator (:)
+        if (JSON_HEDLEY_UNLIKELY(get_token() != token_type::name_separator))
+        {
+            return key_error(sax, allow_recovery, true);
+        }
+
+        // the value begins with the next token
+        get_token();
+        return next_step::parse_value;
+    }
+
+    /*!
+    @brief report a number that is too large for number_float_t, and recover
+           from the error by passing the value on; the SAX parser gets the
+           number's text as well
+
+    This is a separate function, as reading other numbers is measurably
+    slower if the error is handled where they are read.
+
+    @param[in] sax    the SAX parser
+    @param[in] value  the value that is not finite
+    @return whether to continue parsing
+    */
+    template<typename SAX, typename AllowRecovery>
+    bool overflow_error(SAX* sax, const number_float_t value, AllowRecovery allow_recovery)
+    {
+        if (!report_error(sax, out_of_range::create(406, concat("number overflow parsing '", m_lexer.get_token_string(), '\''), nullptr), allow_recovery))
+        {
+            return false;
+        }
+        return sax->number_float(value, m_lexer.get_string());
+    }
+
+    /*!
+    @brief report a missing key, or a missing name separator (:) after the
+           key; the parser for parse() and accept() never recovers
+
+    @param[in] key_read  whether the key was read, so that the name separator
+                         is missing
+    @return std::false_type, see report_error()
+    */
+    template<typename SAX>
+    std::false_type key_error(SAX* sax, std::false_type allow_recovery, const bool key_read)
+    {
+        return report_error(sax, parse_error::create(101, m_lexer.get_position(), key_read
+                            ? exception_message(token_type::name_separator, "object separator")
+                            : exception_message(token_type::value_string, "object key"), nullptr), allow_recovery);
+    }
+
+    /*!
+    @brief report a missing key, or a missing name separator (:) after the
+           key, and recover from it
+
+    @param[in] key_read  whether the key was read, so that the name separator
+                         is missing
+    */
+    template<typename SAX>
+    next_step key_error(SAX* sax, std::true_type allow_recovery, const bool key_read)
+    {
+        if (!key_read)
+        {
+            if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::value_string, "object key"), nullptr), allow_recovery))
+            {
+                return next_step::stop;
+            }
+            return recover_key(sax);
+        }
+
+        if (!report_error(sax, parse_error::create(101, m_lexer.get_position(), exception_message(token_type::name_separator, "object separator"), nullptr), allow_recovery))
+        {
+            return next_step::stop;
+        }
+        return recover_name_separator(sax);
+    }
+
+    /////////////////////
+    // error recovery
+    /////////////////////
+
+    /*
+    The functions below repair an error after the SAX parser's parse_error()
+    returned true (see #3989). Each mistake is repaired by the smallest local
+    edit: a missing ',' or ':' is inserted, a stray token is removed, what can
+    be read of an invalid string or number is kept (see
+    lexer::recover_token()), a missing value becomes null, a wrong closing
+    bracket closes the innermost container, and the end of the input closes
+    all of them. The events stay balanced, and every key() is followed by
+    exactly one value.
+
+    A repair hands a token to the state evaluation, by returning it to the
+    lexer (lexer::unget_token()) so that the state evaluation reads it again,
+    only if it is ',', ']', '}', or the end of the input. The state evaluation
+    hands a token to value or key parsing only if it is none of them, so a
+    token is never handed back and forth. Every other step reads a token or
+    closes a container, so parsing always ends.
+    */
+
+    /*!
+    @brief report an error to the SAX parser; the parser for parse() and
+           accept() never recovers
+
+    @return std::false_type rather than false: its value is known where the
+            function is called even if the call is not inlined, so the code
+            for recovering is not generated
+    */
+    template<typename SAX, typename Exception>
+    std::false_type report_error(SAX* sax, const Exception& ex, std::false_type /*allow_recovery*/)
+    {
+        error_reported = true;
+        static_cast<void>(sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex));
+        return {};
+    }
+
+    /*!
+    @brief report an error to the SAX parser
+    @return whether to recover from the error
+    */
+    template<typename SAX, typename Exception>
+    bool report_error(SAX* sax, const Exception& ex, std::true_type /*allow_recovery*/)
+    {
+        const std::size_t position = m_lexer.get_position().chars_read_total;
+        if (error_reported && position == last_error_position && last_token == last_error_token)
+        {
+            // a repair handed on the token of the error it repaired; the
+            // token was reported already, and the SAX parser asked to recover
+            return true;
+        }
+
+        error_reported = true;
+        last_error_position = position;
+        last_error_token = last_token;
+
+        if (!sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), ex))
+        {
+            return false;
+        }
+
+        // the token string of the next error begins here
+        m_lexer.restart_token_string();
+        return true;
+    }
+
+    /*!
+    @brief keep what can be read of the token that the lexer rejected
+
+    The error was reported for the rejected token, so it is not reported again
+    for the token it is repaired to (see lexer::recover_token()).
+    */
+    token_type recover_token()
+    {
+        last_token = m_lexer.recover_token();
+        last_error_position = m_lexer.get_position().chars_read_total;
+        last_error_token = last_token;
+        return last_token;
+    }
+
+    /// pass the end events of all open containers
+    template<typename SAX>
+    bool close_containers(SAX* sax, std::vector<bool>& states)
+    {
+        while (!states.empty())
+        {
+            const bool is_array = states.back();
+            states.pop_back();
+            if (JSON_HEDLEY_UNLIKELY(is_array ? !sax->end_array() : !sax->end_object()))
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /*!
+    @brief read tokens until one begins a value, skipping everything before
+           the top-level value
+    @return whether a value begins with last_token
+    */
+    bool skip_to_value()
+    {
+        while (true)
+        {
+            switch (get_token())
+            {
+                case token_type::begin_array:
+                case token_type::begin_object:
+                case token_type::literal_false:
+                case token_type::literal_null:
+                case token_type::literal_true:
+                case token_type::value_float:
+                case token_type::value_integer:
+                case token_type::value_string:
+                case token_type::value_unsigned:
+                    return true;
+
+                case token_type::end_of_input:
+                    return false;
+
+                case token_type::parse_error:
+                    recover_token();
+                    if (last_token != token_type::uninitialized)
+                    {
+                        return true;
+                    }
+                    break;
+
+                case token_type::uninitialized:
+                case token_type::end_array:
+                case token_type::end_object:
+                case token_type::name_separator:
+                case token_type::value_separator:
+                case token_type::literal_or_value:
+                default:
+                    break;
+            }
+        }
+    }
+
+    /*!
+    @brief skip the rest of an object member that cannot be read
+
+    Reads tokens, beginning with last_token, until a ',', '}', or ']' that is
+    not inside a container that begins in the skipped tokens, or the end of
+    the input.
+    */
+    void skip_member()
+    {
+        std::size_t depth = 0;
+        while (true)
+        {
+            switch (last_token)
+            {
+                case token_type::begin_array:
+                case token_type::begin_object:
+                    ++depth;
+                    break;
+
+                case token_type::end_array:
+                case token_type::end_object:
+                    if (depth == 0)
+                    {
+                        return;
+                    }
+                    --depth;
+                    break;
+
+                case token_type::value_separator:
+                    if (depth == 0)
+                    {
+                        return;
+                    }
+                    break;
+
+                case token_type::end_of_input:
+                    return;
+
+                case token_type::parse_error:
+                    recover_token();
+                    break;
+
+                case token_type::uninitialized:
+                case token_type::literal_true:
+                case token_type::literal_false:
+                case token_type::literal_null:
+                case token_type::value_string:
+                case token_type::value_unsigned:
+                case token_type::value_integer:
+                case token_type::value_float:
+                case token_type::name_separator:
+                case token_type::literal_or_value:
+                default:
+                    break;
+            }
+            get_token();
+        }
+    }
+
+    /*!
+    @brief pass a value where it is missing
+
+    last_token is ',', ']', '}', or the end of the input, where a value was
+    expected. In an object, the key gets null; in an array, a ',' where a
+    value is missing stands for null (as in JavaScript), while an array that
+    ends there just ends.
+    */
+    template<typename SAX>
+    bool recover_missing_value(SAX* sax, const std::vector<bool>& states)
+    {
+        JSON_ASSERT(!states.empty());
+        if (!states.back() || last_token == token_type::value_separator)
+        {
+            return sax->null();
+        }
+        return true;
+    }
+
+    /// recover from a missing key; last_token is where it was expected
+    template<typename SAX>
+    next_step recover_key(SAX* sax)
+    {
+        switch (last_token)
+        {
+            case token_type::value_separator:
+            case token_type::end_object:
+            case token_type::end_array:
+            case token_type::end_of_input:
+                // no member: the object's state handles the token
+                return next_step::evaluate_state;
+
+            case token_type::parse_error:
+                recover_token();
+                if (last_token == token_type::value_string)
+                {
+                    // a key that could be repaired
+                    return parse_key(sax);
+                }
+                skip_member();
+                return next_step::evaluate_state;
+
+            case token_type::uninitialized:
+            case token_type::literal_true:
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+            case token_type::value_integer:
+            case token_type::value_float:
+            case token_type::begin_array:
+            case token_type::begin_object:
+            case token_type::name_separator:
+            case token_type::literal_or_value:
+            default:
+                // a member without a key
+                skip_member();
+                return next_step::evaluate_state;
+        }
+    }
+
+    /// recover from a missing name separator (:) after the key; last_token
+    /// is where it was expected
+    template<typename SAX>
+    next_step recover_name_separator(SAX* sax)
+    {
+        switch (last_token)
+        {
+            case token_type::value_separator:
+            case token_type::end_object:
+            case token_type::end_array:
+            case token_type::end_of_input:
+                // the value is missing as well
+                return sax->null() ? next_step::evaluate_state : next_step::stop;
+
+            case token_type::uninitialized:
+            case token_type::literal_true:
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+            case token_type::value_integer:
+            case token_type::value_float:
+            case token_type::begin_array:
+            case token_type::begin_object:
+            case token_type::name_separator:
+            case token_type::parse_error:
+            case token_type::literal_or_value:
+            default:
+                // a missing ':'; the value begins here
+                return next_step::parse_value;
+        }
+    }
+
+    /// recover from a token after an object member that is neither ',' nor
+    /// '}' (nor ']' or the end of the input, which the caller handles)
+    template<typename SAX>
+    next_step recover_member(SAX* sax, std::true_type /*allow_recovery*/)
+    {
+        if (last_token == token_type::parse_error)
+        {
+            recover_token();
+        }
+        if (last_token == token_type::value_string)
+        {
+            // a missing ','; the next key begins here
+            return parse_key(sax);
+        }
+        skip_member();
+        return next_step::evaluate_state;
+    }
+
+    /// the parser for parse() and accept() never recovers (and does not come
+    /// here, as report_error() returned false)
+    template<typename SAX>
+    std::false_type recover_member(SAX* /*sax*/, std::false_type /*allow_recovery*/) const noexcept
+    {
+        return {};
+    }
+
     /// get next token from lexer
     token_type get_token()
     {
@@ -575,6 +1181,12 @@ class parser
     const bool allow_exceptions = true;
     /// whether trailing commas in objects and arrays should be ignored (true) or signaled as errors (false)
     const bool ignore_trailing_commas = false;
+    /// whether an error was reported to the SAX parser
+    bool error_reported = false;
+    /// the position of the last reported error
+    std::size_t last_error_position = 0;
+    /// the token of the last reported error
+    token_type last_error_token = token_type::uninitialized;
 };
 
 }  // namespace detail

include/nlohmann/detail/json_ref.hpp

@@ -34,6 +34,7 @@ class json_ref
 
     json_ref(std::initializer_list<json_ref> init)
         : owned_value(init)
+        , braced_list(true)
     {}
 
     template <
@@ -69,9 +70,17 @@ class json_ref
         return &** this;
     }
 
+    /// whether the value was written as a braced list, such as {"key", 1},
+    /// rather than given as a value
+    bool is_braced_list() const noexcept
+    {
+        return braced_list;
+    }
+
   private:
     mutable value_type owned_value = nullptr;
     value_type const* value_ref = nullptr;
+    bool braced_list = false;
 };
 
 }  // namespace detail

include/nlohmann/detail/string_utils.hpp

@@ -13,6 +13,7 @@
 #include <cstddef> // size_t
 #include <cstdint> // uint8_t, uint32_t
 #include <string> // string, to_string
+#include <utility> // move
 
 #include <nlohmann/detail/abi_macros.hpp>
 #include <nlohmann/detail/macro_scope.hpp>

include/nlohmann/json.hpp

@@ -149,7 +149,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     friend class ::nlohmann::detail::iter_impl;
     template<typename BasicJsonType, typename CharType, typename OutputSinkType>
     friend class ::nlohmann::detail::binary_writer;
-    template<typename BasicJsonType, typename InputType, typename SAX>
+    template<typename BasicJsonType, typename InputType, typename SAX, bool AllowRecovery>
     friend class ::nlohmann::detail::binary_reader;
     template<typename BasicJsonType, typename InputAdapterType>
     friend class ::nlohmann::detail::json_sax_dom_parser;
@@ -676,6 +676,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         /// constructor for rvalue binary arrays (internal type)
         json_value(binary_t&& value) : binary(create<binary_t>(std::move(value))) {}
 
+        /// constructors taking ownership of an already created value
+        explicit json_value(string_t* value) noexcept : string(value) {}
+        explicit json_value(object_t* value) noexcept : object(value) {}
+        explicit json_value(array_t* value) noexcept : array(value) {}
+
 private:
         // raw, allocation-free transfer of m_data from src to dst: no
         // set_parents()/assert_invariant() (the former is O(#children) per
@@ -1006,6 +1011,18 @@ public:
 #endif
     }
 
+    /// @brief replace the stored value with an already created one
+    /// The new value must be created before calling this function: if its
+    /// creation throws, the current value is left untouched.
+    void replace_value(value_t t, const json_value& v) noexcept
+    {
+        m_data.m_value.destroy(m_data.m_type);
+        m_data.m_value = v;
+        m_data.m_type = t;
+        set_parents();
+        assert_invariant();
+    }
+
     iterator set_parents(iterator it, std::ptrdiff_t count_set_parents)
     {
 #if JSON_DIAGNOSTICS
@@ -2235,6 +2252,18 @@ public:
                bool type_deduction = true,
                value_t manual_type = value_t::array)
     {
+#if JSON_BRACE_INIT_COPY_SEMANTICS
+        // a single element that is a value rather than a braced list is
+        // copied or moved as is, whatever its content looks like
+        if (type_deduction && init.size() == 1 && !init.begin()->is_braced_list())
+        {
+            *this = init.begin()->moved_or_copied();
+            set_parents();
+            assert_invariant();
+            return;
+        }
+#endif
+
         // check if each element is an array with two elements whose first
         // element is a string
         bool is_an_object = std::all_of(init.begin(), init.end(),
@@ -2265,8 +2294,8 @@ public:
         if (is_an_object)
         {
             // the initializer list is a list of pairs -> create an object
-            m_data.m_type = value_t::object;
             m_data.m_value = value_t::object;
+            m_data.m_type = value_t::object;
 
             for (auto& element_ref : init)
             {
@@ -2288,8 +2317,8 @@ public:
             }
 #endif
             // the initializer list describes an array -> create an array
-            m_data.m_type = value_t::array;
             m_data.m_value.array = create<array_t>(init.begin(), init.end());
+            m_data.m_type = value_t::array;
         }
 
         set_parents();
@@ -2302,8 +2331,8 @@ public:
     static basic_json binary(const typename binary_t::container_type& init)
     {
         auto res = basic_json();
-        res.m_data.m_type = value_t::binary;
         res.m_data.m_value = init;
+        res.m_data.m_type = value_t::binary;
         return res;
     }
 
@@ -2313,8 +2342,8 @@ public:
     static basic_json binary(const typename binary_t::container_type& init, typename binary_t::subtype_type subtype)
     {
         auto res = basic_json();
-        res.m_data.m_type = value_t::binary;
         res.m_data.m_value = binary_t(init, subtype);
+        res.m_data.m_type = value_t::binary;
         return res;
     }
 
@@ -2324,8 +2353,8 @@ public:
     static basic_json binary(typename binary_t::container_type&& init)
     {
         auto res = basic_json();
-        res.m_data.m_type = value_t::binary;
         res.m_data.m_value = std::move(init);
+        res.m_data.m_type = value_t::binary;
         return res;
     }
 
@@ -2335,8 +2364,8 @@ public:
     static basic_json binary(typename binary_t::container_type&& init, typename binary_t::subtype_type subtype)
     {
         auto res = basic_json();
-        res.m_data.m_type = value_t::binary;
         res.m_data.m_value = binary_t(std::move(init), subtype);
+        res.m_data.m_type = value_t::binary;
         return res;
     }
 
@@ -5654,12 +5683,13 @@ public:
                           input_format_t format = input_format_t::json,
                           const bool strict = true,
                           const bool ignore_comments = false,
-                          const bool ignore_trailing_commas = false)
+                          const bool ignore_trailing_commas = false,
+                          const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
     {
         auto ia = detail::input_adapter(std::forward<InputType>(i));
         return format == input_format_t::json
                ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
-               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(sax, strict);
+               : detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(sax, strict, tag_handler);
     }
 
     /// @brief generate SAX events (iterator pair, or iterator+sentinel pair for C++20 ranges support)
@@ -5671,12 +5701,13 @@ public:
                           input_format_t format = input_format_t::json,
                           const bool strict = true,
                           const bool ignore_comments = false,
-                          const bool ignore_trailing_commas = false)
+                          const bool ignore_trailing_commas = false,
+                          const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
     {
         auto ia = detail::input_adapter(std::move(first), std::move(last));
         return format == input_format_t::json
                ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
-               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(sax, strict);
+               : detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(sax, strict, tag_handler);
     }
 
     /// @brief generate SAX events
@@ -5704,7 +5735,8 @@ public:
                           input_format_t format = input_format_t::json,
                           const bool strict = true,
                           const bool ignore_comments = false,
-                          const bool ignore_trailing_commas = false)
+                          const bool ignore_trailing_commas = false,
+                          const cbor_tag_handler_t tag_handler = cbor_tag_handler_t::error)
 #if JSON_DELETE_DEPRECATED_FUNCTIONS
     = delete;
 #else
@@ -5714,7 +5746,7 @@ public:
                // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
                ? parser(std::move(ia), nullptr, true, ignore_comments, ignore_trailing_commas).sax_parse(sax, strict)
                // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg)
-               : detail::binary_reader<basic_json, decltype(ia), SAX>(std::move(ia), format).sax_parse(sax, strict);
+               : detail::binary_reader<basic_json, decltype(ia), SAX, true>(std::move(ia), format).sax_parse(sax, strict, tag_handler);
     }
 #endif
 #if defined(__clang__)

src/modules/json.cppm

@@ -47,6 +47,7 @@ inline namespace json_literals
 namespace detail
 {
     using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_callback_parser;
+    using NLOHMANN_JSON_NAMESPACE::detail::json_sax_dom_parser;
     using NLOHMANN_JSON_NAMESPACE::detail::unknown_size;
 } // namespace detail
 

tests/CMakeLists.txt

@@ -135,12 +135,14 @@ json_test_set_test_options(test-disabled_exceptions
         #$<$<CXX_COMPILER_ID:MSVC>:/EH>
 )
 
-# raise timeout of expensive Unicode test
-json_test_set_test_options(test-unicode4 TEST_PROPERTIES TIMEOUT 3000)
-
 # only the #972 regression test needs thirdparty/fifo_map on its include path
 json_test_set_test_options(test-regression1 LINK_LIBRARIES fifo_map_include)
 
+# GCC's false -Warray-bounds error with JSON_DIAGNOSTICS only shows up when optimizing (#5742)
+json_test_set_test_options(test-diagnostics-optimized
+    COMPILE_OPTIONS $<$<CXX_COMPILER_ID:GNU>:-O3 -Werror=array-bounds>
+)
+
 #############################################################################
 # add unit tests
 #############################################################################

tests/fuzzing.md

@@ -3,6 +3,14 @@
 Each parser of the library (JSON, BJData, BON8, BSON, CBOR, MessagePack, and UBJSON) can be fuzz tested. Currently,
 [libFuzzer](https://llvm.org/docs/LibFuzzer.html) and [afl++](https://github.com/AFLplusplus/AFLplusplus) are supported.
 
+## What the fuzzers check
+
+Each fuzzer driver (`tests/src/fuzzer-parse_*.cpp`) parses its input twice: once with `allow_exceptions = false` and
+once with exceptions. Both calls must agree. Where parsing with exceptions fails, the call without exceptions must
+return a discarded value (or throw the same kind of non-parse error), and it must never throw a `parse_error`. Where
+parsing succeeds, both calls must return the same value. The drivers then serialize the value, parse the result back,
+and check that nothing was lost. The drivers check all of this with `assert`, so they refuse to build with `NDEBUG`.
+
 ## Corpus creation
 
 For most effective fuzzing, a [corpus](https://llvm.org/docs/LibFuzzer.html#corpus) should be provided. A corpus is a
@@ -54,6 +62,9 @@ Then pass the corpus directory as command-line argument (assuming it is located
 The fuzzer should be able to run indefinitely without crashing. In case of a crash, the tested input is dumped into
 a file starting with `crash-`.
 
+To also detect memory leaks, build with AddressSanitizer (`FUZZER_ENGINE="-fsanitize=fuzzer,address"`): libFuzzer then
+runs LeakSanitizer by default (`-detect_leaks=1`). LeakSanitizer is not available with Apple Clang on macOS.
+
 ## afl++
 
 To use afl++, you need to pass `-fsanitize=fuzzer` as `FUZZER_ENGINE`. It will be replaced by a `libAFLDriver.a` to
@@ -76,7 +87,9 @@ directory `out`.
 
 The library is further fuzz-tested 24/7 by Google's [OSS-Fuzz project](https://github.com/google/oss-fuzz). It uses
 the same `fuzzers` target as above and also relies on the `FUZZER_ENGINE` variable. See the used
-[build script](https://github.com/google/oss-fuzz/blob/master/projects/json/build.sh) for more information.
+[build script](https://github.com/google/oss-fuzz/blob/master/projects/json/build.sh) for more information. Its default
+`address` sanitizer includes LeakSanitizer, so OSS-Fuzz and the CIFuzz workflow (`.github/workflows/cifuzz.yml`) report
+memory leaks, too.
 
 In case the build at OSS-Fuzz fails, an issue will be created automatically.
 

tests/src/fuzzer-parse_bjdata.cpp

@@ -10,7 +10,9 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_bjdata(data, allow_exceptions = false)
 - j1 = from_bjdata(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec2 = to_bjdata(j1, use_size = false, use_type = false)
 - vec3 = to_bjdata(j1, use_size = true, use_type = false)
 - vec4 = to_bjdata(j1, use_size = true, use_type = true)
@@ -45,6 +47,10 @@ dumps is stable under exactly the same values that break operator==.
 The unit tests run the same checks on a fixed corpus (see the "BJData round-trip
 invariants" test case), so keep both in sync.
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_bjdata() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -57,8 +63,17 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // value-stable comparison for the round-trip checks below; see the note
 // above on why this compares dump()s rather than the json values directly
 static bool is_value_stable(const json& lhs, const json& rhs)
@@ -69,11 +84,43 @@ static bool is_value_stable(const json& lhs, const json& rhs)
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::bjdata).errors == 0;
+
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_bjdata(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_bjdata(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -107,14 +154,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors may happen if provided sizes are excessive
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_bon8.cpp

@@ -10,7 +10,9 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_bon8(data, allow_exceptions = false)
 - j1 = from_bon8(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec = to_bon8(j1)
 - j2 = from_bon8(vec)
 - assert(to_bon8(j2) == vec)
@@ -19,6 +21,10 @@ It also checks that reading the data from a stream, which reads strings byte by
 byte, gives the same value or error as reading it from contiguous memory, which
 copies strings in bulk.
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_bon8() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -32,8 +38,17 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 namespace
 {
 // the serialization of the value read from @a input, or the error message
@@ -55,17 +70,49 @@ std::string read_bon8(InputType&& input)
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::bon8).errors == 0;
+
     // contiguous and stream input must be read alike
     {
         std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
         assert(read_bon8(std::vector<uint8_t>(data, data + size)) == read_bon8(stream));
     }
 
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_bon8(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_bon8(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -87,14 +134,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors may happen if provided sizes are excessive
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_bson.cpp

@@ -10,11 +10,17 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_bson(data, allow_exceptions = false)
 - j1 = from_bson(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec = to_bson(j1)
 - j2 = from_bson(vec)
 - assert(to_bson(j2) == vec)
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_bson() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -27,16 +33,57 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::bson).errors == 0;
+
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_bson(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_bson(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -58,14 +105,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_cbor.cpp

@@ -10,11 +10,17 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_cbor(data, allow_exceptions = false)
 - j1 = from_cbor(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec = to_cbor(j1)
 - j2 = from_cbor(vec)
 - assert(to_cbor(j2) == vec)
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_cbor() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -27,16 +33,57 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::cbor).errors == 0;
+
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_cbor(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_cbor(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -58,14 +105,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_json.cpp

@@ -10,12 +10,18 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = parse(data, allow_exceptions = false)
 - j1 = parse(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - s1 = serialize(j1)
 - j2 = parse(s1)
 - s2 = serialize(j2)
 - assert(s1 == s2)
 
+Furthermore, it parses data with a SAX parser that recovers from every error
+and checks that the events are balanced, that parsing ends, and that valid
+input is parsed without errors (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -28,15 +34,56 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    {
+        const auto checker = check_recovering_parse(data, size, json::input_format_t::json);
+        assert(checker.events <= (4 * size) + 4);
+        assert((checker.errors == 0) == json::accept(data, data + size));
+    }
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::parse(data, data + size, nullptr, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
         json const j1 = json::parse(data, data + size);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
 
         try
         {
@@ -63,10 +110,12 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors may happen if provided sizes are excessive
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_msgpack.cpp

@@ -10,11 +10,17 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_msgpack(data, allow_exceptions = false)
 - j1 = from_msgpack(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec = to_msgpack(j1)
 - j2 = from_msgpack(vec)
 - assert(to_msgpack(j2) == vec)
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_msgpack() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -27,16 +33,57 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::msgpack).errors == 0;
+
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_msgpack(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_msgpack(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -58,14 +105,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors may happen if provided sizes are excessive
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-parse_ubjson.cpp

@@ -10,7 +10,9 @@
 This file implements a parser test suitable for fuzz testing. Given a byte
 array data, it performs the following steps:
 
+- j0 = from_ubjson(data, allow_exceptions = false)
 - j1 = from_ubjson(data)
+- assert(j0 is discarded if parsing j1 fails, and j0 == j1 otherwise)
 - vec2 = to_ubjson(j1, use_size = false, use_type = false)
 - vec3 = to_ubjson(j1, use_size = true, use_type = false)
 - vec4 = to_ubjson(j1, use_size = true, use_type = true)
@@ -24,6 +26,10 @@ array data, it performs the following steps:
 The unit tests run the same checks on a fixed corpus (see the "UBJSON round-trip
 invariants" test case), so keep both in sync.
 
+Furthermore, it reads data with a SAX parser that recovers from every error
+and checks that the events are balanced, that reading ends, and that it
+reports an error exactly when from_ubjson() fails (see #3989).
+
 The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
 drivers.
 */
@@ -36,16 +42,57 @@ drivers.
     #error "the fuzzer drivers must be built without NDEBUG"
 #endif
 
+#include "fuzzer-recovering_checker.hpp"
+
 using json = nlohmann::json;
 
+// compares dumps rather than values, because NaN != NaN; keep writes strings
+// byte for byte, so ill-formed UTF-8 that a binary reader accepts cannot throw
+static bool same_value(const json& lhs, const json& rhs)
+{
+    return lhs.dump(-1, ' ', false, json::error_handler_t::keep) == rhs.dump(-1, ' ', false, json::error_handler_t::keep);
+}
+
 // see http://llvm.org/docs/LibFuzzer.html
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
 {
+    // recover from all errors, reading from memory and from a stream
+    const bool recovered_without_errors = check_recovering_parse(data, size, json::input_format_t::ubjson).errors == 0;
+
+    std::vector<uint8_t> const vec1(data, data + size);
+
+    // step 0: parse input without exceptions; a parse error must then be
+    // reported as a discarded value, never thrown
+    json j_noexcept;
+    bool noexcept_threw = false;
+    try
+    {
+        j_noexcept = json::from_ubjson(vec1, true, false);
+    }
+    catch (const json::parse_error&)
+    {
+        assert(false);
+    }
+    catch (const json::exception&)
+    {
+        // type and out-of-range errors are not parse errors and still throw
+        noexcept_threw = true;
+    }
+    // whether step 1 succeeded; if not, the catch blocks below check that
+    // step 0 failed, too
+    bool parsed = false;
+
     try
     {
         // step 1: parse input
-        std::vector<uint8_t> const vec1(data, data + size);
         json const j1 = json::from_ubjson(vec1);
+        parsed = true;
+
+        // without exceptions, the same input must give the same value
+        assert(!noexcept_threw && !j_noexcept.is_discarded() && same_value(j_noexcept, j1));
+
+        // the recovering parser must not have reported an error either
+        assert(recovered_without_errors);
 
         try
         {
@@ -77,14 +124,19 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
     catch (const json::parse_error&)
     {
         // parse errors are ok, because input may be random bytes
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
     catch (const json::type_error&)
     {
         // type errors can occur during parsing, too
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
     }
     catch (const json::out_of_range&)
     {
         // out of range errors may happen if provided sizes are excessive
+        assert(parsed || noexcept_threw || j_noexcept.is_discarded());
+        assert(parsed || !recovered_without_errors);
     }
 
     // return 0 - non-zero return values are reserved for future use

tests/src/fuzzer-recovering_checker.hpp

@@ -0,0 +1,154 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <nlohmann/json.hpp>
+
+namespace
+{
+// a SAX parser that recovers from every error and checks that the events are
+// balanced and that every key is followed by exactly one value
+class recovering_checker : public nlohmann::json_sax<nlohmann::json>
+{
+  public:
+    bool null() override
+    {
+        return value();
+    }
+
+    bool boolean(bool /*val*/) override
+    {
+        return value();
+    }
+
+    bool number_integer(number_integer_t /*val*/) override
+    {
+        return value();
+    }
+
+    bool number_unsigned(number_unsigned_t /*val*/) override
+    {
+        return value();
+    }
+
+    bool number_float(number_float_t /*val*/, const string_t& /*s*/) override
+    {
+        return value();
+    }
+
+    bool string(string_t& /*val*/) override
+    {
+        return value();
+    }
+
+    bool binary(binary_t& /*val*/) override
+    {
+        return value();
+    }
+
+    bool start_object(std::size_t /*elements*/) override
+    {
+        value();
+        stack.push_back('o');
+        return true;
+    }
+
+    bool key(string_t& /*val*/) override
+    {
+        ++events;
+        assert(!stack.empty() && stack.back() == 'o');
+        stack.back() = 'v';
+        return true;
+    }
+
+    bool end_object() override
+    {
+        ++events;
+        assert(!stack.empty() && stack.back() == 'o');
+        stack.pop_back();
+        return true;
+    }
+
+    bool start_array(std::size_t /*elements*/) override
+    {
+        value();
+        stack.push_back('a');
+        return true;
+    }
+
+    bool end_array() override
+    {
+        ++events;
+        assert(!stack.empty() && stack.back() == 'a');
+        stack.pop_back();
+        return true;
+    }
+
+    bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const nlohmann::detail::exception& /*ex*/) override
+    {
+        ++errors;
+        return true;
+    }
+
+    bool complete() const
+    {
+        return stack.empty();
+    }
+
+    std::size_t events = 0;
+    std::size_t errors = 0;
+
+  private:
+    bool value()
+    {
+        ++events;
+        if (!stack.empty())
+        {
+            // an array element, or the value of a key
+            assert(stack.back() != 'o');
+            if (stack.back() == 'v')
+            {
+                stack.back() = 'o';
+            }
+        }
+        return true;
+    }
+
+    // 'a' for an array, 'o' for an object that expects a key, 'v' for an
+    // object that expects the value of a key
+    std::vector<char> stack {}; // NOLINT(readability-redundant-member-init)
+};
+
+/// parses @a data with a recovering_checker from memory and from a stream,
+/// checks that both see the same, that the events are balanced, and that the
+/// number of errors is bounded, and returns the checker (see #3989)
+inline recovering_checker check_recovering_parse(const std::uint8_t* data, const std::size_t size, const nlohmann::json::input_format_t format)
+{
+    recovering_checker checker;
+    const bool ok = nlohmann::json::sax_parse(data, data + size, &checker, format);
+    assert(checker.complete());
+    assert(checker.errors <= size + 1);
+    assert(ok == (checker.errors == 0));
+
+    std::istringstream stream(std::string(reinterpret_cast<const char*>(data), size));
+    recovering_checker stream_checker;
+    assert(nlohmann::json::sax_parse(stream, &stream_checker, format) == ok);
+    assert(stream_checker.complete());
+    assert(stream_checker.events == checker.events);
+    assert(stream_checker.errors == checker.errors);
+
+    return checker;
+}
+} // namespace

tests/src/test_utils.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <array> // array
 #include <cstdint> // uint8_t
 #include <cstddef> // size_t
 #include <fstream> // ifstream, ios
@@ -43,6 +44,33 @@ T next_integer_sample(T i, T last, T stride)
     return n < last ? n : last;
 }
 
+// UTF-8 continuation bytes in [lo, hi] that stand in for all of them in the
+// ill-formed UTF-8 tests. Both the lexer's range checks and the serializer's
+// decoder (detail::decode) only distinguish the classes 0x80..0x8F, 0x90..0x9F,
+// and 0xA0..0xBF, so the first and last byte of each class within [lo, hi]
+// exercise every behavior while a test sweeps another byte position through
+// all 256 values (#5418). Define JSON_TEST_UTF8_EXHAUSTIVE to get every byte.
+inline std::vector<int> utf8_continuation_bytes(int lo, int hi)
+{
+    std::vector<int> result;
+#ifdef JSON_TEST_UTF8_EXHAUSTIVE
+    for (int byte = lo; byte <= hi; ++byte)
+    {
+        result.push_back(byte);
+    }
+#else
+    static const std::array<int, 6> class_ends = {{0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF}};
+    for (const int byte : class_ends)
+    {
+        if (lo <= byte && byte <= hi)
+        {
+            result.push_back(byte);
+        }
+    }
+#endif
+    return result;
+}
+
 inline std::vector<std::uint8_t> read_binary_file(const std::string& filename)
 {
     std::ifstream file(filename, std::ios::binary);

tests/src/unit-allocator.cpp

@@ -12,6 +12,11 @@
 #include <nlohmann/json.hpp>
 using nlohmann::json;
 
+#include <valarray>
+#if JSON_HAS_RANGES
+    #include <ranges>
+#endif
+
 namespace
 {
 // special test case to check if memory is leaked if constructor throws
@@ -671,3 +676,173 @@ TEST_CASE("destructor performs no allocation, only deallocation")
         CHECK(counting_allocator_deallocations > deallocations_before);
     }
 }
+
+// the no-exceptions CI job skips every CHECK_THROWS_AS, which would leave
+// next_construct_fails set for the next allocation outside a check
+#if !defined(JSON_NOEXCEPTION)
+TEST_CASE("a failed allocation leaves the value unchanged")
+{
+    // create JSON type using the throwing allocator
+    using my_json = nlohmann::basic_json<std::map,
+          std::vector,
+          std::string,
+          bool,
+          std::int64_t,
+          std::uint64_t,
+          double,
+          my_allocator>;
+
+    // Each of these creates a string, array, object, or binary value. The
+    // value must be created before the type is changed: otherwise, a failed
+    // creation left a value of the new type without anything behind it (an
+    // assertion in its destructor, a null pointer everywhere else) or, when
+    // an old value was destroyed first, with a pointer to that destroyed one.
+
+    SECTION("creating a binary value")
+    {
+        const std::vector<std::uint8_t> bytes = {1, 2, 3};
+        my_json _;
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(_ = my_json::binary(bytes), std::bad_alloc&);
+        next_construct_fails = true;
+        CHECK_THROWS_AS(_ = my_json::binary(bytes, 42), std::bad_alloc&);
+        next_construct_fails = true;
+        CHECK_THROWS_AS(_ = my_json::binary(std::vector<std::uint8_t>(bytes)), std::bad_alloc&);
+        next_construct_fails = true;
+        CHECK_THROWS_AS(_ = my_json::binary(std::vector<std::uint8_t>(bytes), 42), std::bad_alloc&);
+        next_construct_fails = false;
+    }
+
+    SECTION("turning a null value into an array or object")
+    {
+        my_json j;
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j[0], std::bad_alloc&);
+        CHECK(j.is_null());
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j["key"], std::bad_alloc&);
+        CHECK(j.is_null());
+
+#ifdef JSON_HAS_CPP_17
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j[std::string_view("key")], std::bad_alloc&);
+        CHECK(j.is_null());
+#endif
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.push_back(my_json(1)), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        const my_json one = 1;
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.push_back(one), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.push_back(my_json::object_t::value_type("key", 1)), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.emplace_back(1), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.emplace("key", 1), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        const my_json object = {{"key", 1}};
+        next_construct_fails = true;
+        CHECK_THROWS_AS(j.update(object), std::bad_alloc&);
+        CHECK(j.is_null());
+
+        next_construct_fails = false;
+    }
+
+    // With iterator debugging, VS 2015's containers construct a proxy with the
+    // allocator in constructors that cannot report its failure, so a failing
+    // allocator crashes this section there (SIGSEGV with VS 2015 Debug x86).
+#if !(defined(_MSC_VER) && _MSC_VER < 1910 && defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0)
+    SECTION("converting into an existing value")
+    {
+        // to_json replaces the value it is given; the old one must survive a
+        // failed creation of the new one
+        my_json j = "old";
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::string("new")), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::vector<int> {1, 2}), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::vector<bool> {true, false}), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::map<std::string, int> {{"a", 1}}), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, my_json::binary_t({1, 2})), std::bad_alloc&);
+        CHECK(j == "old");
+
+        // the overloads for lvalues of the value types, for the value types
+        // themselves, and for the remaining compatible types
+        const std::string string = "new";
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, string), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, "new"), std::bad_alloc&);
+        CHECK(j == "old");
+
+        // to_json only moves a binary value that it converted from another
+        // container type, which my_json's std::vector<std::uint8_t> is not
+        using binary_constructor = nlohmann::detail::external_constructor<nlohmann::detail::value_t::binary>;
+        next_construct_fails = true;
+        CHECK_THROWS_AS(binary_constructor::construct(j, my_json::binary_t({1, 2})), std::bad_alloc&);
+        CHECK(j == "old");
+
+        my_json::array_t array = {1, 2};
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, array), std::bad_alloc&);
+        CHECK(j == "old");
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::move(array)), std::bad_alloc&);
+        CHECK(j == "old");
+
+        my_json::object_t object = {{"a", 1}};
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, object), std::bad_alloc&);
+        CHECK(j == "old");
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::move(object)), std::bad_alloc&);
+        CHECK(j == "old");
+
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, std::valarray<int> {1, 2}), std::bad_alloc&);
+        CHECK(j == "old");
+
+#if JSON_HAS_RANGES && !defined(__MINGW32__)
+        const std::vector<int> numbers = {1, 2};
+        next_construct_fails = true;
+        CHECK_THROWS_AS(nlohmann::to_json(j, numbers | std::views::filter([](int /*unused*/)
+        {
+            return true;
+        })), std::bad_alloc&);
+        CHECK(j == "old");
+#endif
+
+        next_construct_fails = false;
+        nlohmann::to_json(j, std::vector<int> {1, 2});
+        CHECK(j == my_json({1, 2}));
+    }
+#endif
+}
+#endif

tests/src/unit-alt-string.cpp

@@ -415,6 +415,46 @@ TEST_CASE("alternative string type")
         CHECK(j2.dump() == R"({"/foo/0":"bar","/foo/1":"baz"})");
     }
 
+    SECTION("error recovery")
+    {
+        // a SAX parser that recovers from every error (see #3989)
+        struct recovering_parser : nlohmann::detail::json_sax_dom_parser<alt_json>
+        {
+            explicit recovering_parser(alt_json& j)
+                : nlohmann::detail::json_sax_dom_parser<alt_json>(j, false)
+            {}
+
+            // sax_parse() calls the SAX parser's own parse_error(), so hiding
+            // the one of the base class is what recovering takes
+            // NOLINTNEXTLINE(bugprone-derived-method-shadowing-base-method)
+            bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const nlohmann::detail::exception& /*unused*/)
+            {
+                ++errors;
+                return true;
+            }
+
+            std::size_t errors = 0;
+        };
+
+        alt_json j;
+        recovering_parser sax(j);
+        // not inside CHECK(): MSVC reads the escape in a stringized raw string
+        const std::string input = R"([1., "a\qb", tru, {"k" 2}])";
+        CHECK(!alt_json::sax_parse(input, &sax));
+        CHECK(sax.errors == 4);
+        CHECK(j.dump() == R"([1,"aqb",null,{"k":2}])");
+
+        // a UBJSON high-precision number, a CBOR key that is not a string
+        alt_json u;
+        recovering_parser ubjson_sax(u);
+        CHECK(!alt_json::sax_parse(std::vector<std::uint8_t> {'[', 'H', 'i', 2, '1', '.', ']'}, &ubjson_sax, alt_json::input_format_t::ubjson));
+        CHECK(u.dump() == "[1]");
+        alt_json c;
+        recovering_parser cbor_sax(c);
+        CHECK(!alt_json::sax_parse(std::vector<std::uint8_t> {0xA2, 0x01, 0x02, 0x61, 'a', 0x03}, &cbor_sax, alt_json::input_format_t::cbor));
+        CHECK(c.dump() == R"({"a":3})");
+    }
+
     SECTION("conversion between basic_json specializations (#2649)")
     {
         // explicit conversions are always possible

tests/src/unit-binary_formats.cpp

@@ -19,7 +19,7 @@ using nlohmann::json;
 #include <vector>
 #include "make_test_data_available.hpp"
 
-TEST_CASE("Binary Formats" * doctest::skip())
+TEST_CASE("Binary Formats")
 {
     SECTION("canada.json")
     {
@@ -147,48 +147,6 @@ TEST_CASE("Binary Formats" * doctest::skip())
         CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(84.963));
     }
 
-    SECTION("jeopardy.json")
-    {
-        const auto* filename = TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json";
-        json j = json::parse(std::ifstream(filename));
-
-        const auto json_size = j.dump().size();
-        const auto bjdata_1_size = json::to_bjdata(j).size();
-        const auto bjdata_2_size = json::to_bjdata(j, true).size();
-        const auto bjdata_3_size = json::to_bjdata(j, true, true).size();
-        const auto bon8_size = json::to_bon8(j).size();
-        const auto bson_size = json::to_bson({{"", j}}).size(); // wrap array in object for BSON
-        const auto cbor_size = json::to_cbor(j).size();
-        const auto msgpack_size = json::to_msgpack(j).size();
-        const auto ubjson_1_size = json::to_ubjson(j).size();
-        const auto ubjson_2_size = json::to_ubjson(j, true).size();
-        const auto ubjson_3_size = json::to_ubjson(j, true, true).size();
-
-        CHECK(json_size == 52508728);
-        CHECK(bjdata_1_size == 50710965);
-        CHECK(bjdata_2_size == 51144830);
-        CHECK(bjdata_3_size == 51144830);
-        CHECK(bon8_size == 45942080);
-        CHECK(bson_size == 56008520);
-        CHECK(cbor_size == 46187320);
-        CHECK(msgpack_size == 46158575);
-        CHECK(ubjson_1_size == 50710965);
-        CHECK(ubjson_2_size == 51144830);
-        CHECK(ubjson_3_size == 49861422);
-
-        CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0));
-        CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(96.576));
-        CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(97.402));
-        CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(97.402));
-        CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494));
-        CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(106.665));
-        CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.961));
-        CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.906));
-        CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576));
-        CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402));
-        CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958));
-    }
-
     SECTION("sample.json")
     {
         const auto* filename = TEST_DATA_DIRECTORY "/json_testsuite/sample.json";
@@ -230,6 +188,50 @@ TEST_CASE("Binary Formats" * doctest::skip())
     }
 }
 
+// jeopardy.json is 52 MB and produces ~500 MB of serialization output, so it
+// is kept apart from the cheap corpus files above (#5418)
+TEST_CASE("Binary Formats (jeopardy.json)" * doctest::skip())
+{
+    const auto* filename = TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json";
+    json j = json::parse(std::ifstream(filename));
+
+    const auto json_size = j.dump().size();
+    const auto bjdata_1_size = json::to_bjdata(j).size();
+    const auto bjdata_2_size = json::to_bjdata(j, true).size();
+    const auto bjdata_3_size = json::to_bjdata(j, true, true).size();
+    const auto bon8_size = json::to_bon8(j).size();
+    const auto bson_size = json::to_bson({{"", j}}).size(); // wrap array in object for BSON
+    const auto cbor_size = json::to_cbor(j).size();
+    const auto msgpack_size = json::to_msgpack(j).size();
+    const auto ubjson_1_size = json::to_ubjson(j).size();
+    const auto ubjson_2_size = json::to_ubjson(j, true).size();
+    const auto ubjson_3_size = json::to_ubjson(j, true, true).size();
+
+    CHECK(json_size == 52508728);
+    CHECK(bjdata_1_size == 50710965);
+    CHECK(bjdata_2_size == 51144830);
+    CHECK(bjdata_3_size == 51144830);
+    CHECK(bon8_size == 45942080);
+    CHECK(bson_size == 56008520);
+    CHECK(cbor_size == 46187320);
+    CHECK(msgpack_size == 46158575);
+    CHECK(ubjson_1_size == 50710965);
+    CHECK(ubjson_2_size == 51144830);
+    CHECK(ubjson_3_size == 49861422);
+
+    CHECK((100.0 * double(json_size) / double(json_size)) == Approx(100.0));
+    CHECK((100.0 * double(bjdata_1_size) / double(json_size)) == Approx(96.576));
+    CHECK((100.0 * double(bjdata_2_size) / double(json_size)) == Approx(97.402));
+    CHECK((100.0 * double(bjdata_3_size) / double(json_size)) == Approx(97.402));
+    CHECK((100.0 * double(bon8_size) / double(json_size)) == Approx(87.494));
+    CHECK((100.0 * double(bson_size) / double(json_size)) == Approx(106.665));
+    CHECK((100.0 * double(cbor_size) / double(json_size)) == Approx(87.961));
+    CHECK((100.0 * double(msgpack_size) / double(json_size)) == Approx(87.906));
+    CHECK((100.0 * double(ubjson_1_size) / double(json_size)) == Approx(96.576));
+    CHECK((100.0 * double(ubjson_2_size) / double(json_size)) == Approx(97.402));
+    CHECK((100.0 * double(ubjson_3_size) / double(json_size)) == Approx(94.958));
+}
+
 namespace
 {
 // the binary formats as function pointers for "Binary formats with narrow number types";

tests/src/unit-bjdata.cpp

@@ -4603,3 +4603,49 @@ TEST_CASE("issue #5648 - from_bjdata(ptr, len) must read len bytes, not treat pt
     CHECK(json::from_bjdata(packed.data(), packed.size(), false) == j);
 #endif
 }
+
+TEST_CASE("BJData large strings and binaries (chunked reader)")
+{
+    // Strings share get_ubjson_string() -> get_string() -> get_bytes() with
+    // plain UBJSON. Binary values are different: only a Draft 3 optimized
+    // array (type marker 'B') is read back as a binary value, through
+    // get_binary() -> get_bytes() (see parse_ubjson_internal()'s "If BJData
+    // type marker is 'B'" branch); Draft 2 (the default) writes a binary
+    // value as a plain array of uint8_t numbers instead (see the "round trip
+    // of a binary value is value-stable, not byte-stable" test above), which
+    // never reaches get_bytes(). Both reads happen in bounded chunks
+    // (binary_reader.hpp, chunk_size == 4096); check lengths around and
+    // beyond that size, for both vector (iterator) and pointer inputs.
+    for (const std::size_t len :
+            {
+                std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
+                std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
+            })
+    {
+        CAPTURE(len)
+
+        // string
+        const json j_string = std::string(len, 'x');
+        const std::vector<std::uint8_t> v_string = json::to_bjdata(j_string);
+        CHECK(json::from_bjdata(v_string) == j_string);
+        // pointer input exercises the std::memcpy fast path
+        CHECK(json::from_bjdata(reinterpret_cast<const char*>(v_string.data()),
+                                reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
+
+        // binary, forced into the Draft 3 optimized ('B' marker) encoding
+        const json j_binary = json::binary(std::vector<std::uint8_t>(len, 0xCD));
+        const std::vector<std::uint8_t> v_binary = json::to_bjdata(j_binary, true, true, json::bjdata_version_t::draft3);
+        CHECK(json::from_bjdata(v_binary) == j_binary);
+        CHECK(json::from_bjdata(reinterpret_cast<const char*>(v_binary.data()),
+                                reinterpret_cast<const char*>(v_binary.data()) + v_binary.size()) == j_binary);
+
+        // a truncated payload must still be reported as an error
+        if (len > 16)
+        {
+            std::vector<std::uint8_t> truncated = v_string;
+            truncated.resize(truncated.size() - 8);
+            json _;
+            CHECK_THROWS_AS(_ = json::from_bjdata(truncated), json::parse_error);
+        }
+    }
+}

tests/src/unit-brace-init-copy-semantics.cpp

@@ -72,6 +72,28 @@ TEST_CASE("JSON_BRACE_INIT_COPY_SEMANTICS")
         CHECK(j7 == json::array({1, 2}));
     }
 
+    SECTION("single-element brace initialization copies a pair-shaped array value (#5662)")
+    {
+        // a JSON value that happens to be a 2-element array whose first
+        // element is a string must still be copied, not turned into an
+        // object; only a braced list written in the source, such as the
+        // inner {"key", "value"} of {{"key", "value"}}, describes an object
+        json const pair_shaped = json::array({"key", 42});
+
+        json const j1{pair_shaped};
+        CHECK(j1.is_array());
+        CHECK(j1 == pair_shaped);
+
+        json const j2 = {pair_shaped};
+        CHECK(j2.is_array());
+        CHECK(j2 == pair_shaped);
+
+        // the same holds for an rvalue of the same shape
+        json const j3{json::array({"key", 42})};
+        CHECK(j3.is_array());
+        CHECK(j3 == pair_shaped);
+    }
+
     SECTION("what the macro does not change")
     {
         // lists with more than one element are unaffected

tests/src/unit-bson.cpp

@@ -1993,3 +1993,47 @@ TEST_CASE("Invalid document size handling")
         CHECK(json::from_bson(v, true, false).is_discarded());
     }
 }
+
+TEST_CASE("BSON large strings and binaries (chunked reader)")
+{
+    // get_bson_string()/get_bson_binary() both read through get_string()/
+    // get_binary(), which read in bounded chunks (binary_reader.hpp,
+    // chunk_size == 4096); make sure roundtripping is correct for lengths
+    // around and beyond that chunk size, for both vector (iterator) and
+    // pointer inputs. BSON only accepts an object at the top level, so the
+    // string/binary value is wrapped in one.
+    for (const std::size_t len :
+            {
+                std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
+                std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
+            })
+    {
+        CAPTURE(len)
+
+        // string
+        const json j_string = {{"k", std::string(len, 'x')}};
+        const std::vector<std::uint8_t> v_string = json::to_bson(j_string);
+        CHECK(json::from_bson(v_string) == j_string);
+        // pointer input exercises the std::memcpy fast path
+        CHECK(json::from_bson(reinterpret_cast<const char*>(v_string.data()),
+                              reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
+
+        // binary (BSON binary values always carry a subtype, so give one
+        // explicitly; otherwise from_bson() would round-trip to subtype 0
+        // rather than back to the original "no subtype" value)
+        const json j_binary = {{"k", json::binary(std::vector<std::uint8_t>(len, 0xCD), std::uint8_t{0})}};
+        const std::vector<std::uint8_t> v_binary = json::to_bson(j_binary);
+        CHECK(json::from_bson(v_binary) == j_binary);
+        CHECK(json::from_bson(reinterpret_cast<const char*>(v_binary.data()),
+                              reinterpret_cast<const char*>(v_binary.data()) + v_binary.size()) == j_binary);
+
+        // a truncated payload must still be reported as an error
+        if (len > 16)
+        {
+            std::vector<std::uint8_t> truncated = v_string;
+            truncated.resize(truncated.size() - 8);
+            json _;
+            CHECK_THROWS_AS(_ = json::from_bson(truncated), json::parse_error);
+        }
+    }
+}

tests/src/unit-cbor.cpp

@@ -1699,7 +1699,7 @@ TEST_CASE("CBOR")
             CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0xA1, 0x61, 0X61})), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
             CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0xBF, 0x61, 0X61})), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
             CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&);
-            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&);
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B); last byte: 0x00", json::parse_error&);
             CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x41})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&);
 
             CHECK(json::from_cbor(std::vector<uint8_t>({0x18}), true, false).is_discarded());
@@ -2305,22 +2305,21 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
 {
     // Reading an indefinite-length string or byte array used to call itself
     // once per chunk, so a payload of repeated 0x7F (or 0x5F) bytes exhausted
-    // the call stack before any of the input was rejected. The open levels are
-    // counted now, and the levels below prove the reader still reads the same
-    // values and reports the same errors at the same byte offsets.
+    // the call stack before any of the input was rejected. Nested indefinite
+    // chunks are now rejected at the second byte, without recursing.
     json _;
 
-    SECTION("many open levels are reported, not crashed on")
+    SECTION("nested levels are rejected, not crashed on")
     {
         const std::vector<uint8_t> input(200000, 0x7F);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR string: unexpected end of input", json::parse_error&);
+        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&);
         CHECK(json::from_cbor(input, true, false).is_discarded());
     }
 
-    SECTION("many open levels are reported, not crashed on (binary)")
+    SECTION("nested levels are rejected, not crashed on (binary)")
     {
         const std::vector<uint8_t> input(200000, 0x5F);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.110] parse error at byte 200001: syntax error while parsing CBOR binary: unexpected end of input", json::parse_error&);
+        CHECK_THROWS_WITH_AS(_ = json::from_cbor(input), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&);
         CHECK(json::from_cbor(input, true, false).is_discarded());
     }
 
@@ -2328,22 +2327,22 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
     {
         CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0xFF})) == json(""));
         CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x61, 0x61, 0xFF})) == json("a"));
-        // nested indefinite-length strings are concatenated across levels
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0x61, 0x62, 0xFF})) == json("ab"));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x7F, 0x61, 0x7A, 0xFF, 0xFF, 0xFF})) == json("z"));
+        // empty and nonempty definite-length chunks concatenate in order
+        CHECK(json::from_cbor(std::vector<uint8_t>({0x7F, 0x61, 'a', 0x60, 0x61, 'b', 0x61, 'c', 0xFF})) == json("abc"));
         CHECK(json::from_cbor(std::vector<uint8_t>({0xA1, 0x7F, 0x61, 0x61, 0xFF, 0x01})) == json({{"a", 1}}));
     }
 
     SECTION("chunks are still concatenated (binary)")
     {
         CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x41, 0x61, 0xFF})) == json::binary({0x61}));
-        CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0x41, 0x62, 0xFF})) == json::binary({0x61, 0x62}));
+        CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0xFF})) == json::binary({}));
+        CHECK(json::from_cbor(std::vector<uint8_t>({0x5F, 0x41, 0x61, 0x40, 0x41, 0x62, 0x41, 0x63, 0xFF})) == json::binary({0x61, 0x62, 0x63}));
     }
 
     SECTION("a chunk that is not a string is still rejected")
     {
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7F, 0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x00", json::parse_error&);
-        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B) or indefinite binary array type (0x5F); last byte: 0x00", json::parse_error&);
+        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B); last byte: 0x00", json::parse_error&);
+        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x5F, 0x00})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: expected length specification (0x40-0x5B); last byte: 0x00", json::parse_error&);
     }
 
     SECTION("a break marker outside an indefinite-length string is not a string")
@@ -2896,9 +2895,17 @@ TEST_CASE("examples from RFC 8949 Appendix A")
     {
         const auto packed = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.cbor");
         json j;
-        CHECK_NOTHROW(j = json::from_cbor(packed));
+        // the fixture's tail contains nested indefinite-length byte strings.
+        CHECK_THROWS_WITH_AS(j = json::from_cbor(packed), "[json.exception.parse_error.113] parse error at byte 513: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&);
 
-        const auto expected = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.out");
+        // keep the byte-for-byte decoding check for its valid prefix: the first
+        // 512 encoded bytes contain 468 payload bytes in definite-length chunks.
+        auto valid_prefix = packed;
+        valid_prefix.resize(512);
+        valid_prefix.push_back(0xFF);
+        auto expected = utils::read_binary_file(TEST_DATA_DIRECTORY "/binary_data/cbor_binary.out");
+        expected.resize(468);
+        CHECK_NOTHROW(j = json::from_cbor(valid_prefix));
         CHECK(j == json::binary(expected));
 
         // 0xd8

tests/src/unit-class_parser.cpp

@@ -143,11 +143,13 @@ class SaxEventLogger
     {
         errored = true;
         events.push_back("parse_error(" + std::to_string(position) + ")");
-        return false;
+        return recover;
     }
 
     std::vector<std::string> events {}; // NOLINT(readability-redundant-member-init)
     bool errored = false;
+    /// whether parse_error() asks the parser to recover from the error (see #3989)
+    bool recover = false;
 };
 
 class SaxCountdown : public nlohmann::json::json_sax_t
@@ -3026,3 +3028,583 @@ TEST_CASE("diagnostic positions: value lifetime, input adapters, and SAX")
     }
 }
 #endif
+
+namespace
+{
+/// builds a value like json::parse(), but asks the parser to recover from
+/// errors (see #3989), and checks that the events it receives are balanced
+class RecoveringDomParser
+{
+  public:
+    explicit RecoveringDomParser(json& j, std::size_t max_errors_ = static_cast<std::size_t>(-1))
+        : dom(j, false)
+        , max_errors(max_errors_)
+    {}
+
+    bool null()
+    {
+        value();
+        return dom.null();
+    }
+
+    bool boolean(bool val)
+    {
+        value();
+        return dom.boolean(val);
+    }
+
+    bool number_integer(json::number_integer_t val)
+    {
+        value();
+        return dom.number_integer(val);
+    }
+
+    bool number_unsigned(json::number_unsigned_t val)
+    {
+        value();
+        return dom.number_unsigned(val);
+    }
+
+    bool number_float(json::number_float_t val, const std::string& s)
+    {
+        value();
+        return dom.number_float(val, s);
+    }
+
+    bool string(std::string& val)
+    {
+        value();
+        return dom.string(val);
+    }
+
+    bool binary(json::binary_t& val)
+    {
+        value();
+        return dom.binary(val);
+    }
+
+    bool start_object(std::size_t elements)
+    {
+        value();
+        stack.push_back('o');
+        return dom.start_object(elements);
+    }
+
+    bool key(std::string& val)
+    {
+        ++events;
+        if (stack.empty() || stack.back() != 'o')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.back() = 'v';
+        return dom.key(val);
+    }
+
+    bool end_object()
+    {
+        ++events;
+        if (stack.empty() || stack.back() != 'o')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.pop_back();
+        return dom.end_object();
+    }
+
+    bool start_array(std::size_t elements)
+    {
+        value();
+        stack.push_back('a');
+        return dom.start_array(elements);
+    }
+
+    bool end_array()
+    {
+        ++events;
+        if (stack.empty() || stack.back() != 'a')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.pop_back();
+        return dom.end_array();
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& ex)
+    {
+        errors.emplace_back(ex.what());
+        return errors.size() < max_errors;
+    }
+
+    /// whether the events were balanced and every key was followed by a value
+    bool balanced() const
+    {
+        return well_formed && stack.empty();
+    }
+
+    /// builds the value
+    nlohmann::detail::json_sax_dom_parser<json> dom;
+    std::vector<std::string> errors {}; // NOLINT(readability-redundant-member-init)
+    std::size_t events = 0;
+    /// the open containers: 'a' for an array, 'o' for an object that expects
+    /// a key, 'v' for an object that expects the value of a key
+    std::vector<char> stack {}; // NOLINT(readability-redundant-member-init)
+    bool well_formed = true;
+    std::size_t max_errors;
+
+  private:
+    /// a value is passed: it is an array element, or the value of a key
+    void value()
+    {
+        ++events;
+        if (!stack.empty())
+        {
+            if (stack.back() == 'v')
+            {
+                stack.back() = 'o';
+            }
+            else if (stack.back() == 'o')
+            {
+                // a value without a key
+                well_formed = false;
+            }
+        }
+    }
+
+};
+
+struct RecoveryResult
+{
+    json value;
+    std::vector<std::string> errors;
+    std::size_t events;
+    bool ok;
+    bool balanced;
+};
+
+template<typename InputType>
+RecoveryResult parse_recovering(InputType&& input, const bool strict = true,
+                                const bool ignore_comments = false, const bool ignore_trailing_commas = false)
+{
+    json j;
+    RecoveringDomParser sax(j);
+    const bool ok = json::sax_parse(std::forward<InputType>(input), &sax, json::input_format_t::json,
+                                    strict, ignore_comments, ignore_trailing_commas);
+    return {j, sax.errors, sax.events, ok, sax.balanced()};
+}
+
+/// stops after a number of events, but recovers from errors
+class RecoveringCountdown : public SaxCountdown
+{
+  public:
+    using SaxCountdown::SaxCountdown;
+
+    bool parse_error(std::size_t /*position*/, const std::string& /*last_token*/, const json::exception& /*ex*/) override
+    {
+        return true;
+    }
+};
+
+/// a repaired input: the value it is repaired to, and the number of errors
+struct Repair
+{
+    const char* input;
+    const char* expected;
+    std::size_t errors;
+};
+} // namespace
+
+TEST_CASE("parser error recovery (#3989)")
+{
+    SECTION("repairs")
+    {
+        const std::vector<Repair> repairs =
+        {
+            // a missing separator is inserted
+            {"[1 2]", "[1,2]", 1},
+            {R"({"a":1 "b":2})", R"({"a":1,"b":2})", 1},
+            {R"({"a" 1})", R"({"a":1})", 1},
+            {"[1 tru 2]", "[1,null,2]", 2},
+            {R"({"a" "b": 1})", R"({"a":"b"})", 2},
+
+            // a missing value is null in an object; in an array, a ',' stands
+            // for null, while an array that ends there just ends
+            {R"({"a":})", R"({"a":null})", 1},
+            {R"({"a"})", R"({"a":null})", 1},
+            {R"({"a","b":1})", R"({"a":null,"b":1})", 1},
+            {"[1,,2]", "[1,null,2]", 1},
+            {"[,1]", "[null,1]", 1},
+            {"[1,]", "[1]", 1},
+            {"[1,2,3,]", "[1,2,3]", 1},
+            {R"({"a":1,})", R"({"a":1})", 1},
+
+            // a broken string keeps what can be read
+            {R"(["a\qb"])", R"(["aqb"])", 1},
+            {R"({"na\me":1})", R"({"name":1})", 1},
+            {"[\"\xFF\"]", R"(["\uFFFD"])", 1},
+            {"[\"a\xC3(\"]", R"(["a\uFFFD("])", 1},
+            {"[\"\xE2\x82\"]", R"(["\uFFFD"])", 1},
+            {"[\"\xC3\\\\\", 1]", R"(["\uFFFD\\",1])", 1},
+            {R"(["\u12"])", R"(["\uFFFD"])", 1},
+            {R"(["\u12G4"])", R"(["\uFFFDG4"])", 1},
+            {R"(["\uDC00x"])", R"(["\uFFFDx"])", 1},
+            {R"(["\uD800x"])", R"(["\uFFFDx"])", 1},
+            {R"(["\uD800\u0041"])", R"(["\uFFFDA"])", 1},
+            {R"(["\uD800\uD800\uDC00"])", R"(["\uFFFD\uD800\uDC00"])", 1},
+            {R"(["\uD800\uD800\uD800x"])", R"(["\uFFFD\uFFFD\uFFFDx"])", 1},
+            {
+                R"(["\uD800\"x", 1])", R"(["\uFFFD\"x",1])", 1
+            },
+            {R"(["\uD800\q"])", R"(["\uFFFDq"])", 1},
+            {"[\"a\tb\"]", R"(["a\tb"])", 1},
+            {R"(["a\qb\u0041\x"])", R"(["aqbAx"])", 1},
+
+            // a broken number keeps its longest valid prefix
+            {"[1.]", "[1]", 1},
+            {"[-2.]", "[-2]", 1},
+            {"[1.5e]", "[1.5]", 1},
+            {"[1e+]", "[1]", 1},
+            {"[1.x2, 3]", "[1,3]", 1},
+
+            // what cannot be read at all is null
+            {"[1,NaN,3]", "[1,null,3]", 1},
+            {"[tru]", "[null]", 1},
+            {"[-]", "[null]", 1},
+            {R"({"a":Infinity})", R"({"a":null})", 1},
+
+            // a stray token is dropped
+            {"[:1]", "[1]", 1},
+            {R"(["a":1])", R"(["a",1])", 1},
+            {R"({"a"::1})", R"({"a":1})", 1},
+
+            // a member that cannot be read is skipped
+            {R"({1:2,"b":3})", R"({"b":3})", 1},
+            {R"({"a":1 2})", R"({"a":1})", 1},
+            {R"({,"a":1})", R"({"a":1})", 1},
+            {R"({"a":1,,"b":2})", R"({"a":1,"b":2})", 1},
+            {"{a:1}", "{}", 1},
+            {R"({"a":1 [1,{"b":2}], "c":3})", R"({"a":1,"c":3})", 1},
+            {R"([{1}, "a"])", R"([{},"a"])", 1},
+
+            // a wrong closing bracket closes the innermost container
+            {R"({"a":[1,2}, "b":3})", R"({"a":[1,2],"b":3})", 1},
+            {R"([{"a":1], 2])", R"([{"a":1},2])", 1},
+            {"{]", "{}", 1},
+            {"[}", "[]", 1},
+
+            // the end of the input closes all containers
+            {R"({"a":[1,2)", R"({"a":[1,2]})", 1},
+            {"[", "[]", 1},
+            {"{", "{}", 1},
+            {R"({"a")", R"({"a":null})", 1},
+            {R"({"a":)", R"({"a":null})", 1},
+            {"[1,", "[1]", 1},
+            {"[[[1", "[[[1]]]", 1},
+            {
+                R"(["abc)", R"(["abc"])", 2
+            },
+            {"[1,tr", "[1,null]", 2},
+            {"\"abc", "\"abc\"", 1},
+            {"[\"ab\ncd\"]", R"(["ab",null,"]"])", 4},
+
+            // what comes before the top-level value is skipped
+            {")]}'\n{\"a\":1}", R"({"a":1})", 1},
+            {R"(data: {"a":1})", R"({"a":1})", 1},
+            {"\xEF\xBB[1]", "[1]", 1},
+
+            // what comes after it is an error that ends parsing
+            {R"({"a":1}})", R"({"a":1})", 1},
+            {"[1}]", "[1]", 2},
+            {"[1] [2]", "[1]", 1},
+        };
+
+        for (const auto& repair : repairs)
+        {
+            CAPTURE(repair.input)
+            const auto result = parse_recovering(std::string(repair.input));
+            CHECK(!result.ok);
+            CHECK(result.balanced);
+            CHECK(result.value == json::parse(repair.expected));
+            CHECK(result.errors.size() == repair.errors);
+        }
+    }
+
+    SECTION("number overflow")
+    {
+        const auto result = parse_recovering(std::string("[1e999,-1e999]"));
+        CHECK(!result.ok);
+        CHECK(result.balanced);
+        CHECK(result.errors.size() == 2);
+        CHECK(result.errors[0] == "[json.exception.out_of_range.406] number overflow parsing '1e999'");
+        REQUIRE(result.value.size() == 2);
+        CHECK(result.value[0].is_number_float());
+        CHECK(result.value[0].get<double>() == std::numeric_limits<double>::infinity());
+        CHECK(result.value[1].get<double>() == -std::numeric_limits<double>::infinity());
+
+        // the SAX parser gets the number's text
+        SaxEventLogger logger;
+        logger.recover = true;
+        CHECK(!json::sax_parse("1e999", &logger));
+        CHECK(logger.events == std::vector<std::string>({"parse_error(5)", "number_float(1e999)"}));
+    }
+
+    SECTION("nothing to recover")
+    {
+        for (const std::string s :
+                {
+                    "", "   ", "]", "tru", "NaN", ",:", "/* comment"
+                })
+        {
+            CAPTURE(s)
+            const auto result = parse_recovering(s, true, true);
+            CHECK(!result.ok);
+            CHECK(result.balanced);
+            CHECK(result.events == 0);
+            CHECK(result.value == nullptr);
+            CHECK(result.errors.size() == 1);
+        }
+    }
+
+    SECTION("error messages")
+    {
+        // the first error is reported as without recovery
+        for (const std::string s :
+                {
+                    "[1 2]", R"({"a":1 "b":2})", R"({"a" 1})", R"({"a":})", "[1,]", "[1.]",
+                    R"(["a\qb"])", "[1e999]", "{1:2}", R"({"a":[1,2}})", "[1,", "[1] [2]", "{a:1}"
+                })
+        {
+            CAPTURE(s)
+            const auto result = parse_recovering(s);
+            REQUIRE(!result.errors.empty());
+            json _;
+            CHECK_THROWS_WITH_STD_STR(_ = json::parse(s), result.errors.front());
+        }
+
+        // the token of an error begins where the previous error was
+        const auto result = parse_recovering(std::string("[tru, fals, nul]"));
+        CHECK(result.errors == std::vector<std::string>(
+        {
+            "[json.exception.parse_error.101] parse error at line 1, column 5: syntax error while parsing value - invalid literal; last read: '[tru,'",
+            "[json.exception.parse_error.101] parse error at line 1, column 11: syntax error while parsing value - invalid literal; last read: ', fals,'",
+            "[json.exception.parse_error.101] parse error at line 1, column 16: syntax error while parsing value - invalid literal; last read: ', nul]'"
+        }));
+        CHECK(result.value == json::parse("[null,null,null]"));
+    }
+
+    SECTION("events")
+    {
+        // see #4522
+        SaxEventLogger logger;
+        logger.recover = true;
+        CHECK(!json::sax_parse(R"([{1}, "a"])", &logger));
+        CHECK(logger.events == std::vector<std::string>(
+        {
+            "start_array()", "start_object()", "parse_error(3)", "end_object()", "string(a)", "end_array()"
+        }));
+    }
+
+    SECTION("options")
+    {
+        SECTION("strict")
+        {
+            const auto result = parse_recovering(std::string("[1 2] [3]"), false);
+            CHECK(!result.ok);
+            CHECK(result.value == json::parse("[1,2]"));
+            CHECK(result.errors.size() == 1);
+        }
+
+        SECTION("ignore_trailing_commas")
+        {
+            for (const std::string s :
+                    {
+                        "[1,]", R"({"a":1,})", "[[1,],]"
+                    })
+            {
+                CAPTURE(s)
+                const auto result = parse_recovering(s, true, false, true);
+                CHECK(result.ok);
+                CHECK(result.errors.empty());
+            }
+
+            auto result = parse_recovering(std::string("[1,,]"), true, false, true);
+            CHECK(result.value == json::parse("[1,null]"));
+            CHECK(result.errors.size() == 1);
+
+            result = parse_recovering(std::string(R"({"a":1,,})"), true, false, true);
+            CHECK(result.value == json::parse(R"({"a":1})"));
+            CHECK(result.errors.size() == 1);
+        }
+
+        SECTION("ignore_comments")
+        {
+            auto result = parse_recovering(std::string("[1 /* one */ 2]"), true, true);
+            CHECK(result.value == json::parse("[1,2]"));
+            CHECK(result.errors.size() == 1);
+
+            // a comment that is not closed runs to the end of the input, which
+            // is not reported again
+            result = parse_recovering(std::string("[1, 2 /* unterminated"), true, true);
+            CHECK(result.balanced);
+            CHECK(result.value == json::parse("[1,2]"));
+            CHECK(result.errors.size() == 1);
+
+            // a '/' that does not begin a comment is garbage
+            result = parse_recovering(std::string("[1, /x, 2]"), true, true);
+            CHECK(result.balanced);
+            CHECK(result.value == json::parse("[1,null,2]"));
+            CHECK(result.errors.size() == 1);
+        }
+    }
+
+    SECTION("null bytes")
+    {
+        // a null byte ends the input, unless JSON_STRICT_NUL_HANDLING is set
+        const auto result = parse_recovering(std::string("[1,\0x", 5));
+        CHECK(result.balanced);
+        CHECK(!result.ok);
+#ifdef JSON_TEST_STRICT_NUL_HANDLING_ENABLED
+        CHECK(result.value == json::parse("[1,null]"));
+#else
+        CHECK(result.value == json::parse("[1]"));
+        CHECK(result.errors.size() == 1);
+#endif
+
+        const auto in_string = parse_recovering(std::string("[\"a\0b\"]", 7));
+        CHECK(in_string.balanced);
+#ifdef JSON_TEST_STRICT_NUL_HANDLING_ENABLED
+        CHECK(in_string.value == json::array({std::string("a\0b", 3)}));
+#else
+        CHECK(in_string.value == json::parse(R"(["a"])"));
+#endif
+    }
+
+    SECTION("the SAX parser stops recovering")
+    {
+        json j;
+        RecoveringDomParser sax(j, 2);
+        CHECK(!json::sax_parse("[1 2 3 4 5]", &sax));
+        CHECK(sax.errors.size() == 2);
+
+        // an error at a delimiter that an invalid token consumed is reported
+        // to the SAX parser, too
+        json j2;
+        RecoveringDomParser sax2(j2, 2);
+        CHECK(!json::sax_parse("[tru}, 1]", &sax2));
+        CHECK(sax2.errors.size() == 2);
+    }
+
+    SECTION("an event stops parsing during a repair")
+    {
+        // start_object() and key() are passed, then null() for the missing
+        // value returns false
+        RecoveringCountdown countdown(2);
+        CHECK(!json::sax_parse(R"({"a":})", &countdown));
+
+        // the end of the input: end_array() for the second array returns false
+        RecoveringCountdown countdown2(4);
+        CHECK(!json::sax_parse("[[1", &countdown2));
+    }
+
+    SECTION("input adapters")
+    {
+        // the lexer reads contiguous and streaming input differently, and it
+        // puts back a character that ended an invalid token
+        for (const std::string s :
+                {
+                    "[1 2]", "[tru}, 1]", R"({"a" "b\q", "c":[1.x, 2}})", "[\"\xFF\xC3(\", -, 1e+]", "{a:1,\"b\":2", ")]}' [1]"
+                })
+        {
+            CAPTURE(s)
+            const auto reference = parse_recovering(s);
+            CHECK(reference.balanced);
+
+            const auto from_c_string = parse_recovering(s.c_str());
+            CHECK(from_c_string.value == reference.value);
+            CHECK(from_c_string.errors == reference.errors);
+
+            const std::list<char> l(s.begin(), s.end());
+            json j;
+            RecoveringDomParser sax(j);
+            CHECK(!json::sax_parse(l.begin(), l.end(), &sax));
+            CHECK(j == reference.value);
+            CHECK(sax.errors == reference.errors);
+
+            std::istringstream ss(s);
+            const auto from_stream = parse_recovering(ss);
+            CHECK(from_stream.value == reference.value);
+            CHECK(from_stream.errors == reference.errors);
+        }
+    }
+
+    SECTION("long runs of errors")
+    {
+        // no error may copy all the input read before it
+        const auto closing = parse_recovering("[" + std::string(100000, '}'));
+        CHECK(closing.balanced);
+        CHECK(closing.value == json::array());
+
+        const auto garbage = parse_recovering("[" + std::string(100000, 'x') + "]");
+        CHECK(garbage.balanced);
+        CHECK(garbage.errors.size() == 1);
+
+        const auto commas = parse_recovering("{" + std::string(100000, ',') + "}");
+        CHECK(commas.balanced);
+        CHECK(commas.value == json::object());
+    }
+
+    SECTION("mutations of valid input")
+    {
+        // whatever the input, the events are balanced, every error is reported
+        // at most once, and valid input is parsed as usual
+        const std::vector<std::string> documents =
+        {
+            R"({"name": "value", "list": [1, -2.5, true, null, {"x": [[]]}], "e": "\u00e9"})",
+            R"([{"a": [1, 2, {"b": "c"}]}, [], {}, "\ud83d\ude00", 1e10])",
+            "{\"\xC3\xA9\": \"\xF0\x9F\x98\x80\"}",
+            R"(  {"k" : [ "v" , 0 ] }  )",
+        };
+        // each character that can be inserted, including a null byte
+        const std::string insertions("[]{},:\"x\\\0\xFF", 11);
+
+        std::vector<std::string> inputs;
+        for (const auto& doc : documents)
+        {
+            for (std::size_t i = 0; i <= doc.size(); ++i)
+            {
+                inputs.push_back(doc.substr(0, i));
+                if (i < doc.size())
+                {
+                    inputs.push_back(doc.substr(0, i) + doc.substr(i + 1));
+                }
+                for (const char c : insertions)
+                {
+                    inputs.push_back(doc.substr(0, i) + c + doc.substr(i));
+                }
+            }
+        }
+
+        for (const auto& s : inputs)
+        {
+            CAPTURE(s)
+            const auto result = parse_recovering(s);
+            CHECK(result.balanced);
+            CHECK(result.errors.size() <= s.size() + 1);
+            CHECK(result.events <= (4 * s.size()) + 4);
+            if (json::accept(s))
+            {
+                CHECK(result.ok);
+                CHECK(result.errors.empty());
+                CHECK(result.value == json::parse(s));
+            }
+            else
+            {
+                CHECK(!result.ok);
+                CHECK(!result.errors.empty());
+            }
+        }
+    }
+}

tests/src/unit-diagnostics-optimized.cpp

@@ -0,0 +1,80 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// Regression test for https://github.com/nlohmann/json/issues/5742: with
+// JSON_DIAGNOSTICS, GCC (12 to at least 16) reported a false -Warray-bounds
+// error in the inlined set_parents() at -O3. The type of a new string was set
+// before the string was allocated, so GCC had to assume that operator new
+// could change it again and checked the object branch of set_parents()
+// against the string's allocation. Setting the type after creating the value
+// avoids this. The warning depends on GCC's inlining decisions, so the
+// sections cover two patterns that trigger it on different GCC versions
+// (#4819 and #5742).
+// On GCC, this file is compiled with -O3 -Werror=array-bounds (see
+// tests/CMakeLists.txt), so the test fails to build if the warning returns.
+
+#include "doctest_compatibility.h"
+
+#ifdef JSON_DIAGNOSTICS
+    #undef JSON_DIAGNOSTICS
+#endif
+
+#define JSON_DIAGNOSTICS 1
+
+#include <nlohmann/json.hpp>
+using nlohmann::json;
+
+#include <algorithm>
+#include <iterator>
+#include <utility>
+#include <vector>
+
+namespace
+{
+enum class diag_color
+{
+    red,
+    green,
+    blue
+};
+
+void to_json(json& j, const diag_color& c)
+{
+    static const std::pair<diag_color, json> m[] = // NOLINT(cppcoreguidelines-avoid-c-arrays,hicpp-avoid-c-arrays,modernize-avoid-c-arrays)
+    {
+        {diag_color::red, "r"},
+        {diag_color::green, "g"},
+        {diag_color::blue, "b"},
+    };
+    const auto* it = std::find_if(std::begin(m), std::end(m), [c](const std::pair<diag_color, json>& p)
+    {
+        return p.first == c;
+    });
+    j = it->second;
+}
+} // namespace
+
+TEST_CASE("diagnostics with optimization")
+{
+    SECTION("issue #4819 - object in vector")
+    {
+        std::vector<json> jsons{};
+        jsons.emplace_back(json({{"key", "value"}}));
+        CHECK(jsons.back()["key"] == "value");
+    }
+
+    SECTION("issue #5742 - string values from a static table")
+    {
+        json j = json::array();
+        j.push_back(diag_color::red);
+        j.push_back(diag_color::green);
+        j.push_back(diag_color::blue);
+        CHECK(j.dump() == R"(["r","g","b"])");
+        CHECK_THROWS_WITH_AS(j[1].get<int>(), "[json.exception.type_error.302] (/1) type must be number, but is string", json::type_error);
+    }
+}

tests/src/unit-msgpack-cpp17.cpp

@@ -0,0 +1,99 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.12.0
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+// This file contains the C++17-only part of unit-msgpack.cpp (std::byte
+// input). It is kept in a separate translation unit so the (much larger)
+// unit-msgpack.cpp does not need to be compiled and run a second time just
+// for this one test case (#5418).
+
+#include "doctest_compatibility.h"
+
+#include <nlohmann/json.hpp>
+using nlohmann::json;
+
+#ifdef JSON_HAS_CPP_17
+#include <cstddef>
+#include <vector>
+
+// Test suite for verifying MessagePack handling with std::byte input
+TEST_CASE("MessagePack with std::byte")
+{
+
+    SECTION("std::byte compatibility")
+    {
+        SECTION("vector roundtrip")
+        {
+            json original =
+            {
+                {"name", "test"},
+                {"value", 42},
+                {"array", {1, 2, 3}}
+            };
+
+            std::vector<uint8_t> temp = json::to_msgpack(original);
+            // Convert the uint8_t vector to std::byte vector
+            std::vector<std::byte> msgpack_data(temp.size());
+            for (size_t i = 0; i < temp.size(); ++i)
+            {
+                msgpack_data[i] = std::byte(temp[i]);
+            }
+            // Deserialize from std::byte vector back to JSON
+            json from_bytes;
+            CHECK_NOTHROW(from_bytes = json::from_msgpack(msgpack_data));
+
+            CHECK(from_bytes == original);
+        }
+
+        SECTION("empty vector")
+        {
+            const std::vector<std::byte> empty_data;
+            CHECK_THROWS_WITH_AS([&]()
+            {
+                [[maybe_unused]] auto result = json::from_msgpack(empty_data);
+                return true;
+            }
+            (),
+            "[json.exception.parse_error.110] parse error at byte 1: syntax error while parsing MessagePack value: unexpected end of input",
+            json::parse_error&);
+        }
+
+        SECTION("comparison with workaround")
+        {
+            json original =
+            {
+                {"string", "hello"},
+                {"integer", 42},
+                {"float", 3.14},
+                {"boolean", true},
+                {"null", nullptr},
+                {"array", {1, 2, 3}},
+                {"object", {{"key", "value"}}}
+            };
+
+            std::vector<uint8_t> temp = json::to_msgpack(original);
+
+            std::vector<std::byte> msgpack_data(temp.size());
+            for (size_t i = 0; i < temp.size(); ++i)
+            {
+                msgpack_data[i] = std::byte(temp[i]);
+            }
+            // Attempt direct deserialization using std::byte input
+            const json direct_result = json::from_msgpack(msgpack_data);
+
+            // Test the workaround approach: reinterpret as unsigned char* and use iterator range
+            const auto* const char_start = reinterpret_cast<unsigned char const*>(msgpack_data.data());
+            const auto* const char_end = char_start + msgpack_data.size();
+            json workaround_result = json::from_msgpack(char_start, char_end);
+
+            // Verify that the final deserialized JSON matches the original JSON
+            CHECK(direct_result == workaround_result);
+            CHECK(direct_result == original);
+        }
+    }
+}
+#endif

tests/src/unit-msgpack.cpp

@@ -2173,85 +2173,6 @@ TEST_CASE("MessagePack roundtrips" * doctest::skip())
     }
 }
 
-#ifdef JSON_HAS_CPP_17
-// Test suite for verifying MessagePack handling with std::byte input
-TEST_CASE("MessagePack with std::byte")
-{
-
-    SECTION("std::byte compatibility")
-    {
-        SECTION("vector roundtrip")
-        {
-            json original =
-            {
-                {"name", "test"},
-                {"value", 42},
-                {"array", {1, 2, 3}}
-            };
-
-            std::vector<uint8_t> temp = json::to_msgpack(original);
-            // Convert the uint8_t vector to std::byte vector
-            std::vector<std::byte> msgpack_data(temp.size());
-            for (size_t i = 0; i < temp.size(); ++i)
-            {
-                msgpack_data[i] = std::byte(temp[i]);
-            }
-            // Deserialize from std::byte vector back to JSON
-            json from_bytes;
-            CHECK_NOTHROW(from_bytes = json::from_msgpack(msgpack_data));
-
-            CHECK(from_bytes == original);
-        }
-
-        SECTION("empty vector")
-        {
-            const std::vector<std::byte> empty_data;
-            CHECK_THROWS_WITH_AS([&]()
-            {
-                [[maybe_unused]] auto result = json::from_msgpack(empty_data);
-                return true;
-            }
-            (),
-            "[json.exception.parse_error.110] parse error at byte 1: syntax error while parsing MessagePack value: unexpected end of input",
-            json::parse_error&);
-        }
-
-        SECTION("comparison with workaround")
-        {
-            json original =
-            {
-                {"string", "hello"},
-                {"integer", 42},
-                {"float", 3.14},
-                {"boolean", true},
-                {"null", nullptr},
-                {"array", {1, 2, 3}},
-                {"object", {{"key", "value"}}}
-            };
-
-            std::vector<uint8_t> temp = json::to_msgpack(original);
-
-            std::vector<std::byte> msgpack_data(temp.size());
-            for (size_t i = 0; i < temp.size(); ++i)
-            {
-                msgpack_data[i] = std::byte(temp[i]);
-            }
-            // Attempt direct deserialization using std::byte input
-            const json direct_result = json::from_msgpack(msgpack_data);
-
-            // Test the workaround approach: reinterpret as unsigned char* and use iterator range
-            const auto* const char_start = reinterpret_cast<unsigned char const*>(msgpack_data.data());
-            const auto* const char_end = char_start + msgpack_data.size();
-            json workaround_result = json::from_msgpack(char_start, char_end);
-
-            // Verify that the final deserialized JSON matches the original JSON
-            CHECK(direct_result == workaround_result);
-            CHECK(direct_result == original);
-        }
-    }
-}
-#endif
-
 // the fake sizes below do not fit into a 32-bit std::size_t
 // with clang and libstdc++ 10, the std::filesystem::path conversion that
 // C++17 builds consider for every string type is ambiguous for a class
@@ -2561,3 +2482,43 @@ TEST_CASE("MessagePack numbers use the active union member (see #5644)")
         CHECK(json::from_msgpack(result) == j);
     }
 }
+
+TEST_CASE("MessagePack large strings and binaries (chunked reader)")
+{
+    // get_msgpack_string()/get_msgpack_binary() both read through get_binary(),
+    // which reads in bounded chunks (binary_reader.hpp, chunk_size == 4096);
+    // make sure roundtripping is correct for lengths around and beyond that
+    // chunk size, for both vector (iterator) and pointer inputs.
+    for (const std::size_t len :
+            {
+                std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
+                std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
+            })
+    {
+        CAPTURE(len)
+
+        // string
+        const json j_string = std::string(len, 'x');
+        const std::vector<std::uint8_t> v_string = json::to_msgpack(j_string);
+        CHECK(json::from_msgpack(v_string) == j_string);
+        // pointer input exercises the std::memcpy fast path
+        CHECK(json::from_msgpack(reinterpret_cast<const char*>(v_string.data()),
+                                 reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
+
+        // binary
+        const json j_binary = json::binary(std::vector<std::uint8_t>(len, 0xCD));
+        const std::vector<std::uint8_t> v_binary = json::to_msgpack(j_binary);
+        CHECK(json::from_msgpack(v_binary) == j_binary);
+        CHECK(json::from_msgpack(reinterpret_cast<const char*>(v_binary.data()),
+                                 reinterpret_cast<const char*>(v_binary.data()) + v_binary.size()) == j_binary);
+
+        // a truncated payload must still be reported as an error
+        if (len > 16)
+        {
+            std::vector<std::uint8_t> truncated = v_string;
+            truncated.resize(truncated.size() - 8);
+            json _;
+            CHECK_THROWS_AS(_ = json::from_msgpack(truncated), json::parse_error);
+        }
+    }
+}

tests/src/unit-regression2.cpp

@@ -1020,6 +1020,599 @@ TEST_CASE("regression test - excessive binary container size honors allow_except
     CHECK(json::from_cbor(std::vector<std::uint8_t> {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded());
 }
 
+namespace
+{
+/// builds a value from SAX events, asks the parser to recover from its first
+/// 100 errors, and checks that the events are balanced (see #3989)
+template<typename BasicJsonType>
+class BasicRecoveringParser
+{
+  public:
+    explicit BasicRecoveringParser(BasicJsonType& j)
+        : dom(j, false)
+    {}
+
+    bool null()
+    {
+        value();
+        return dom.null();
+    }
+
+    bool boolean(bool val)
+    {
+        value();
+        return dom.boolean(val);
+    }
+
+    bool number_integer(typename BasicJsonType::number_integer_t val)
+    {
+        value();
+        return dom.number_integer(val);
+    }
+
+    bool number_unsigned(typename BasicJsonType::number_unsigned_t val)
+    {
+        value();
+        return dom.number_unsigned(val);
+    }
+
+    bool number_float(typename BasicJsonType::number_float_t val, const std::string& s)
+    {
+        value();
+        return dom.number_float(val, s);
+    }
+
+    bool string(std::string& val)
+    {
+        value();
+        return dom.string(val);
+    }
+
+    bool binary(typename BasicJsonType::binary_t& val)
+    {
+        value();
+        return dom.binary(val);
+    }
+
+    bool start_object(std::size_t elements)
+    {
+        value();
+        stack.push_back('o');
+        return dom.start_object(elements);
+    }
+
+    bool key(std::string& val)
+    {
+        if (stack.empty() || stack.back() != 'o')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.back() = 'v';
+        return dom.key(val);
+    }
+
+    bool end_object()
+    {
+        if (stack.empty() || stack.back() != 'o')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.pop_back();
+        return dom.end_object();
+    }
+
+    bool start_array(std::size_t elements)
+    {
+        value();
+        stack.push_back('a');
+        return dom.start_array(elements);
+    }
+
+    bool end_array()
+    {
+        if (stack.empty() || stack.back() != 'a')
+        {
+            well_formed = false;
+            return false;
+        }
+        stack.pop_back();
+        return dom.end_array();
+    }
+
+    bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& ex)
+    {
+        messages.emplace_back(ex.what());
+        // a limit, so that a reader that does not stop fails the test
+        // instead of making it hang
+        return ++errors < 100;
+    }
+
+    /// whether the events were balanced and every key was followed by a value
+    bool balanced() const
+    {
+        return well_formed && stack.empty();
+    }
+
+    /// builds the value
+    nlohmann::detail::json_sax_dom_parser<BasicJsonType> dom;
+    std::size_t errors = 0;
+    std::vector<std::string> messages {}; // NOLINT(readability-redundant-member-init)
+    std::vector<char> stack {}; // NOLINT(readability-redundant-member-init)
+    bool well_formed = true;
+
+  private:
+    void value()
+    {
+        if (!stack.empty())
+        {
+            if (stack.back() == 'v')
+            {
+                stack.back() = 'o';
+            }
+            else if (stack.back() == 'o')
+            {
+                well_formed = false;
+            }
+        }
+    }
+
+};
+
+using RecoveringParser = BasicRecoveringParser<json>;
+
+struct BinaryParseResult
+{
+    json value;
+    std::size_t errors;
+    std::vector<std::string> messages;
+    bool ok;
+    bool balanced;
+};
+
+BinaryParseResult parse_binary_recovering(const std::vector<std::uint8_t>& input, const json::input_format_t format)
+{
+    json j;
+    RecoveringParser sax(j);
+    const bool ok = json::sax_parse(input, &sax, format);
+    return {j, sax.errors, sax.messages, ok, sax.balanced()};
+}
+
+#if !defined(JSON_NOEXCEPTION)
+/// the message of the exception that reading @a input into a JSON value
+/// throws, or an empty string if reading succeeds
+std::string binary_error_message(const std::vector<std::uint8_t>& input, const json::input_format_t format)
+{
+    try
+    {
+        json _;
+        switch (format)
+        {
+            case json::input_format_t::cbor:
+                _ = json::from_cbor(input);
+                break;
+            case json::input_format_t::msgpack:
+                _ = json::from_msgpack(input);
+                break;
+            case json::input_format_t::ubjson:
+                _ = json::from_ubjson(input);
+                break;
+            case json::input_format_t::bjdata:
+                _ = json::from_bjdata(input);
+                break;
+            case json::input_format_t::bson:
+                _ = json::from_bson(input);
+                break;
+            case json::input_format_t::bon8:
+                _ = json::from_bon8(input);
+                break;
+            case json::input_format_t::json:
+            default:
+                break;
+        }
+    }
+    catch (const json::exception& e)
+    {
+        return e.what();
+    }
+    return "";
+}
+#endif
+
+/// a BSON element: its type, its name, and its value
+std::vector<std::uint8_t> bson_element(const std::uint8_t type, const std::string& name, const std::vector<std::uint8_t>& value)
+{
+    std::vector<std::uint8_t> result = {type};
+    result.insert(result.end(), name.begin(), name.end());
+    result.push_back(0x00);
+    result.insert(result.end(), value.begin(), value.end());
+    return result;
+}
+
+/// a BSON document of the given elements; @a size_offset is added to the
+/// size it declares
+std::vector<std::uint8_t> bson_document(const std::vector<std::vector<std::uint8_t>>& elements, const int size_offset = 0)
+{
+    std::vector<std::uint8_t> body;
+    for (const auto& element : elements)
+    {
+        body.insert(body.end(), element.begin(), element.end());
+    }
+    const auto size = static_cast<std::uint32_t>(static_cast<int>(body.size()) + 5 + size_offset);
+    std::vector<std::uint8_t> result = {static_cast<std::uint8_t>(size & 0xFFu), static_cast<std::uint8_t>((size >> 8u) & 0xFFu),
+                                        static_cast<std::uint8_t>((size >> 16u) & 0xFFu), static_cast<std::uint8_t>((size >> 24u) & 0xFFu)
+                                       };
+    result.insert(result.end(), body.begin(), body.end());
+    result.push_back(0x00);
+    return result;
+}
+
+/// a BSON int32 value
+std::vector<std::uint8_t> bson_int32(const std::int32_t value)
+{
+    const auto u = static_cast<std::uint32_t>(value);
+    return {static_cast<std::uint8_t>(u & 0xFFu), static_cast<std::uint8_t>((u >> 8u) & 0xFFu),
+            static_cast<std::uint8_t>((u >> 16u) & 0xFFu), static_cast<std::uint8_t>((u >> 24u) & 0xFFu)};
+}
+
+/// a BSON string value, whose length is @a length_offset off
+std::vector<std::uint8_t> bson_string(const std::string& value, const std::int32_t length_offset = 0)
+{
+    auto result = bson_int32(static_cast<std::int32_t>(value.size() + 1) + length_offset);
+    result.insert(result.end(), value.begin(), value.end());
+    result.push_back(0x00);
+    return result;
+}
+
+/// @a count bytes of value 0xAB
+std::vector<std::uint8_t> bytes(const std::size_t count)
+{
+    return std::vector<std::uint8_t>(count, 0xAB);
+}
+
+template<typename... Parts>
+std::vector<std::uint8_t> concatenated(const std::vector<std::uint8_t>& first, const Parts& ... rest)
+{
+    std::vector<std::uint8_t> result = first;
+    for (const auto& part : std::initializer_list<std::vector<std::uint8_t>> {rest...})
+    {
+        result.insert(result.end(), part.begin(), part.end());
+    }
+    return result;
+}
+
+/// U+FFFD REPLACEMENT CHARACTER
+std::string replacement_character()
+{
+    return "\xEF\xBF\xBD";
+}
+}  // namespace
+
+TEST_CASE("regression test - #3989 SAX parse_error() returning true")
+{
+    SECTION("binary formats complete what was read before the input ends")
+    {
+        const json j = {{"a", {1, -2, {{"b", "c"}}, json::array()}}, {"d", {{"e", nullptr}, {"f", true}}}, {"g", 1.5}, {"h", json::binary({1, 2, 3})}};
+
+        const std::vector<std::pair<json::input_format_t, std::vector<std::uint8_t>>> encodings =
+        {
+            {json::input_format_t::cbor, json::to_cbor(j)},
+            {json::input_format_t::msgpack, json::to_msgpack(j)},
+            {json::input_format_t::ubjson, json::to_ubjson(j)},
+            {json::input_format_t::ubjson, json::to_ubjson(j, true, true)},
+            {json::input_format_t::bjdata, json::to_bjdata(j)},
+            {json::input_format_t::bjdata, json::to_bjdata(j, true, true)},
+            {json::input_format_t::bson, json::to_bson(j)},
+            {json::input_format_t::bon8, json::to_bon8(j)},
+        };
+
+        for (const auto& encoding : encodings)
+        {
+            const auto format = encoding.first;
+            const auto& bytes = encoding.second;
+            CAPTURE(format)
+
+            // every prefix is truncated input
+            for (std::size_t length = 0; length < bytes.size(); ++length)
+            {
+                CAPTURE(length)
+                const auto result = parse_binary_recovering(std::vector<std::uint8_t>(bytes.begin(), bytes.begin() + static_cast<std::ptrdiff_t>(length)), format);
+                CHECK(!result.ok);
+                CHECK(result.errors == 1);
+                CHECK(result.balanced);
+            }
+
+            // the complete input is read as usual (binary values do not
+            // round-trip through every format, so compare with a plain parse)
+            json expected;
+            nlohmann::detail::json_sax_dom_parser<json> dom(expected);
+            CHECK(json::sax_parse(bytes, &dom, format));
+            const auto complete = parse_binary_recovering(bytes, format);
+            CHECK(complete.ok);
+            CHECK(complete.errors == 0);
+            CHECK(complete.value == expected);
+
+            // a byte after the value
+            auto trailing_bytes = bytes;
+            trailing_bytes.push_back(0x01);
+            const auto trailing = parse_binary_recovering(trailing_bytes, format);
+            CHECK(!trailing.ok);
+            CHECK(trailing.errors == 1);
+            CHECK(trailing.value == expected);
+        }
+    }
+
+    SECTION("containers without an end")
+    {
+        // these made the readers loop, or read on, after the error
+        const auto cbor_array = parse_binary_recovering({0x9F}, json::input_format_t::cbor);
+        CHECK(cbor_array.errors == 1);
+        CHECK(cbor_array.value == json::array());
+
+        const auto cbor_map = parse_binary_recovering({0xBF, 0x61, 'a'}, json::input_format_t::cbor);
+        CHECK(cbor_map.errors == 1);
+        CHECK(cbor_map.value == json({{"a", nullptr}}));
+
+        const auto msgpack_array = parse_binary_recovering({0xDD, 0xFF, 0xFF, 0xFF, 0xFF}, json::input_format_t::msgpack);
+        CHECK(msgpack_array.errors == 1);
+        CHECK(msgpack_array.value == json::array());
+
+        const auto msgpack_map = parse_binary_recovering({0x81, 0xA1, 'a', 0x92, 0x01}, json::input_format_t::msgpack);
+        CHECK(msgpack_map.errors == 1);
+        CHECK(msgpack_map.value == json({{"a", {1}}}));
+    }
+
+    SECTION("BJData ndarray")
+    {
+        // a 2x3 int8 array with two of its six elements; the annotated array
+        // format opens an object and two arrays of its own
+        const auto result = parse_binary_recovering({'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2}, json::input_format_t::bjdata);
+        CHECK(result.errors == 1);
+        CHECK(result.balanced);
+        CHECK(result.value == json({{"_ArrayType_", "int8"}, {"_ArraySize_", {2, 3}}, {"_ArrayData_", {1, 2}}}));
+    }
+
+    SECTION("binary formats repair items whose end is known")
+    {
+        struct Repair
+        {
+            json::input_format_t format;
+            std::vector<std::uint8_t> input;
+            json expected;
+            std::size_t errors;
+        };
+
+        const std::vector<Repair> repairs =
+        {
+            // CBOR: tags are ignored (here tag 1 and the self-describe tag 55799)
+            {json::input_format_t::cbor, {0x82, 0xC1, 0x05, 0xD9, 0xD9, 0xF7, 0x06}, {5, 6}, 2},
+            // CBOR: undefined and other simple values become null
+            {json::input_format_t::cbor, {0x84, 0xF7, 0xE0, 0xF8, 0x20, 0x01}, {nullptr, nullptr, nullptr, 1}, 3},
+            // CBOR: members whose key is not a string are skipped, whatever their key and value
+            {json::input_format_t::cbor, {0xA4, 0x01, 0x02, 0x82, 0x01, 0x02, 0xA1, 0x61, 'x', 0x9F, 0xFF, 0xC1, 0x01, 0x5F, 0x41, 0x00, 0xFF, 0x61, 'a', 0x03}, {{"a", 3}}, 3},
+            {json::input_format_t::cbor, {0xBF, 0xF5, 0xBF, 0x61, 'x', 0x7F, 0x61, 'y', 0xFF, 0xFF, 0x61, 'a', 0x03, 0xFF}, {{"a", 3}}, 1},
+            // MessagePack: members whose key is not a string are skipped
+            {json::input_format_t::msgpack, {0x84, 0x01, 0x02, 0x81, 0xA1, 'x', 0x01, 0x92, 0x01, 0x02, 0xD4, 0x01, 0x02, 0xC0, 0xA1, 'a', 0x04}, {{"a", 4}}, 3},
+            // UBJSON: a char that is not ASCII becomes U+FFFD
+            {json::input_format_t::ubjson, {'[', 'C', 0x80, 'C', 'A', ']'}, {replacement_character(), "A"}, 1},
+            // UBJSON: the longest beginning of a high-precision number is kept
+            {json::input_format_t::ubjson, {'[', 'H', 'i', 5, '1', '2', 'a', 'b', 'c', 'H', 'i', 2, '1', '.', 'H', 'i', 3, 'a', 'b', 'c', 'H', 'i', 3, '4', '.', '5', ']'}, {12, 1, nullptr, 4.5}, 3},
+            // BJData, too
+            {json::input_format_t::bjdata, {'[', 'C', 0xFF, 'H', 'i', 2, '-', '1', 'H', 'i', 2, '-', 'x', ']'}, {replacement_character(), -1, nullptr}, 2},
+            // a NUL ends a high-precision number, as it ends JSON text
+            {json::input_format_t::ubjson, {'[', 'H', 'i', 4, '1', '2', 0, '9', 'H', 'i', 2, 0, '1', 'i', 3, ']'}, {12, nullptr, 3}, 2},
+            // BON8: members whose key is not a string are skipped
+            {json::input_format_t::bon8, {0x89, 0x91, 0x92, 0xC9, 0x40, 0x82, 0x91, 0x92, 0x61, 0x93}, {{"a", 3}}, 2},
+            {json::input_format_t::bon8, {0x8B, 0x91, 0x85, 0x91, 0xFE, 0xFA, 0x8B, 'x', 0x91, 0xFE, 0x61, 0x93, 0xFE}, {{"a", 3}}, 2},
+            // BSON: elements of types the library does not read become null
+            {
+                json::input_format_t::bson, bson_document(
+                {
+                    bson_element(0x07, "_id", bytes(12)),                         // ObjectId
+                    bson_element(0x09, "date", bytes(8)),                         // UTC datetime
+                    bson_element(0x13, "decimal", bytes(16)),                     // 128-bit decimal
+                    bson_element(0x0B, "regex", {'a', '+', 0, 'i', 0}),           // regular expression
+                    bson_element(0x0D, "code", bson_string("f()")),               // JavaScript code
+                    bson_element(0x0E, "symbol", bson_string("s")),               // symbol
+                    bson_element(0x0C, "pointer", concatenated(bson_string("c"), bytes(12))), // DBPointer
+                    bson_element(0x0F, "scope", concatenated(bson_int32(15), bson_string("g"), bson_document({}))), // code with scope
+                    bson_element(0x06, "undefined", {}),                          // undefined
+                    bson_element(0xFF, "min", {}),                                // min key
+                    bson_element(0x7F, "max", {}),                                // max key
+                    bson_element(0x10, "z", bson_int32(7)),
+                }),
+                {{"_id", nullptr}, {"date", nullptr}, {"decimal", nullptr}, {"regex", nullptr}, {"code", nullptr}, {"symbol", nullptr}, {"pointer", nullptr}, {"scope", nullptr}, {"undefined", nullptr}, {"min", nullptr}, {"max", nullptr}, {"z", 7}},
+                11
+            },
+            // BSON: an element of an unknown type becomes null, and the rest of its document is skipped
+            {
+                json::input_format_t::bson, bson_document(
+                {
+                    bson_element(0x03, "inner", bson_document({bson_element(0x10, "a", bson_int32(1)), bson_element(0x42, "x", bytes(3)), bson_element(0x10, "b", bson_int32(2))})),
+                    bson_element(0x04, "array", bson_document({bson_element(0x10, "0", bson_int32(1)), bson_element(0x42, "1", bytes(3))})),
+                    bson_element(0x10, "after", bson_int32(3)),
+                }),
+                {{"inner", {{"a", 1}, {"x", nullptr}}}, {"array", {1, nullptr}}, {"after", 3}},
+                2
+            },
+            // BSON: so does a string or byte array whose length cannot be right
+            {
+                json::input_format_t::bson, bson_document(
+                {
+                    bson_element(0x03, "inner", bson_document({bson_element(0x02, "s", bson_string("abc", -10)), bson_element(0x10, "b", bson_int32(2))})),
+                    bson_element(0x03, "bin", bson_document({bson_element(0x05, "b", concatenated(bson_int32(-1), bytes(1))), bson_element(0x10, "b", bson_int32(2))})),
+                    bson_element(0x10, "after", bson_int32(3)),
+                }),
+                {{"inner", {{"s", nullptr}}}, {"bin", {{"b", nullptr}}}, {"after", 3}},
+                2
+            },
+            // BSON: a string without its terminator, and a document whose size does not match, are kept
+            {
+                json::input_format_t::bson, bson_document(
+                {
+                    bson_element(0x02, "s", {2, 0, 0, 0, 'a', 'X'}),
+                    bson_element(0x03, "inner", bson_document({bson_element(0x10, "a", bson_int32(1))}, 1)),
+                }),
+                {{"s", "a"}, {"inner", {{"a", 1}}}},
+                2
+            },
+        };
+
+        for (const auto& repair : repairs)
+        {
+            CAPTURE(repair.format)
+            CAPTURE(repair.input)
+            const auto result = parse_binary_recovering(repair.input, repair.format);
+            CHECK(!result.ok);
+            CHECK(result.balanced);
+            CHECK(result.errors == repair.errors);
+            CHECK(result.value == repair.expected);
+            REQUIRE(!result.messages.empty());
+#if !defined(JSON_NOEXCEPTION)
+            // the first error is the one reported without recovering; under
+            // JSON_NOEXCEPTION, reading without recovering aborts instead of
+            // throwing, so there is no message to compare with
+            CHECK(result.messages.front() == binary_error_message(repair.input, repair.format));
+#endif
+        }
+    }
+
+    SECTION("binary formats repair numbers that are out of range")
+    {
+        // CBOR: a double too large for a float number_float_t
+        using float_json = nlohmann::basic_json<std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, float>;
+        float_json cbor;
+        BasicRecoveringParser<float_json> sax(cbor);
+        const std::vector<std::uint8_t> cbor_input = {0x82, 0xFB, 0x7E, 0x37, 0xE4, 0x3C, 0x88, 0x00, 0x75, 0x9C, 0x01}; // [1e300, 1]
+        CHECK(!float_json::sax_parse(cbor_input, &sax, float_json::input_format_t::cbor));
+        CHECK(sax.errors == 1);
+        CHECK(sax.messages.front() == "[json.exception.out_of_range.406] syntax error while parsing CBOR value: number overflow");
+        REQUIRE(cbor.size() == 2);
+        CHECK(std::isinf(cbor[0].get<float>()));
+        CHECK(cbor[1] == 1);
+
+        // UBJSON: a high-precision number too large for number_float_t
+        const auto ubjson = parse_binary_recovering({'H', 'i', 5, '1', 'e', '9', '9', '9'}, json::input_format_t::ubjson);
+        CHECK(ubjson.errors == 1);
+        CHECK(ubjson.value.is_number_float());
+        CHECK(std::isinf(ubjson.value.get<double>()));
+    }
+
+    SECTION("binary formats stop where the end of an item is not known")
+    {
+        // a byte that begins no item
+        const auto cbor = parse_binary_recovering({0x82, 0x01, 0x1C, 0x02}, json::input_format_t::cbor);
+        CHECK(cbor.errors == 1);
+        CHECK(cbor.value == json({1}));
+
+        // a key that is no item: the unused MessagePack byte, a CBOR break
+        // in a map of known size, and the end of a BON8 container
+        const auto msgpack = parse_binary_recovering({0x82, 0xA1, 'a', 0x01, 0xC1, 0x02}, json::input_format_t::msgpack);
+        CHECK(msgpack.errors == 1);
+        CHECK(msgpack.value == json({{"a", 1}}));
+        const auto cbor_break = parse_binary_recovering({0xA2, 0x61, 'a', 0x01, 0xFF, 0x02}, json::input_format_t::cbor);
+        CHECK(cbor_break.errors == 1);
+        CHECK(cbor_break.value == json({{"a", 1}}));
+        const auto bon8 = parse_binary_recovering({0x88, 0x61, 0x91, 0xFE}, json::input_format_t::bon8);
+        CHECK(bon8.errors == 1);
+        CHECK(bon8.value == json({{"a", 1}}));
+
+        // an indefinite-length string inside an indefinite-length string
+        const auto nested = parse_binary_recovering({0x82, 0x01, 0x7F, 0x7F, 0x61, 'a', 0xFF, 0xFF}, json::input_format_t::cbor);
+        CHECK(nested.errors == 1);
+        CHECK(nested.value == json({1}));
+
+        // a skipped member that the input ends in
+        const auto truncated = parse_binary_recovering({0xA2, 0x01, 0x82, 0x01}, json::input_format_t::cbor);
+        CHECK(truncated.errors == 2);
+        CHECK(truncated.balanced);
+        CHECK(truncated.value == json::object());
+
+        // a BSON element of an unknown type in a document whose size cannot be right
+        const auto bson = parse_binary_recovering(bson_document({bson_element(0x10, "a", bson_int32(1)), bson_element(0x42, "x", bytes(3))}, -10), json::input_format_t::bson);
+        CHECK(bson.errors == 1);
+        CHECK(bson.value == json({{"a", 1}, {"x", nullptr}}));
+    }
+
+    SECTION("changed bytes in binary input")
+    {
+        const json j = {{"a", {1, -2, {{"b", "c"}}, json::array()}}, {"d", {{"e", nullptr}, {"f", true}}}, {"g", 1.5}, {"h", json::binary({1, 2, 3})}, {"i", "\xC3\xA4"}};
+
+        const std::vector<std::pair<json::input_format_t, std::vector<std::uint8_t>>> encodings =
+        {
+            {json::input_format_t::cbor, json::to_cbor(j)},
+            {json::input_format_t::msgpack, json::to_msgpack(j)},
+            {json::input_format_t::ubjson, json::to_ubjson(j)},
+            {json::input_format_t::ubjson, json::to_ubjson(j, true, true)},
+            {json::input_format_t::bjdata, json::to_bjdata(j)},
+            {json::input_format_t::bjdata, json::to_bjdata(j, true, true)},
+            {json::input_format_t::bson, json::to_bson(j)},
+            {json::input_format_t::bon8, json::to_bon8(j)},
+        };
+        const std::vector<std::uint8_t> replacements = {0x00, 0x01, 0x7F, 0x80, 0xC1, 0xD9, 0xE0, 0xF7, 0xFE, 0xFF};
+
+        for (const auto& encoding : encodings)
+        {
+            const auto format = encoding.first;
+            const auto& original = encoding.second;
+            CAPTURE(format)
+
+            std::vector<std::vector<std::uint8_t>> inputs;
+            for (std::size_t position = 0; position < original.size(); ++position)
+            {
+                for (const auto replacement : replacements)
+                {
+                    auto changed = original;
+                    changed[position] = replacement;
+                    inputs.push_back(changed);
+                }
+                auto removed = original;
+                removed.erase(removed.begin() + static_cast<std::ptrdiff_t>(position));
+                inputs.push_back(removed);
+            }
+
+            for (const auto& input : inputs)
+            {
+                CAPTURE(input)
+                const auto result = parse_binary_recovering(input, format);
+                CHECK(result.balanced);
+                CHECK(result.errors <= input.size() + 1);
+#if !defined(JSON_NOEXCEPTION)
+                // an error is reported exactly if reading into a JSON value
+                // fails, and the first one is the same (under JSON_NOEXCEPTION,
+                // that reading aborts instead of throwing)
+                const auto message = binary_error_message(input, format);
+                CHECK(result.ok == message.empty());
+                if (!result.ok && result.errors < 100)
+                {
+                    CHECK(result.messages.front() == message);
+                }
+#endif
+            }
+        }
+    }
+
+    SECTION("JSON text")
+    {
+        // the parser stopped, but reported success
+        json j;
+        RecoveringParser sax(j);
+        CHECK(!json::sax_parse("[1,2,3,]", &sax));
+        CHECK(sax.errors == 1);
+        CHECK(j == json({1, 2, 3}));
+    }
+
+    SECTION("the SAX parsers of the library stop")
+    {
+        json _;
+        CHECK(json::from_cbor(std::vector<std::uint8_t> {0x9F}, true, false).is_discarded());
+        CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t> {0x9F}), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing CBOR value: unexpected end of input", json::parse_error&);
+        CHECK(json::parse("[1,2,3,]", nullptr, false).is_discarded());
+        CHECK(!json::accept("[1,2,3,]"));
+    }
+}
+
 #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION)
 TEST_CASE("regression test #5135 - destructor never allocates, even under memory pressure")
 {

tests/src/unit-regression3.cpp

@@ -533,7 +533,7 @@ TEST_CASE("regression tests 3")
     }
 #endif
 
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
     SECTION("issue #4916 - constructing array from C++20 ranges view does not work")
     {
         std::vector<int> nums{1, 2, 37, 42, 21};
@@ -548,7 +548,7 @@ TEST_CASE("regression tests 3")
 #endif
 
     // owning_view is not available in libstdc++ < 12
-#if JSON_HAS_RANGES && !defined(__MINGW32__) && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12)
+#if JSON_HAS_RANGE_VIEW_CONVERSION && !(defined(__GLIBCXX__) && _GLIBCXX_RELEASE < 12)
     SECTION("issue #4916 - constructing array from prvalue C++20 ranges view (owning_view)")
     {
         json const j(std::vector<int> {1, 2, 37, 42, 21} | std::views::filter([](int i)
@@ -560,7 +560,7 @@ TEST_CASE("regression tests 3")
     }
 #endif
 
-#if JSON_HAS_RANGES && !defined(__MINGW32__)
+#if JSON_HAS_RANGE_VIEW_CONVERSION
     SECTION("issue #4916 - constructing array from C++20 transform view (prvalue elements)")
     {
         std::vector<int> nums{1, 2, 3};
@@ -862,6 +862,46 @@ TEST_CASE("issue #5338 - truncated CBOR tagged binary subtype is rejected")
     }
 }
 
+TEST_CASE("issue #5676 - SAX parsing of CBOR tags")
+{
+    const json expected = json::binary({1, 2, 3}, 42);
+    const auto cbor = json::to_cbor(expected);
+
+    nlohmann::detail::json_sax_acceptor<json> acceptor;
+    CHECK_FALSE(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor));
+    CHECK_FALSE(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor,
+                                true, false, false, json::cbor_tag_handler_t::error));
+
+    CHECK(json::sax_parse(cbor, &acceptor, json::input_format_t::cbor,
+                          true, false, false, json::cbor_tag_handler_t::ignore));
+
+    json parsed;
+    nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> sax(parsed);
+    CHECK(json::sax_parse(cbor, &sax, json::input_format_t::cbor,
+                          true, false, false, json::cbor_tag_handler_t::store));
+    CHECK(parsed == expected);
+
+    json iterator_parsed;
+    nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> iterator_sax(iterator_parsed);
+    CHECK(json::sax_parse(cbor.begin(), cbor.end(), &iterator_sax, json::input_format_t::cbor,
+                          true, false, false, json::cbor_tag_handler_t::store));
+    CHECK(iterator_parsed == expected);
+
+    json span_parsed;
+    nlohmann::detail::json_sax_dom_parser<json, nlohmann::detail::string_input_adapter_type> span_sax(span_parsed);
+    CHECK(json::sax_parse(nlohmann::detail::span_input_adapter(cbor.data(), cbor.size()), &span_sax,
+                          json::input_format_t::cbor, true, false, false, json::cbor_tag_handler_t::store));
+    CHECK(span_parsed == expected);
+
+    const std::string text = "null";
+    CHECK(json::sax_parse(text, &acceptor, json::input_format_t::json,
+                          true, false, false, json::cbor_tag_handler_t::store));
+    CHECK(json::sax_parse(text.begin(), text.end(), &acceptor, json::input_format_t::json,
+                          true, false, false, json::cbor_tag_handler_t::store));
+    CHECK(json::sax_parse(nlohmann::detail::span_input_adapter(text.data(), text.size()), &acceptor,
+                          json::input_format_t::json, true, false, false, json::cbor_tag_handler_t::store));
+}
+
 TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with an object")
 {
     json t = {{"k", 1}};
@@ -920,4 +960,12 @@ TEST_CASE("regression test #5476 - array type without reserve()")
     }
 }
 
+TEST_CASE("issue #5317 - nested indefinite-length CBOR string chunks are rejected")
+{
+    json _;
+    CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t>({0x7F, 0x7F, 0x61, 0x61, 0xFF, 0xFF})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&);
+    CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t>({0x5F, 0x5F, 0x41, 0x61, 0xFF, 0xFF})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR binary: indefinite-length binary array is not allowed inside indefinite-length binary array; last byte: 0x5F", json::parse_error&);
+    CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<std::uint8_t>({0xA1, 0x7F, 0x7F, 0xFF, 0xFF, 0x01})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: indefinite-length string is not allowed inside indefinite-length string; last byte: 0x7F", json::parse_error&);
+}
+
 DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-serialization.cpp

@@ -809,3 +809,178 @@ TEST_CASE("serializer buffers are flushed mid-string and mid-binary")
         CHECK(j.dump(2) == "{\n  \"bytes\": [" + expected_pretty_bytes + "],\n  \"subtype\": null\n}");
     }
 }
+
+TEST_CASE("serialization boundary values for the write buffer")
+{
+    // write_buffer is a std::array<char, 1024> (write_buffer_size). put_string()
+    // guards it with two checks, and each must be exercised exactly on and one
+    // past its own boundary: a heap overflow in a different manual buffer path
+    // (the dump(1100) indent buffer) once survived 100% line coverage because
+    // every test that touched it only ever grew the buffer by a single step,
+    // never landing on the exact edge of the comparison that protects it.
+    //
+    // - straight-through: put_string() bypasses write_buffer entirely and
+    //   writes directly to the output adapter once `length >= write_buffer.size()`.
+    // - flush-then-copy: otherwise, if `write_buffer_pos + length > write_buffer.size()`,
+    //   put_string() flushes what is pending and then memcpy's the new run into
+    //   the freshly emptied buffer.
+
+    SECTION("top-level string exercises the straight-through guard (length >= 1024)")
+    {
+        // dump() of a bare string writes the opening quote with put_char()
+        // (write_buffer_pos: 0 -> 1), then the body with put_string(). With
+        // write_buffer_pos == 1, `1 + length > 1024` and `length >= 1024` flip
+        // together at length 1024, so 1023/1024/1025 cover "just under",
+        // "exactly at" and "just over" the guard in one move: 1023 is copied
+        // into the buffer (filling it exactly), 1024 and 1025 bypass it.
+        for (const std::size_t len :
+                {
+                    std::size_t{1023}, std::size_t{1024}, std::size_t{1025}
+                })
+        {
+            CAPTURE(len)
+            const std::string body(len, 'a');
+            const json j = body;
+            const std::string expected = '"' + body + '"';
+
+            CHECK(j.dump() == expected);
+
+            std::ostringstream o;
+            o << j;
+            CHECK(o.str() == expected);
+        }
+    }
+
+    SECTION("string nested in an array exercises the flush-then-copy guard")
+    {
+        // json::array({body}) writes '[' then '"' before the body, so
+        // write_buffer_pos == 2 when put_string() is entered for it. The
+        // body's last byte then lands at logical offset 2 + len: len == 1022
+        // lands exactly on offset 1024 (2 + 1022 == write_buffer.size(), so the
+        // strict "> " guard does not fire and the body fits snugly), while
+        // len == 1023 lands one past it at offset 1025 (2 + 1023 > 1024),
+        // which must flush what's pending before copying the body in.
+        for (const std::size_t len :
+                {
+                    std::size_t{1022}, std::size_t{1023}
+                })
+        {
+            CAPTURE(len)
+            const std::string body(len, 'a');
+            const json j = json::array({body});
+            const std::string expected = "[\"" + body + "\"]";
+
+            CHECK(j.dump() == expected);
+
+            std::ostringstream o;
+            o << j;
+            CHECK(o.str() == expected);
+
+            CHECK(json::parse(j.dump()) == j);
+        }
+    }
+}
+
+TEST_CASE("serialization boundary values for the string buffer")
+{
+    // string_buffer is a std::array<char, 512>. dump_escaped_impl() flushes it
+    // mid-string once fewer than 13 bytes remain (`string_buffer.size() - bytes
+    // < 13`), 13 being one more than the most a single code point can ever
+    // write at once (a surrogate pair: two back-to-back "\uXXXX" escapes, 12
+    // bytes). Every write into string_buffer that this check protects happens
+    // in steps of 2 (a simple "\\x" escape) or 6 (one "\uXXXX" unit), so
+    // `bytes` only ever takes even values at the point the check runs - the
+    // tightest values actually reachable are therefore 498 (512 - 498 == 14,
+    // one simple escape away from the threshold) and 500 (512 - 500 == 12,
+    // where the flush fires immediately and resets bytes to 0).
+
+    SECTION("a run of 2-byte escapes lands bytes on, and one step past, the flush threshold")
+    {
+        for (const int count :
+                {
+                    249, 250, 251
+                })
+        {
+            CAPTURE(count)
+            const json j = std::string(static_cast<std::size_t>(count), '\n');
+            std::string expected = "\"";
+            for (int i = 0; i < count; ++i)
+            {
+                expected += "\\n";
+            }
+            expected += '"';
+            CHECK(j.dump() == expected);
+        }
+    }
+
+    SECTION("an ASCII prefix leaves the tightest reachable margin before a 12-byte surrogate pair")
+    {
+        // U+1F600 (the "\xF0\x9F\x98\x80" UTF-8 bytes) is dumped under
+        // ensure_ascii as the 12-byte surrogate pair "\ud83d\ude00"; that
+        // write happens in a single step with no intermediate flush check, so
+        // it is the write most exposed by an off-by-one in the "< 13" guard.
+        // A prefix of 249 newlines leaves exactly 14 bytes of headroom
+        // (512 - 498), the smallest margin the guard ever actually allows
+        // into a new code point; 250 newlines instead trigger the guard's own
+        // flush first, so the emoji starts from a freshly emptied (512-byte)
+        // buffer, and 251 repeats that with one more escape already past the
+        // reset. Together they cover the margin the guard allows landing on,
+        // one step before, and one step after - all must still produce the
+        // identical, correct escapes.
+        for (const int prefix_count :
+                {
+                    249, 250, 251
+                })
+        {
+            CAPTURE(prefix_count)
+            const std::string prefix(static_cast<std::size_t>(prefix_count), '\n');
+            const std::string emoji = "\xF0\x9F\x98\x80";
+            const json j = prefix + emoji;
+
+            std::string expected_prefix;
+            for (int i = 0; i < prefix_count; ++i)
+            {
+                expected_prefix += "\\n";
+            }
+
+            // newline escaping does not depend on ensure_ascii: only the
+            // emoji differs (raw UTF-8 bytes vs. a \u-escaped surrogate pair)
+            CHECK(j.dump(-1, ' ', false) == '"' + expected_prefix + emoji + '"');
+            CHECK(j.dump(-1, ' ', true) == '"' + expected_prefix + "\\ud83d\\ude00\"");
+            CHECK(json::parse(j.dump(-1, ' ', true)) == j);
+            CHECK(json::parse(j.dump(-1, ' ', false)) == j);
+        }
+    }
+
+    SECTION("SWAR bulk-copy stride: k plain bytes followed by a byte handled individually")
+    {
+        // string_bulk_run()/find_ascii_copyable_run() (string_scan.hpp) scan 8
+        // bytes at a time and fall back to a byte-at-a-time tail scan for
+        // what is left over. k from 0 to 17 spans zero, one and two full
+        // 8-byte strides plus a 1-byte tail, so every possible stopping point
+        // within and right after the SIMD stride is covered.
+        for (std::size_t k = 0; k <= 17; ++k)
+        {
+            CAPTURE(k)
+            const std::string prefix(k, 'a');
+
+            // (a) the run is stopped by a quote that must itself be escaped
+            {
+                const json j = prefix + "\"";
+                CHECK(j.dump() == '"' + prefix + "\\\"" + '"');
+            }
+
+            // (b) the run is stopped by a control character
+            {
+                const json j = prefix + "\x01";
+                CHECK(j.dump() == '"' + prefix + "\\u0001" + '"');
+            }
+
+            // (c) the run is stopped by a non-ASCII byte under ensure_ascii
+            {
+                const json j = prefix + "\xC3\xA9"; // prefix + 'é'
+                CHECK(j.dump(-1, ' ', true) == '"' + prefix + "\\u00e9" + '"');
+            }
+        }
+    }
+}

tests/src/unit-ubjson.cpp

@@ -3308,3 +3308,43 @@ TEST_CASE("UBJSON and BJData integer markers at every range edge")
         }
     }
 }
+
+TEST_CASE("UBJSON large strings (chunked reader)")
+{
+    // get_ubjson_string() reads through get_string(), which reads in bounded
+    // chunks (binary_reader.hpp, chunk_size == 4096); make sure roundtripping
+    // is correct for lengths around and beyond that chunk size, for both
+    // vector (iterator) and pointer inputs.
+    //
+    // A binary value is not included here: plain UBJSON (unlike BJData, see
+    // the "BJData large strings and binaries" test) has no reader-side binary
+    // type, so even the optimized uint8_t-array encoding of a binary value is
+    // read back element-by-element as a JSON array of numbers rather than
+    // through get_binary() - it never reaches the chunked path this test is
+    // about (see the "roundtrip only works to an array of numbers" case
+    // above).
+    for (const std::size_t len :
+            {
+                std::size_t{0}, std::size_t{1}, std::size_t{4095}, std::size_t{4096},
+                std::size_t{4097}, std::size_t{8192}, std::size_t{100000}
+            })
+    {
+        CAPTURE(len)
+
+        const json j_string = std::string(len, 'x');
+        const std::vector<std::uint8_t> v_string = json::to_ubjson(j_string);
+        CHECK(json::from_ubjson(v_string) == j_string);
+        // pointer input exercises the std::memcpy fast path
+        CHECK(json::from_ubjson(reinterpret_cast<const char*>(v_string.data()),
+                                reinterpret_cast<const char*>(v_string.data()) + v_string.size()) == j_string);
+
+        // a truncated payload must still be reported as an error
+        if (len > 16)
+        {
+            std::vector<std::uint8_t> truncated = v_string;
+            truncated.resize(truncated.size() - 8);
+            json _;
+            CHECK_THROWS_AS(_ = json::from_ubjson(truncated), json::parse_error);
+        }
+    }
+}

tests/src/unit-unicode1.cpp

@@ -1,623 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// for some reason including this after the json header leads to linker errors with VS 2017...
-#include <locale>
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-
-#include <fstream>
-#include <sstream>
-#include <iomanip>
-#include "make_test_data_available.hpp"
-#include "test_utils.hpp"
-
-TEST_CASE("Unicode (1/5)" * doctest::skip())
-{
-    SECTION("\\uxxxx sequences")
-    {
-        // create an escaped string from a code point
-        const auto codepoint_to_unicode = [](std::size_t cp)
-        {
-            // code points are represented as a six-character sequence: a
-            // reverse solidus, followed by the lowercase letter u, followed
-            // by four hexadecimal digits that encode the character's code
-            // point
-            std::stringstream ss;
-            ss << "\\u" << std::setw(4) << std::setfill('0') << std::hex << cp;
-            return ss.str();
-        };
-
-        SECTION("correct sequences")
-        {
-            // generate all UTF-8 code points; in total, 1112064 code points are
-            // generated: 0x1FFFFF code points - 2048 invalid values between
-            // 0xD800 and 0xDFFF.
-            for (std::size_t cp = 0; cp <= 0x10FFFFu; ++cp)
-            {
-                // string to store the code point as in \uxxxx format
-                std::string json_text = "\"";
-
-                // decide whether to use one or two \uxxxx sequences
-                if (cp < 0x10000u)
-                {
-                    // The Unicode standard permanently reserves these code point
-                    // values for UTF-16 encoding of the high and low surrogates, and
-                    // they will never be assigned a character, so there should be no
-                    // reason to encode them. The official Unicode standard says that
-                    // no UTF forms, including UTF-16, can encode these code points.
-                    if (cp >= 0xD800u && cp <= 0xDFFFu)
-                    {
-                        // if we would not skip these code points, we would get a
-                        // "missing low surrogate" exception
-                        continue;
-                    }
-
-                    // code points in the Basic Multilingual Plane can be
-                    // represented with one \uxxxx sequence
-                    json_text += codepoint_to_unicode(cp);
-                }
-                else
-                {
-                    // To escape an extended character that is not in the Basic
-                    // Multilingual Plane, the character is represented as a
-                    // 12-character sequence, encoding the UTF-16 surrogate pair
-                    const auto codepoint1 = 0xd800u + (((cp - 0x10000u) >> 10) & 0x3ffu);
-                    const auto codepoint2 = 0xdc00u + ((cp - 0x10000u) & 0x3ffu);
-                    json_text += codepoint_to_unicode(codepoint1) + codepoint_to_unicode(codepoint2);
-                }
-
-                json_text += "\"";
-                CAPTURE(json_text)
-                json _;
-                CHECK_NOTHROW(_ = json::parse(json_text));
-            }
-        }
-
-        SECTION("incorrect sequences")
-        {
-            SECTION("incorrect surrogate values")
-            {
-                json _;
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uDC00\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 7: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uDC00'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD7FF\\uDC00\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF; last read: '\"\\uD7FF\\uDC00'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800]\""), "[json.exception.parse_error.101] parse error at line 1, column 8: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800]'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\v\""), "[json.exception.parse_error.101] parse error at line 1, column 9: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\v'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\u123\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: '\\u' must be followed by 4 hex digits; last read: '\"\\uD800\\u123\"'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uDBFF\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uDBFF'", json::parse_error&);
-
-                CHECK_THROWS_WITH_AS(_ = json::parse("\"\\uD800\\uE000\""), "[json.exception.parse_error.101] parse error at line 1, column 13: syntax error while parsing value - invalid string: surrogate U+D800..U+DBFF must be followed by U+DC00..U+DFFF; last read: '\"\\uD800\\uE000'", json::parse_error&);
-            }
-        }
-
-#if 0 // NOLINT(readability-avoid-unconditional-preprocessor-if)
-        SECTION("incorrect sequences")
-        {
-            SECTION("high surrogate without low surrogate")
-            {
-                // D800..DBFF are high surrogates and must be followed by low
-                // surrogates DC00..DFFF; here, nothing follows
-                for (std::size_t cp = 0xD800u; cp <= 0xDBFFu; ++cp)
-                {
-                    std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
-                    CAPTURE(json_text)
-                    CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
-                }
-            }
-
-            SECTION("high surrogate with wrong low surrogate")
-            {
-                // D800..DBFF are high surrogates and must be followed by low
-                // surrogates DC00..DFFF; here a different sequence follows
-                for (std::size_t cp1 = 0xD800u; cp1 <= 0xDBFFu; ++cp1)
-                {
-                    for (std::size_t cp2 = 0x0000u; cp2 <= 0xFFFFu; ++cp2)
-                    {
-                        if (0xDC00u <= cp2 && cp2 <= 0xDFFFu)
-                        {
-                            continue;
-                        }
-
-                        std::string json_text = "\"" + codepoint_to_unicode(cp1) + codepoint_to_unicode(cp2) + "\"";
-                        CAPTURE(json_text)
-                        CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
-                    }
-                }
-            }
-
-            SECTION("low surrogate without high surrogate")
-            {
-                // low surrogates DC00..DFFF must follow high surrogates; here,
-                // they occur alone
-                for (std::size_t cp = 0xDC00u; cp <= 0xDFFFu; ++cp)
-                {
-                    std::string json_text = "\"" + codepoint_to_unicode(cp) + "\"";
-                    CAPTURE(json_text)
-                    CHECK_THROWS_AS(json::parse(json_text), json::parse_error&);
-                }
-            }
-
-        }
-#endif
-    }
-
-    SECTION("read all unicode characters")
-    {
-        // read a file with all Unicode characters stored as single-character
-        // strings in a JSON array
-        std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/all_unicode.json");
-        json j;
-        CHECK_NOTHROW(f >> j);
-
-        // the array has 1112064 + 1 elements (a terminating "null" value)
-        // Note: 1112064 = 0x1FFFFF code points - 2048 invalid values between
-        // 0xD800 and 0xDFFF.
-        CHECK(j.size() == 1112065);
-
-        SECTION("check JSON Pointers")
-        {
-            for (const auto& s : j)
-            {
-                // skip non-string JSON values
-                if (!s.is_string())
-                {
-                    continue;
-                }
-
-                auto ptr = s.get<std::string>();
-
-                // tilde must be followed by 0 or 1
-                if (ptr == "~")
-                {
-                    ptr += "0";
-                }
-
-                // JSON Pointers must begin with "/"
-                ptr.insert(0, "/");
-
-                CHECK_NOTHROW(json::json_pointer("/" + ptr));
-
-                // check escape/unescape roundtrip
-                auto escaped = nlohmann::detail::escape(ptr);
-                nlohmann::detail::unescape(escaped);
-                CHECK(escaped == ptr);
-            }
-        }
-    }
-
-    SECTION("ignore byte-order-mark")
-    {
-        SECTION("in a stream")
-        {
-            // read a file with a UTF-8 BOM
-            std::ifstream f(TEST_DATA_DIRECTORY "/json_nlohmann_tests/bom.json");
-            json j;
-            CHECK_NOTHROW(f >> j);
-        }
-
-        SECTION("with an iterator")
-        {
-            std::string i = "\xef\xbb\xbf{\n   \"foo\": true\n}";
-            json _;
-            CHECK_NOTHROW(_ = json::parse(i.begin(), i.end()));
-        }
-    }
-
-    SECTION("error for incomplete/wrong BOM")
-    {
-        json _;
-        CHECK_THROWS_AS(_ = json::parse("\xef\xbb"), json::parse_error&);
-        CHECK_THROWS_AS(_ = json::parse("\xef\xbb\xbb"), json::parse_error&);
-    }
-}
-
-namespace
-{
-void roundtrip(bool success_expected, const std::string& s);
-
-void roundtrip(bool success_expected, const std::string& s)
-{
-    CAPTURE(s)
-    json _;
-
-    // create JSON string value
-    const json j = s;
-    // create JSON text
-    const std::string ps = std::string("\"") + s + "\"";
-
-    if (success_expected)
-    {
-        // serialization succeeds
-        // dump() is nodiscard; this only checks that dumping does not throw
-        CHECK_NOTHROW(utils::ignore_return_value(j.dump()));
-
-        // exclude parse test for U+0000
-        if (s[0] != '\0')
-        {
-            // parsing JSON text succeeds
-            CHECK_NOTHROW(_ = json::parse(ps));
-        }
-
-        // roundtrip succeeds
-        CHECK_NOTHROW(_ = json::parse(j.dump()));
-
-        // after roundtrip, the same string is stored
-        const json jr = json::parse(j.dump());
-        CHECK(jr.get<std::string>() == s);
-    }
-    else
-    {
-        // serialization fails
-        // dump() is nodiscard; the exception is thrown by dump() itself before it would return
-        CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&);
-
-        // parsing JSON text fails
-        CHECK_THROWS_AS(_ = json::parse(ps), json::parse_error&);
-    }
-}
-} // namespace
-
-TEST_CASE("Markus Kuhn's UTF-8 decoder capability and stress test")
-{
-    // Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> - 2015-08-28 - CC BY 4.0
-    // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
-
-    SECTION("1  Some correct UTF-8 text")
-    {
-        roundtrip(true, "κόσμε");
-    }
-
-    SECTION("2  Boundary condition test cases")
-    {
-        SECTION("2.1  First possible sequence of a certain length")
-        {
-            // 2.1.1  1 byte  (U-00000000)
-            roundtrip(true, std::string("\0", 1));
-            // 2.1.2  2 bytes (U-00000080)
-            roundtrip(true, "\xc2\x80");
-            // 2.1.3  3 bytes (U-00000800)
-            roundtrip(true, "\xe0\xa0\x80");
-            // 2.1.4  4 bytes (U-00010000)
-            roundtrip(true, "\xf0\x90\x80\x80");
-
-            // 2.1.5  5 bytes (U-00200000)
-            roundtrip(false, "\xF8\x88\x80\x80\x80");
-            // 2.1.6  6 bytes (U-04000000)
-            roundtrip(false, "\xFC\x84\x80\x80\x80\x80");
-        }
-
-        SECTION("2.2  Last possible sequence of a certain length")
-        {
-            // 2.2.1  1 byte  (U-0000007F)
-            roundtrip(true, "\x7f");
-            // 2.2.2  2 bytes (U-000007FF)
-            roundtrip(true, "\xdf\xbf");
-            // 2.2.3  3 bytes (U-0000FFFF)
-            roundtrip(true, "\xef\xbf\xbf");
-
-            // 2.2.4  4 bytes (U-001FFFFF)
-            roundtrip(false, "\xF7\xBF\xBF\xBF");
-            // 2.2.5  5 bytes (U-03FFFFFF)
-            roundtrip(false, "\xFB\xBF\xBF\xBF\xBF");
-            // 2.2.6  6 bytes (U-7FFFFFFF)
-            roundtrip(false, "\xFD\xBF\xBF\xBF\xBF\xBF");
-        }
-
-        SECTION("2.3  Other boundary conditions")
-        {
-            // 2.3.1  U-0000D7FF = ed 9f bf
-            roundtrip(true, "\xed\x9f\xbf");
-            // 2.3.2  U-0000E000 = ee 80 80
-            roundtrip(true, "\xee\x80\x80");
-            // 2.3.3  U-0000FFFD = ef bf bd
-            roundtrip(true, "\xef\xbf\xbd");
-            // 2.3.4  U-0010FFFF = f4 8f bf bf
-            roundtrip(true, "\xf4\x8f\xbf\xbf");
-
-            // 2.3.5  U-00110000 = f4 90 80 80
-            roundtrip(false, "\xf4\x90\x80\x80");
-        }
-    }
-
-    SECTION("3  Malformed sequences")
-    {
-        SECTION("3.1  Unexpected continuation bytes")
-        {
-            // Each unexpected continuation byte should be separately signalled as a
-            // malformed sequence of its own.
-
-            // 3.1.1  First continuation byte 0x80
-            roundtrip(false, "\x80");
-            // 3.1.2  Last  continuation byte 0xbf
-            roundtrip(false, "\xbf");
-
-            // 3.1.3  2 continuation bytes
-            roundtrip(false, "\x80\xbf");
-            // 3.1.4  3 continuation bytes
-            roundtrip(false, "\x80\xbf\x80");
-            // 3.1.5  4 continuation bytes
-            roundtrip(false, "\x80\xbf\x80\xbf");
-            // 3.1.6  5 continuation bytes
-            roundtrip(false, "\x80\xbf\x80\xbf\x80");
-            // 3.1.7  6 continuation bytes
-            roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf");
-            // 3.1.8  7 continuation bytes
-            roundtrip(false, "\x80\xbf\x80\xbf\x80\xbf\x80");
-
-            // 3.1.9  Sequence of all 64 possible continuation bytes (0x80-0xbf)
-            roundtrip(false, "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf");
-        }
-
-        SECTION("3.2  Lonely start characters")
-        {
-            // 3.2.1  All 32 first bytes of 2-byte sequences (0xc0-0xdf)
-            roundtrip(false, "\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 \xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf \xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 \xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf");
-            // 3.2.2  All 16 first bytes of 3-byte sequences (0xe0-0xef)
-            roundtrip(false, "\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 \xe8 \xe9 \xea \xeb \xec \xed \xee \xef");
-            // 3.2.3  All 8 first bytes of 4-byte sequences (0xf0-0xf7)
-            roundtrip(false, "\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7");
-            // 3.2.4  All 4 first bytes of 5-byte sequences (0xf8-0xfb)
-            roundtrip(false, "\xf8 \xf9 \xfa \xfb");
-            // 3.2.5  All 2 first bytes of 6-byte sequences (0xfc-0xfd)
-            roundtrip(false, "\xfc \xfd");
-        }
-
-        SECTION("3.3  Sequences with last continuation byte missing")
-        {
-            // All bytes of an incomplete sequence should be signalled as a single
-            // malformed sequence, i.e., you should see only a single replacement
-            // character in each of the next 10 tests. (Characters as in section 2)
-
-            // 3.3.1  2-byte sequence with last byte missing (U+0000)
-            roundtrip(false, "\xc0");
-            // 3.3.2  3-byte sequence with last byte missing (U+0000)
-            roundtrip(false, "\xe0\x80");
-            // 3.3.3  4-byte sequence with last byte missing (U+0000)
-            roundtrip(false, "\xf0\x80\x80");
-            // 3.3.4  5-byte sequence with last byte missing (U+0000)
-            roundtrip(false, "\xf8\x80\x80\x80");
-            // 3.3.5  6-byte sequence with last byte missing (U+0000)
-            roundtrip(false, "\xfc\x80\x80\x80\x80");
-            // 3.3.6  2-byte sequence with last byte missing (U-000007FF)
-            roundtrip(false, "\xdf");
-            // 3.3.7  3-byte sequence with last byte missing (U-0000FFFF)
-            roundtrip(false, "\xef\xbf");
-            // 3.3.8  4-byte sequence with last byte missing (U-001FFFFF)
-            roundtrip(false, "\xf7\xbf\xbf");
-            // 3.3.9  5-byte sequence with last byte missing (U-03FFFFFF)
-            roundtrip(false, "\xfb\xbf\xbf\xbf");
-            // 3.3.10 6-byte sequence with last byte missing (U-7FFFFFFF)
-            roundtrip(false, "\xfd\xbf\xbf\xbf\xbf");
-        }
-
-        SECTION("3.4  Concatenation of incomplete sequences")
-        {
-            // All the 10 sequences of 3.3 concatenated, you should see 10 malformed
-            // sequences being signalled:
-            roundtrip(false, "\xc0\xe0\x80\xf0\x80\x80\xf8\x80\x80\x80\xfc\x80\x80\x80\x80\xdf\xef\xbf\xf7\xbf\xbf\xfb\xbf\xbf\xbf\xfd\xbf\xbf\xbf\xbf");
-        }
-
-        SECTION("3.5  Impossible bytes")
-        {
-            // The following two bytes cannot appear in a correct UTF-8 string
-
-            // 3.5.1  fe
-            roundtrip(false, "\xfe");
-            // 3.5.2  ff
-            roundtrip(false, "\xff");
-            // 3.5.3  fe fe ff ff
-            roundtrip(false, "\xfe\xfe\xff\xff");
-        }
-    }
-
-    SECTION("4  Overlong sequences")
-    {
-        // The following sequences are not malformed according to the letter of
-        // the Unicode 2.0 standard. However, they are longer then necessary and
-        // a correct UTF-8 encoder is not allowed to produce them. A "safe UTF-8
-        // decoder" should reject them just like malformed sequences for two
-        // reasons: (1) It helps to debug applications if overlong sequences are
-        // not treated as valid representations of characters, because this helps
-        // to spot problems more quickly. (2) Overlong sequences provide
-        // alternative representations of characters, that could maliciously be
-        // used to bypass filters that check only for ASCII characters. For
-        // instance, a 2-byte encoded line feed (LF) would not be caught by a
-        // line counter that counts only 0x0a bytes, but it would still be
-        // processed as a line feed by an unsafe UTF-8 decoder later in the
-        // pipeline. From a security point of view, ASCII compatibility of UTF-8
-        // sequences means also, that ASCII characters are *only* allowed to be
-        // represented by ASCII bytes in the range 0x00-0x7f. To ensure this
-        // aspect of ASCII compatibility, use only "safe UTF-8 decoders" that
-        // reject overlong UTF-8 sequences for which a shorter encoding exists.
-
-        SECTION("4.1  Examples of an overlong ASCII character")
-        {
-            // With a safe UTF-8 decoder, all the following five overlong
-            // representations of the ASCII character slash ("/") should be rejected
-            // like a malformed UTF-8 sequence, for instance by substituting it with
-            // a replacement character. If you see a slash below, you do not have a
-            // safe UTF-8 decoder!
-
-            // 4.1.1 U+002F = c0 af
-            roundtrip(false, "\xc0\xaf");
-            // 4.1.2 U+002F = e0 80 af
-            roundtrip(false, "\xe0\x80\xaf");
-            // 4.1.3 U+002F = f0 80 80 af
-            roundtrip(false, "\xf0\x80\x80\xaf");
-            // 4.1.4 U+002F = f8 80 80 80 af
-            roundtrip(false, "\xf8\x80\x80\x80\xaf");
-            // 4.1.5 U+002F = fc 80 80 80 80 af
-            roundtrip(false, "\xfc\x80\x80\x80\x80\xaf");
-        }
-
-        SECTION("4.2  Maximum overlong sequences")
-        {
-            // Below you see the highest Unicode value that is still resulting in an
-            // overlong sequence if represented with the given number of bytes. This
-            // is a boundary test for safe UTF-8 decoders. All five characters should
-            // be rejected like malformed UTF-8 sequences.
-
-            // 4.2.1  U-0000007F = c1 bf
-            roundtrip(false, "\xc1\xbf");
-            // 4.2.2  U-000007FF = e0 9f bf
-            roundtrip(false, "\xe0\x9f\xbf");
-            // 4.2.3  U-0000FFFF = f0 8f bf bf
-            roundtrip(false, "\xf0\x8f\xbf\xbf");
-            // 4.2.4  U-001FFFFF = f8 87 bf bf bf
-            roundtrip(false, "\xf8\x87\xbf\xbf\xbf");
-            // 4.2.5  U-03FFFFFF = fc 83 bf bf bf bf
-            roundtrip(false, "\xfc\x83\xbf\xbf\xbf\xbf");
-        }
-
-        SECTION("4.3  Overlong representation of the NUL character")
-        {
-            // The following five sequences should also be rejected like malformed
-            // UTF-8 sequences and should not be treated like the ASCII NUL
-            // character.
-
-            // 4.3.1  U+0000 = c0 80
-            roundtrip(false, "\xc0\x80");
-            // 4.3.2  U+0000 = e0 80 80
-            roundtrip(false, "\xe0\x80\x80");
-            // 4.3.3  U+0000 = f0 80 80 80
-            roundtrip(false, "\xf0\x80\x80\x80");
-            // 4.3.4  U+0000 = f8 80 80 80 80
-            roundtrip(false, "\xf8\x80\x80\x80\x80");
-            // 4.3.5  U+0000 = fc 80 80 80 80 80
-            roundtrip(false, "\xfc\x80\x80\x80\x80\x80");
-        }
-    }
-
-    SECTION("5  Illegal code positions")
-    {
-        // The following UTF-8 sequences should be rejected like malformed
-        // sequences, because they never represent valid ISO 10646 characters and
-        // a UTF-8 decoder that accepts them might introduce security problems
-        // comparable to overlong UTF-8 sequences.
-
-        SECTION("5.1 Single UTF-16 surrogates")
-        {
-            // 5.1.1  U+D800 = ed a0 80
-            roundtrip(false, "\xed\xa0\x80");
-            // 5.1.2  U+DB7F = ed ad bf
-            roundtrip(false, "\xed\xad\xbf");
-            // 5.1.3  U+DB80 = ed ae 80
-            roundtrip(false, "\xed\xae\x80");
-            // 5.1.4  U+DBFF = ed af bf
-            roundtrip(false, "\xed\xaf\xbf");
-            // 5.1.5  U+DC00 = ed b0 80
-            roundtrip(false, "\xed\xb0\x80");
-            // 5.1.6  U+DF80 = ed be 80
-            roundtrip(false, "\xed\xbe\x80");
-            // 5.1.7  U+DFFF = ed bf bf
-            roundtrip(false, "\xed\xbf\xbf");
-        }
-
-        SECTION("5.2 Paired UTF-16 surrogates")
-        {
-            // 5.2.1  U+D800 U+DC00 = ed a0 80 ed b0 80
-            roundtrip(false, "\xed\xa0\x80\xed\xb0\x80");
-            // 5.2.2  U+D800 U+DFFF = ed a0 80 ed bf bf
-            roundtrip(false, "\xed\xa0\x80\xed\xbf\xbf");
-            // 5.2.3  U+DB7F U+DC00 = ed ad bf ed b0 80
-            roundtrip(false, "\xed\xad\xbf\xed\xb0\x80");
-            // 5.2.4  U+DB7F U+DFFF = ed ad bf ed bf bf
-            roundtrip(false, "\xed\xad\xbf\xed\xbf\xbf");
-            // 5.2.5  U+DB80 U+DC00 = ed ae 80 ed b0 80
-            roundtrip(false, "\xed\xae\x80\xed\xb0\x80");
-            // 5.2.6  U+DB80 U+DFFF = ed ae 80 ed bf bf
-            roundtrip(false, "\xed\xae\x80\xed\xbf\xbf");
-            // 5.2.7  U+DBFF U+DC00 = ed af bf ed b0 80
-            roundtrip(false, "\xed\xaf\xbf\xed\xb0\x80");
-            // 5.2.8  U+DBFF U+DFFF = ed af bf ed bf bf
-            roundtrip(false, "\xed\xaf\xbf\xed\xbf\xbf");
-        }
-
-        SECTION("5.3 Noncharacter code positions")
-        {
-            // The following "noncharacters" are "reserved for internal use" by
-            // applications, and according to older versions of the Unicode Standard
-            // "should never be interchanged". Unicode Corrigendum #9 dropped the
-            // latter restriction. Nevertheless, their presence in incoming UTF-8 data
-            // can remain a potential security risk, depending on what use is made of
-            // these codes subsequently. Examples of such internal use:
-            //
-            //  - Some file APIs with 16-bit characters may use the integer value -1
-            //    = U+FFFF to signal an end-of-file (EOF) or error condition.
-            //
-            //  - In some UTF-16 receivers, code point U+FFFE might trigger a
-            //    byte-swap operation (to convert between UTF-16LE and UTF-16BE).
-            //
-            // With such internal use of noncharacters, it may be desirable and safer
-            // to block those code points in UTF-8 decoders, as they should never
-            // occur legitimately in incoming UTF-8 data, and could trigger unsafe
-            // behaviour in subsequent processing.
-
-            // Particularly problematic noncharacters in 16-bit applications:
-
-            // 5.3.1  U+FFFE = ef bf be
-            roundtrip(true, "\xef\xbf\xbe");
-            // 5.3.2  U+FFFF = ef bf bf
-            roundtrip(true, "\xef\xbf\xbf");
-
-            // 5.3.3  U+FDD0 .. U+FDEF
-            roundtrip(true, "\xEF\xB7\x90");
-            roundtrip(true, "\xEF\xB7\x91");
-            roundtrip(true, "\xEF\xB7\x92");
-            roundtrip(true, "\xEF\xB7\x93");
-            roundtrip(true, "\xEF\xB7\x94");
-            roundtrip(true, "\xEF\xB7\x95");
-            roundtrip(true, "\xEF\xB7\x96");
-            roundtrip(true, "\xEF\xB7\x97");
-            roundtrip(true, "\xEF\xB7\x98");
-            roundtrip(true, "\xEF\xB7\x99");
-            roundtrip(true, "\xEF\xB7\x9A");
-            roundtrip(true, "\xEF\xB7\x9B");
-            roundtrip(true, "\xEF\xB7\x9C");
-            roundtrip(true, "\xEF\xB7\x9D");
-            roundtrip(true, "\xEF\xB7\x9E");
-            roundtrip(true, "\xEF\xB7\x9F");
-            roundtrip(true, "\xEF\xB7\xA0");
-            roundtrip(true, "\xEF\xB7\xA1");
-            roundtrip(true, "\xEF\xB7\xA2");
-            roundtrip(true, "\xEF\xB7\xA3");
-            roundtrip(true, "\xEF\xB7\xA4");
-            roundtrip(true, "\xEF\xB7\xA5");
-            roundtrip(true, "\xEF\xB7\xA6");
-            roundtrip(true, "\xEF\xB7\xA7");
-            roundtrip(true, "\xEF\xB7\xA8");
-            roundtrip(true, "\xEF\xB7\xA9");
-            roundtrip(true, "\xEF\xB7\xAA");
-            roundtrip(true, "\xEF\xB7\xAB");
-            roundtrip(true, "\xEF\xB7\xAC");
-            roundtrip(true, "\xEF\xB7\xAD");
-            roundtrip(true, "\xEF\xB7\xAE");
-            roundtrip(true, "\xEF\xB7\xAF");
-
-            // 5.3.4  U+nFFFE U+nFFFF (for n = 1..10)
-            roundtrip(true, "\xF0\x9F\xBF\xBF");
-            roundtrip(true, "\xF0\xAF\xBF\xBF");
-            roundtrip(true, "\xF0\xBF\xBF\xBF");
-            roundtrip(true, "\xF1\x8F\xBF\xBF");
-            roundtrip(true, "\xF1\x9F\xBF\xBF");
-            roundtrip(true, "\xF1\xAF\xBF\xBF");
-            roundtrip(true, "\xF1\xBF\xBF\xBF");
-            roundtrip(true, "\xF2\x8F\xBF\xBF");
-            roundtrip(true, "\xF2\x9F\xBF\xBF");
-            roundtrip(true, "\xF2\xAF\xBF\xBF");
-        }
-    }
-}

tests/src/unit-unicode2.cpp

@@ -1,612 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// for some reason including this after the json header leads to linker errors with VS 2017...
-#include <locale>
-
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-
-#include <fstream>
-#include <sstream>
-#include <iostream>
-#include <iomanip>
-#include "make_test_data_available.hpp"
-#include "test_utils.hpp"
-
-// this test suite uses static variables with non-trivial destructors
-DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
-DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
-
-namespace
-{
-extern size_t calls;
-size_t calls = 0;
-
-void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    static std::string json_string;
-    json_string.clear();
-
-    CAPTURE(byte1)
-    CAPTURE(byte2)
-    CAPTURE(byte3)
-    CAPTURE(byte4)
-
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    CAPTURE(json_string)
-
-    // store the string in a JSON value
-    static json j;
-    static json j2;
-    j = json_string;
-    j2 = "abc" + json_string + "xyz";
-
-    static std::string s_ignored;
-    static std::string s_ignored2;
-    static std::string s_ignored_ascii;
-    static std::string s_ignored2_ascii;
-    static std::string s_replaced;
-    static std::string s_replaced2;
-    static std::string s_replaced_ascii;
-    static std::string s_replaced2_ascii;
-
-    // dumping with ignore/replace must not throw in any case
-    s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
-    s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
-
-    if (success_expected)
-    {
-        static std::string s_strict;
-        // strict mode must not throw if success is expected
-        s_strict = j.dump();
-        // all dumps should agree on the string
-        CHECK(s_strict == s_ignored);
-        CHECK(s_strict == s_replaced);
-    }
-    else
-    {
-        // strict mode must throw if success is not expected
-        // dump() is nodiscard; the exception is thrown by dump() itself before it would return
-        CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&);
-        // ignore and replace must create different dumps
-        CHECK(s_ignored != s_replaced);
-
-        // check that replace string contains a replacement character
-        CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
-    }
-
-    // check that prefix and suffix are preserved
-    CHECK(s_ignored2.substr(1, 3) == "abc");
-    CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz");
-    CHECK(s_ignored2_ascii.substr(1, 3) == "abc");
-    CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2.substr(1, 3) == "abc");
-    CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
-    CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
-}
-
-void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-// create and check a JSON string with up to four UTF-8 bytes
-void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    if (++calls % 100000 == 0)
-    {
-        std::cout << calls << " of 455355 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl)
-    }
-
-    static std::string json_string;
-    json_string = "\"";
-
-    CAPTURE(byte1)
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        CAPTURE(byte2)
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        CAPTURE(byte3)
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        CAPTURE(byte4)
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    json_string += "\"";
-
-    CAPTURE(json_string)
-
-    json _;
-    if (success_expected)
-    {
-        CHECK_NOTHROW(_ = json::parse(json_string));
-    }
-    else
-    {
-        CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&);
-    }
-}
-} // namespace
-
-TEST_CASE("Unicode (2/5)" * doctest::skip())
-{
-    SECTION("RFC 3629")
-    {
-        /*
-        RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as
-        follows:
-
-            A UTF-8 string is a sequence of octets representing a sequence of UCS
-            characters.  An octet sequence is valid UTF-8 only if it matches the
-            following syntax, which is derived from the rules for encoding UTF-8
-            and is expressed in the ABNF of [RFC2234].
-
-            UTF8-octets = *( UTF8-char )
-            UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
-            UTF8-1      = %x00-7F
-            UTF8-2      = %xC2-DF UTF8-tail
-            UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
-                          %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
-            UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
-                          %xF4 %x80-8F 2( UTF8-tail )
-            UTF8-tail   = %x80-BF
-        */
-
-        SECTION("ill-formed first byte")
-        {
-            for (int byte1 = 0x80; byte1 <= 0xC1; ++byte1)
-            {
-                check_utf8string(false, byte1);
-                check_utf8dump(false, byte1);
-            }
-
-            for (int byte1 = 0xF5; byte1 <= 0xFF; ++byte1)
-            {
-                check_utf8string(false, byte1);
-                check_utf8dump(false, byte1);
-            }
-        }
-
-        SECTION("UTF8-1 (x00-x7F)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0x00; byte1 <= 0x7F; ++byte1)
-                {
-                    // unescaped control characters are parse errors in JSON
-                    if (0x00 <= byte1 && byte1 <= 0x1F)
-                    {
-                        check_utf8string(false, byte1);
-                        continue;
-                    }
-
-                    // a single quote is a parse error in JSON
-                    if (byte1 == 0x22)
-                    {
-                        check_utf8string(false, byte1);
-                        continue;
-                    }
-
-                    // a single backslash is a parse error in JSON
-                    if (byte1 == 0x5C)
-                    {
-                        check_utf8string(false, byte1);
-                        continue;
-                    }
-
-                    // all other characters are OK
-                    check_utf8string(true, byte1);
-                    check_utf8dump(true, byte1);
-                }
-            }
-        }
-
-        SECTION("UTF8-2 (xC2-xDF UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(true, byte1, byte2);
-                        check_utf8dump(true, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xC2; byte1 <= 0xDF; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-        }
-
-        SECTION("UTF8-3 (xE0 xA0-BF UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1)
-                {
-                    for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(true, byte1, byte2, byte3);
-                            check_utf8dump(true, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1)
-                {
-                    for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0xA0 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xE0; byte1 <= 0xE0; ++byte1)
-                {
-                    for (int byte2 = 0xA0; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-        }
-
-        SECTION("UTF8-3 (xE1-xEC UTF8-tail UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(true, byte1, byte2, byte3);
-                            check_utf8dump(true, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xE1; byte1 <= 0xEC; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-        }
-
-        SECTION("UTF8-3 (xED x80-9F UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xED; byte1 <= 0xED; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(true, byte1, byte2, byte3);
-                            check_utf8dump(true, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xED; byte1 <= 0xED; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xED; byte1 <= 0xED; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xED; byte1 <= 0xED; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0x9F)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xED; byte1 <= 0xED; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x9F; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-        }
-
-        SECTION("UTF8-3 (xEE-xEF UTF8-tail UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(true, byte1, byte2, byte3);
-                            check_utf8dump(true, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xEE; byte1 <= 0xEF; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-unicode3.cpp

@@ -1,326 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// for some reason including this after the json header leads to linker errors with VS 2017...
-#include <locale>
-
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-
-#include <fstream>
-#include <sstream>
-#include <iostream>
-#include <iomanip>
-#include "make_test_data_available.hpp"
-#include "test_utils.hpp"
-
-// this test suite uses static variables with non-trivial destructors
-DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
-DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
-
-namespace
-{
-extern size_t calls;
-size_t calls = 0;
-
-void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    static std::string json_string;
-    json_string.clear();
-
-    CAPTURE(byte1)
-    CAPTURE(byte2)
-    CAPTURE(byte3)
-    CAPTURE(byte4)
-
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    CAPTURE(json_string)
-
-    // store the string in a JSON value
-    static json j;
-    static json j2;
-    j = json_string;
-    j2 = "abc" + json_string + "xyz";
-
-    static std::string s_ignored;
-    static std::string s_ignored2;
-    static std::string s_ignored_ascii;
-    static std::string s_ignored2_ascii;
-    static std::string s_replaced;
-    static std::string s_replaced2;
-    static std::string s_replaced_ascii;
-    static std::string s_replaced2_ascii;
-
-    // dumping with ignore/replace must not throw in any case
-    s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
-    s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
-
-    if (success_expected)
-    {
-        static std::string s_strict;
-        // strict mode must not throw if success is expected
-        s_strict = j.dump();
-        // all dumps should agree on the string
-        CHECK(s_strict == s_ignored);
-        CHECK(s_strict == s_replaced);
-    }
-    else
-    {
-        // strict mode must throw if success is not expected
-        // dump() is nodiscard; the exception is thrown by dump() itself before it would return
-        CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&);
-        // ignore and replace must create different dumps
-        CHECK(s_ignored != s_replaced);
-
-        // check that replace string contains a replacement character
-        CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
-    }
-
-    // check that prefix and suffix are preserved
-    CHECK(s_ignored2.substr(1, 3) == "abc");
-    CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz");
-    CHECK(s_ignored2_ascii.substr(1, 3) == "abc");
-    CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2.substr(1, 3) == "abc");
-    CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
-    CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
-}
-
-void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-// create and check a JSON string with up to four UTF-8 bytes
-void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    if (++calls % 100000 == 0)
-    {
-        std::cout << calls << " of 1641521 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl)
-    }
-
-    static std::string json_string;
-    json_string = "\"";
-
-    CAPTURE(byte1)
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        CAPTURE(byte2)
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        CAPTURE(byte3)
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        CAPTURE(byte4)
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    json_string += "\"";
-
-    CAPTURE(json_string)
-
-    json _;
-    if (success_expected)
-    {
-        CHECK_NOTHROW(_ = json::parse(json_string));
-    }
-    else
-    {
-        CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&);
-    }
-}
-} // namespace
-
-TEST_CASE("Unicode (3/5)" * doctest::skip())
-{
-    SECTION("RFC 3629")
-    {
-        /*
-        RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as
-        follows:
-
-            A UTF-8 string is a sequence of octets representing a sequence of UCS
-            characters.  An octet sequence is valid UTF-8 only if it matches the
-            following syntax, which is derived from the rules for encoding UTF-8
-            and is expressed in the ABNF of [RFC2234].
-
-            UTF8-octets = *( UTF8-char )
-            UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
-            UTF8-1      = %x00-7F
-            UTF8-2      = %xC2-DF UTF8-tail
-            UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
-                          %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
-            UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
-                          %xF4 %x80-8F 2( UTF8-tail )
-            UTF8-tail   = %x80-BF
-        */
-
-        SECTION("UTF8-4 (xF0 x90-BF UTF8-tail UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(true, byte1, byte2, byte3, byte4);
-                                check_utf8dump(true, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing fourth byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x90 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong fourth byte")
-            {
-                for (int byte1 = 0xF0; byte1 <= 0xF0; ++byte1)
-                {
-                    for (int byte2 = 0x90; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
-                            {
-                                // skip correct fourth byte
-                                if (0x80 <= byte4 && byte4 <= 0xBF)
-                                {
-                                    continue;
-                                }
-
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-unicode4.cpp

@@ -1,326 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// for some reason including this after the json header leads to linker errors with VS 2017...
-#include <locale>
-
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-
-#include <fstream>
-#include <sstream>
-#include <iostream>
-#include <iomanip>
-#include "make_test_data_available.hpp"
-#include "test_utils.hpp"
-
-// this test suite uses static variables with non-trivial destructors
-DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
-DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
-
-namespace
-{
-extern size_t calls;
-size_t calls = 0;
-
-void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    static std::string json_string;
-    json_string.clear();
-
-    CAPTURE(byte1)
-    CAPTURE(byte2)
-    CAPTURE(byte3)
-    CAPTURE(byte4)
-
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    CAPTURE(json_string)
-
-    // store the string in a JSON value
-    static json j;
-    static json j2;
-    j = json_string;
-    j2 = "abc" + json_string + "xyz";
-
-    static std::string s_ignored;
-    static std::string s_ignored2;
-    static std::string s_ignored_ascii;
-    static std::string s_ignored2_ascii;
-    static std::string s_replaced;
-    static std::string s_replaced2;
-    static std::string s_replaced_ascii;
-    static std::string s_replaced2_ascii;
-
-    // dumping with ignore/replace must not throw in any case
-    s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
-    s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
-
-    if (success_expected)
-    {
-        static std::string s_strict;
-        // strict mode must not throw if success is expected
-        s_strict = j.dump();
-        // all dumps should agree on the string
-        CHECK(s_strict == s_ignored);
-        CHECK(s_strict == s_replaced);
-    }
-    else
-    {
-        // strict mode must throw if success is not expected
-        // dump() is nodiscard; the exception is thrown by dump() itself before it would return
-        CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&);
-        // ignore and replace must create different dumps
-        CHECK(s_ignored != s_replaced);
-
-        // check that replace string contains a replacement character
-        CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
-    }
-
-    // check that prefix and suffix are preserved
-    CHECK(s_ignored2.substr(1, 3) == "abc");
-    CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz");
-    CHECK(s_ignored2_ascii.substr(1, 3) == "abc");
-    CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2.substr(1, 3) == "abc");
-    CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
-    CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
-}
-
-void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-// create and check a JSON string with up to four UTF-8 bytes
-void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    if (++calls % 100000 == 0)
-    {
-        std::cout << calls << " of 5517507 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl)
-    }
-
-    static std::string json_string;
-    json_string = "\"";
-
-    CAPTURE(byte1)
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        CAPTURE(byte2)
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        CAPTURE(byte3)
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        CAPTURE(byte4)
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    json_string += "\"";
-
-    CAPTURE(json_string)
-
-    json _;
-    if (success_expected)
-    {
-        CHECK_NOTHROW(_ = json::parse(json_string));
-    }
-    else
-    {
-        CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&);
-    }
-}
-} // namespace
-
-TEST_CASE("Unicode (4/5)" * doctest::skip())
-{
-    SECTION("RFC 3629")
-    {
-        /*
-        RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as
-        follows:
-
-            A UTF-8 string is a sequence of octets representing a sequence of UCS
-            characters.  An octet sequence is valid UTF-8 only if it matches the
-            following syntax, which is derived from the rules for encoding UTF-8
-            and is expressed in the ABNF of [RFC2234].
-
-            UTF8-octets = *( UTF8-char )
-            UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
-            UTF8-1      = %x00-7F
-            UTF8-2      = %xC2-DF UTF8-tail
-            UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
-                          %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
-            UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
-                          %xF4 %x80-8F 2( UTF8-tail )
-            UTF8-tail   = %x80-BF
-        */
-
-        SECTION("UTF8-4 (xF1-F3 UTF8-tail UTF8-tail UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(true, byte1, byte2, byte3, byte4);
-                                check_utf8dump(true, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing fourth byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0xBF)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong fourth byte")
-            {
-                for (int byte1 = 0xF1; byte1 <= 0xF3; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0xBF; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
-                            {
-                                // skip correct fourth byte
-                                if (0x80 <= byte4 && byte4 <= 0xBF)
-                                {
-                                    continue;
-                                }
-
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-DOCTEST_CLANG_SUPPRESS_WARNING_POP

tests/src/unit-unicode5.cpp

@@ -1,326 +0,0 @@
-//     __ _____ _____ _____
-//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
-// |  |  |__   |  |  | | | |  version 3.12.0
-// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
-//
-// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
-// SPDX-License-Identifier: MIT
-
-#include "doctest_compatibility.h"
-
-// for some reason including this after the json header leads to linker errors with VS 2017...
-#include <locale>
-
-#include <nlohmann/json.hpp>
-using nlohmann::json;
-
-#include <fstream>
-#include <sstream>
-#include <iostream>
-#include <iomanip>
-#include "make_test_data_available.hpp"
-#include "test_utils.hpp"
-
-// this test suite uses static variables with non-trivial destructors
-DOCTEST_CLANG_SUPPRESS_WARNING_PUSH
-DOCTEST_CLANG_SUPPRESS_WARNING("-Wexit-time-destructors")
-
-namespace
-{
-extern size_t calls;
-size_t calls = 0;
-
-void check_utf8dump(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-void check_utf8dump(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    static std::string json_string;
-    json_string.clear();
-
-    CAPTURE(byte1)
-    CAPTURE(byte2)
-    CAPTURE(byte3)
-    CAPTURE(byte4)
-
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    CAPTURE(json_string)
-
-    // store the string in a JSON value
-    static json j;
-    static json j2;
-    j = json_string;
-    j2 = "abc" + json_string + "xyz";
-
-    static std::string s_ignored;
-    static std::string s_ignored2;
-    static std::string s_ignored_ascii;
-    static std::string s_ignored2_ascii;
-    static std::string s_replaced;
-    static std::string s_replaced2;
-    static std::string s_replaced_ascii;
-    static std::string s_replaced2_ascii;
-
-    // dumping with ignore/replace must not throw in any case
-    s_ignored = j.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored2 = j2.dump(-1, ' ', false, json::error_handler_t::ignore);
-    s_ignored_ascii = j.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_ignored2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::ignore);
-    s_replaced = j.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced2 = j2.dump(-1, ' ', false, json::error_handler_t::replace);
-    s_replaced_ascii = j.dump(-1, ' ', true, json::error_handler_t::replace);
-    s_replaced2_ascii = j2.dump(-1, ' ', true, json::error_handler_t::replace);
-
-    if (success_expected)
-    {
-        static std::string s_strict;
-        // strict mode must not throw if success is expected
-        s_strict = j.dump();
-        // all dumps should agree on the string
-        CHECK(s_strict == s_ignored);
-        CHECK(s_strict == s_replaced);
-    }
-    else
-    {
-        // strict mode must throw if success is not expected
-        // dump() is nodiscard; the exception is thrown by dump() itself before it would return
-        CHECK_THROWS_AS(utils::ignore_return_value(j.dump()), json::type_error&);
-        // ignore and replace must create different dumps
-        CHECK(s_ignored != s_replaced);
-
-        // check that replace string contains a replacement character
-        CHECK(s_replaced.find("\xEF\xBF\xBD") != std::string::npos);
-    }
-
-    // check that prefix and suffix are preserved
-    CHECK(s_ignored2.substr(1, 3) == "abc");
-    CHECK(s_ignored2.substr(s_ignored2.size() - 4, 3) == "xyz");
-    CHECK(s_ignored2_ascii.substr(1, 3) == "abc");
-    CHECK(s_ignored2_ascii.substr(s_ignored2_ascii.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2.substr(1, 3) == "abc");
-    CHECK(s_replaced2.substr(s_replaced2.size() - 4, 3) == "xyz");
-    CHECK(s_replaced2_ascii.substr(1, 3) == "abc");
-    CHECK(s_replaced2_ascii.substr(s_replaced2_ascii.size() - 4, 3) == "xyz");
-}
-
-void check_utf8string(bool success_expected, int byte1, int byte2, int byte3, int byte4);
-
-// create and check a JSON string with up to four UTF-8 bytes
-void check_utf8string(bool success_expected, int byte1, int byte2 = -1, int byte3 = -1, int byte4 = -1)
-{
-    if (++calls % 100000 == 0)
-    {
-        std::cout << calls << " of 1246225 UTF-8 strings checked" << std::endl; // NOLINT(performance-avoid-endl)
-    }
-
-    static std::string json_string;
-    json_string = "\"";
-
-    CAPTURE(byte1)
-    json_string += std::string(1, static_cast<char>(byte1));
-
-    if (byte2 != -1)
-    {
-        CAPTURE(byte2)
-        json_string += std::string(1, static_cast<char>(byte2));
-    }
-
-    if (byte3 != -1)
-    {
-        CAPTURE(byte3)
-        json_string += std::string(1, static_cast<char>(byte3));
-    }
-
-    if (byte4 != -1)
-    {
-        CAPTURE(byte4)
-        json_string += std::string(1, static_cast<char>(byte4));
-    }
-
-    json_string += "\"";
-
-    CAPTURE(json_string)
-
-    json _;
-    if (success_expected)
-    {
-        CHECK_NOTHROW(_ = json::parse(json_string));
-    }
-    else
-    {
-        CHECK_THROWS_AS(_ = json::parse(json_string), json::parse_error&);
-    }
-}
-} // namespace
-
-TEST_CASE("Unicode (5/5)" * doctest::skip())
-{
-    SECTION("RFC 3629")
-    {
-        /*
-        RFC 3629 describes in Sect. 4 the syntax of UTF-8 byte sequences as
-        follows:
-
-            A UTF-8 string is a sequence of octets representing a sequence of UCS
-            characters.  An octet sequence is valid UTF-8 only if it matches the
-            following syntax, which is derived from the rules for encoding UTF-8
-            and is expressed in the ABNF of [RFC2234].
-
-            UTF8-octets = *( UTF8-char )
-            UTF8-char   = UTF8-1 / UTF8-2 / UTF8-3 / UTF8-4
-            UTF8-1      = %x00-7F
-            UTF8-2      = %xC2-DF UTF8-tail
-            UTF8-3      = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) /
-                          %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail )
-            UTF8-4      = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) /
-                          %xF4 %x80-8F 2( UTF8-tail )
-            UTF8-tail   = %x80-BF
-        */
-
-        SECTION("UTF8-4 (xF4 x80-8F UTF8-tail UTF8-tail)")
-        {
-            SECTION("well-formed")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(true, byte1, byte2, byte3, byte4);
-                                check_utf8dump(true, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing second byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    check_utf8string(false, byte1);
-                    check_utf8dump(false, byte1);
-                }
-            }
-
-            SECTION("ill-formed: missing third byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
-                    {
-                        check_utf8string(false, byte1, byte2);
-                        check_utf8dump(false, byte1, byte2);
-                    }
-                }
-            }
-
-            SECTION("ill-formed: missing fourth byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            check_utf8string(false, byte1, byte2, byte3);
-                            check_utf8dump(false, byte1, byte2, byte3);
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong second byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x00; byte2 <= 0xFF; ++byte2)
-                    {
-                        // skip correct second byte
-                        if (0x80 <= byte2 && byte2 <= 0x8F)
-                        {
-                            continue;
-                        }
-
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong third byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
-                    {
-                        for (int byte3 = 0x00; byte3 <= 0xFF; ++byte3)
-                        {
-                            // skip correct third byte
-                            if (0x80 <= byte3 && byte3 <= 0xBF)
-                            {
-                                continue;
-                            }
-
-                            for (int byte4 = 0x80; byte4 <= 0xBF; ++byte4)
-                            {
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-
-            SECTION("ill-formed: wrong fourth byte")
-            {
-                for (int byte1 = 0xF4; byte1 <= 0xF4; ++byte1)
-                {
-                    for (int byte2 = 0x80; byte2 <= 0x8F; ++byte2)
-                    {
-                        for (int byte3 = 0x80; byte3 <= 0xBF; ++byte3)
-                        {
-                            for (int byte4 = 0x00; byte4 <= 0xFF; ++byte4)
-                            {
-                                // skip correct fourth byte
-                                if (0x80 <= byte4 && byte4 <= 0xBF)
-                                {
-                                    continue;
-                                }
-
-                                check_utf8string(false, byte1, byte2, byte3, byte4);
-                                check_utf8dump(false, byte1, byte2, byte3, byte4);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-DOCTEST_CLANG_SUPPRESS_WARNING_POP