Cancel CI runs of merged and closed pull requests (#5748)

* Re-amalgamate single_include

#5737 changed 13 headers under include/ but merged without the matching
single_include/nlohmann/json.hpp update, so the amalgamated header still
had, among others, the GCC C++20 -Wignored-attributes pragma block and
the clang -Wdocumentation push/pop that #5737 removed, the forwarding
from_json tuple/array helpers it replaced with const references, and
lacked the output_adapter char_traits changes it added.

Regenerated with `make amalgamate` (astyle 3.4.13). The diff is exactly
`git diff b54ed188e e5a89d671 -- include/` (164+/95-); json_fwd.hpp and
json_literals.hpp were already up to date.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

* Run the amalgamation check on pushes to develop

#5737 was merged 39 seconds after its last push, while its own
check_amalgamation run was still queued (earlier runs had been
cancelled by the concurrency group), so the stale single_include
reached develop without any failing check.

Also run the check on pushes to develop, without cancelling in-progress
develop runs. The "save" job (PR number/author for the comment
workflow) only runs for pull requests, the checkout falls back to
github.sha, and comment_check_amalgamation.yml only comments for
PR-triggered runs, since push runs have no PR and no "pr" artifact.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

---------

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

.github/workflows/cancel_closed_pr_runs.yml

@@ -0,0 +1,53 @@
+name: "Cancel runs of closed pull requests"
+
+# The concurrency groups of the other workflows cancel superseded runs when a
+# pull request gets new commits, but nothing stops the runs of its last commit
+# once the pull request is merged or closed. They then keep the runners busy
+# for hours while the queue of the open pull requests waits.
+#
+# pull_request_target is needed to get a token that can cancel runs for pull
+# requests from forks. This is safe because the workflow never checks out or
+# runs code from the pull request; it only calls the API.
+on:
+  pull_request_target:
+    types: [closed]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  cancel:
+    permissions:
+      actions: write
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
+        with:
+          egress-policy: audit
+
+      - name: Cancel unfinished runs of the pull request's head commit
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          SELF: ${{ github.run_id }}
+        # only runs triggered by the pull request: when a branch is pushed to
+        # develop directly, its push runs share the head commit
+        run: |
+          gh api --paginate "repos/$GH_REPO/actions/runs?head_sha=$HEAD_SHA&per_page=100" \
+            --jq ".workflow_runs[]
+                  | select(.status != \"completed\" and .id != $SELF)
+                  | select(.event == \"pull_request\" or .event == \"pull_request_target\")
+                  | \"\(.id) \(.name)\"" |
+          while read -r id name; do
+            echo "Cancelling run $id ($name)"
+            # a run may finish between listing and cancelling; that is not an error
+            gh run cancel "$id" || true
+          done

.github/workflows/check_amalgamation.yml

@@ -2,16 +2,23 @@ name: "Check amalgamation"
 
 on:
   pull_request:
+  # also check develop itself: a PR can be merged before its own run of this
+  # workflow completes (e.g. while it is still queued), leaving single_include
+  # stale on develop without any failing check
+  push:
+    branches:
+      - develop
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
 
 jobs:
   save:
+    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
@@ -43,11 +50,11 @@ jobs:
         with:
           egress-policy: audit
 
-      - name: Checkout pull request
+      - name: Checkout pull request or pushed commit
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           path: main
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
           persist-credentials: false
 
       - name: Checkout tools

.github/workflows/comment_check_amalgamation.yml

@@ -10,7 +10,8 @@ permissions:
 
 jobs:
   comment:
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    # push runs on develop have no PR to comment on (and no "pr" artifact)
+    if: ${{ github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.event == 'pull_request' }}
     runs-on: ubuntu-latest
     permissions:
       contents: read

docs/mkdocs/docs/api/basic_json/to_bjdata.md

@@ -56,8 +56,6 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 
 - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size`
   is false.
-- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
-  not valid UTF-8
 
 ## Complexity
 
@@ -91,5 +89,4 @@ Linear in the size of the JSON value `j`.
 ## Version history
 
 - Added in version 3.11.0.
-- BJData version parameter (for draft3 binary encoding) added in version 3.12.0.
-- Throwing `type_error.316` for a string or object key that is not valid UTF-8 added in version 3.13.0.
+- BJData version parameter (for draft3 binary encoding) added in version 3.12.0.

docs/mkdocs/docs/api/basic_json/to_bson.md

@@ -46,8 +46,6 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 - Throws [`out_of_range.415`](../../home/exceptions.md#jsonexceptionout_of_range415) if the subtype of a binary value
   exceeds 255, the maximum of the BSON binary subtype; example:
   `"subtype 70000 is too large for the BSON binary subtype (max 255)"`
-- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key is
-  not valid UTF-8
 
 ## Complexity
 
@@ -84,5 +82,3 @@ pass before anything is written.
 - Added in version 3.4.0.
 - Linear in the size of `j`, and no longer limited by the call stack for deeply nested values, since version 3.13.0.
 - `out_of_range.415` is now detected before anything is written, like the other exceptions above, since version 3.13.0.
-- Throwing `type_error.316` for a string value or object key that is not valid UTF-8, detected before anything is
-  written, added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/to_cbor.md

@@ -35,11 +35,6 @@ The exact mapping and its limitations are described on a [dedicated page](../../
 
 Strong guarantee: if an exception is thrown, there are no changes in the JSON value.
 
-## Exceptions
-
-- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
-  not valid UTF-8
-
 ## Complexity
 
 Linear in the size of the JSON value `j`.
@@ -73,4 +68,3 @@ Linear in the size of the JSON value `j`.
 
 - Added in version 2.0.9.
 - Compact representation of floating-point numbers added in version 3.8.0.
-- Throwing `type_error.316` for a string or object key that is not valid UTF-8 added in version 3.13.0.

docs/mkdocs/docs/api/basic_json/to_ubjson.md

@@ -49,8 +49,6 @@ Strong guarantee: if an exception is thrown, there are no changes in the JSON va
 
 - Throws [`other_error.502`](../../home/exceptions.md#jsonexceptionother_error502) if `use_type` is true and `use_size`
   is false.
-- Throws [type_error.316](../../home/exceptions.md#jsonexceptiontype_error316) if a string or object key in `j` is
-  not valid UTF-8
 
 ## Complexity
 
@@ -84,4 +82,3 @@ Linear in the size of the JSON value `j`.
 ## Version history
 
 - Added in version 3.1.0.
-- Throwing `type_error.316` for a string or object key that is not valid UTF-8 added in version 3.13.0.

docs/mkdocs/docs/features/binary_formats/bjdata.md

@@ -63,12 +63,6 @@ The library uses the following mapping from JSON values types to BJData types ac
 
       - strings with more than 18446744073709551615 bytes, i.e., 2<sup>64</sup>-1 bytes (theoretical)
 
-!!! warning "UTF-8 validation of string values and object keys"
-
-    BJData strings must use UTF-8 encoding. `to_bjdata()` validates the bytes of every string value and object key
-    and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8, so a
-    value with such a string cannot be serialized in the first place.
-
 !!! info "Unused BJData markers"
 
     The following markers are not used in the conversion:
@@ -214,15 +208,6 @@ The library maps BJData types to JSON value types as follows:
 
     The mapping is **complete** in the sense that any BJData value can be converted to a JSON value.
 
-!!! warning "Ill-formed UTF-8 in string values and object keys"
-
-    BJData strings must use UTF-8 encoding, but this is not enforced on read: `from_bjdata()` accepts a string
-    value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However,
-    [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
-    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error
-    handler is passed that replaces or ignores the ill-formed bytes. `to_bjdata()` is strict as well (see above), so
-    a value read this way cannot be written back to BJData.
-
 !!! info "Round trips"
 
     A value returned by [`from_bjdata`](../../api/basic_json/from_bjdata.md) can be serialized with

docs/mkdocs/docs/features/binary_formats/bson.md

@@ -109,17 +109,14 @@ The library maps BSON record types to JSON value types as follows:
     If BSON input must be validated for strict specification compliance, validate it separately before passing it to
     `from_bson()`.
 
-!!! warning "Ill-formed UTF-8 in string values"
+!!! warning "UTF-8 validation of string values"
 
-    The BSON specification requires `string` values (type `0x02`) to be valid UTF-8, but this is not required of a
-    decoder. `from_bson()` accepts a `string` value whose bytes are not valid UTF-8 and hands them back unchanged.
-    However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
-    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error
-    handler is passed that replaces or ignores the ill-formed bytes. `to_bson()` is strict as well and throws the
-    same exception for a string value or element (key) name that is not valid UTF-8, so an object with such a key
-    or value cannot be produced in the first place, even though `from_bson()` would accept it from another source.
-    Element (key) names are never validated on read, since they are read byte-by-byte as a C string. `binary`
-    values (type `0x05`) are unaffected, since they are not required to hold text.
+    The BSON specification requires `string` values (type `0x02`) to be valid UTF-8. This library validates the
+    bytes of every such string at decode time and rejects ill-formed UTF-8 with a
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with `allow_exceptions`
+    set to `false`, a discarded value), rather than only failing later when the resulting value is dumped. Element
+    (key) names and `binary` values (type `0x05`) are unaffected and are never validated, since they are read
+    byte-by-byte as a C string, or are not required to hold text, respectively.
 
 ??? example
 

docs/mkdocs/docs/features/binary_formats/cbor.md

@@ -189,16 +189,15 @@ The library maps CBOR types to JSON value types as follows:
     ([RFC 8392](https://www.rfc-editor.org/rfc/rfc8392.html)), cannot be read with this library and need a
     general-purpose CBOR library instead.
 
-!!! warning "Ill-formed UTF-8 in text strings"
+!!! warning "UTF-8 validation of text strings"
 
     [RFC 8949, Section 3.1](https://www.rfc-editor.org/rfc/rfc8949.html#section-3.1) requires CBOR text strings
-    (major type 3) to be valid UTF-8, but leaves it up to the decoder whether to enforce this. This library does
-    not: `from_cbor()` accepts a text string (object keys included) whose bytes are not valid UTF-8 and hands them
-    back unchanged. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
-    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error
-    handler is passed that replaces or ignores the ill-formed bytes. `to_cbor()` is strict as well and throws the
-    same exception for a string value or object key that is not valid UTF-8, so such a value cannot be written back
-    to CBOR. Byte strings (major type 2) are unaffected, since they are not required to hold text.
+    (major type 3) to be valid UTF-8. This library validates the bytes of every text string (object keys included) at
+    decode time and rejects ill-formed UTF-8 with a
+    [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or, with
+    `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting value is
+    dumped. Byte strings (major type 2) are unaffected and are never validated, since they are not required to hold
+    text.
 
 !!! warning "Tagged items"
 

docs/mkdocs/docs/features/binary_formats/messagepack.md

@@ -153,15 +153,14 @@ The library maps MessagePack types to JSON value types as follows:
     This applies to the [SAX interface](../parsing/sax_interface.md) as well, as the key is read before it is passed
     on. Such input needs a general-purpose MessagePack library instead.
 
-!!! warning "Ill-formed UTF-8 in string values"
+!!! warning "UTF-8 validation of string values"
 
-    The MessagePack specification explicitly allows a `str` value (`fixstr`, `str 8`, `str 16`, `str 32`) to contain
-    a byte sequence that is not valid UTF-8, and expects a deserializer to hand the original bytes back unchanged.
-    This library follows that: `from_msgpack()` reads `str` bytes (object keys included) as-is, without validating
-    them, and `to_msgpack()` writes them back as-is, so such a value round-trips through `from_msgpack(to_msgpack(j))`
-    byte for byte. However, [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
-    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for a value read this way, unless an
-    error handler is passed that replaces or ignores the ill-formed bytes.
+    The MessagePack specification requires `str` values (`fixstr`, `str 8`, `str 16`, `str 32`) to be valid UTF-8.
+    This library validates the bytes of every such string (object keys included) at decode time and rejects
+    ill-formed UTF-8 with a [`parse_error.113`](../../home/exceptions.md#jsonexceptionparse_error113) exception (or,
+    with `allow_exceptions` set to `false`, a discarded value), rather than only failing later when the resulting
+    value is dumped. `bin`/`ext`/`fixext` values are unaffected and are never validated, since they are not required
+    to hold text.
 
 ??? example
 

docs/mkdocs/docs/features/binary_formats/ubjson.md

@@ -47,12 +47,6 @@ The library uses the following mapping from JSON values types to UBJSON types ac
 
       - strings with more than 9223372036854775807 bytes (theoretical)
 
-!!! warning "UTF-8 validation of string values and object keys"
-
-    UBJSON's required string encoding is UTF-8. `to_ubjson()` validates the bytes of every string value and object
-    key and throws [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for ill-formed UTF-8, so
-    a value with such a string cannot be serialized in the first place.
-
 !!! info "Unused UBJSON markers"
 
     The following markers are not used in the conversion:
@@ -126,15 +120,6 @@ The library maps UBJSON types to JSON value types as follows:
 
     The mapping is **complete** in the sense that any UBJSON value can be converted to a JSON value.
 
-!!! warning "Ill-formed UTF-8 in string values and object keys"
-
-    UBJSON's required string encoding is UTF-8, but this is not enforced on read: `from_ubjson()` accepts a string
-    value or object key whose bytes are not valid UTF-8 and hands them back unchanged. However,
-    [`dump()`](../../api/basic_json/dump.md) still requires valid UTF-8 and throws
-    [`type_error.316`](../../home/exceptions.md#jsonexceptiontype_error316) for such a value, unless an error
-    handler is passed that replaces or ignores the ill-formed bytes. `to_ubjson()` is strict as well (see above), so
-    a value read this way cannot be written back to UBJSON.
-
 ??? example
 
     ```cpp

docs/mkdocs/docs/home/exceptions.md

@@ -340,9 +340,8 @@ An unexpected byte was read in a [binary format](../features/binary_formats/inde
 ### json.exception.parse_error.113
 
 A string could not be read from a [binary format](../features/binary_formats/index.md): either a value that is not a
-string was read where one was required (for instance as a map key), or the string's length specification is invalid.
-The bytes of a string itself are not checked for valid UTF-8 on read; see the ill-formed UTF-8 notes on the
-individual [binary format](../features/binary_formats/index.md) pages for how such a string is handled afterward.
+string was read where one was required (for instance as a map key), the string's length specification is invalid, or
+the string's bytes are not valid UTF-8.
 
 CBOR and MessagePack allow map keys of any type, but JSON object keys are always strings. Maps with keys of any other
 type (for instance integers or `null`) are therefore not supported; see the notes on
@@ -365,6 +364,9 @@ type (for instance integers or `null`) are therefore not supported; see the note
     ```
     [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData string: string length must not be negative
     ```
+    ```
+    [json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte
+    ```
 
 ### json.exception.parse_error.114
 

include/nlohmann/detail/input/binary_reader.hpp

@@ -4031,13 +4031,28 @@ class binary_reader
                     const NumberType len,
                     string_t& result)
     {
-        // Strings are taken as is: none of CBOR (RFC 8949 §3.1 leaves the
-        // choice to the decoder), MessagePack (whose spec explicitly allows
-        // a str object to contain an invalid byte sequence), UBJSON, BJData,
-        // or BSON requires a decoder to reject ill-formed UTF-8. The bytes
-        // are kept unchanged; dump() and the binary writers are the ones
-        // that check them and report type_error.316 if they are not valid.
-        return get_bytes(format, len, "string", result);
+        // get_bytes() appends to result, and CBOR indefinite-length strings
+        // collect all their chunks in the same result; validating only the
+        // newly read bytes keeps the check linear in the input size
+        const std::size_t old_size = result.size();
+        if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result)))
+        {
+            return false;
+        }
+
+        // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications
+        // all require text strings to be valid UTF-8; reject anything else
+        // right here so malformed input is caught at decode time instead of
+        // only surfacing later as a type_error.316 when the value is dumped
+        // (which would defeat allow_exceptions=false / strict discarding).
+        if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size)))
+        {
+            return sax->parse_error(chars_read, get_token_string(),
+                                    parse_error::create(113, chars_read,
+                                            exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr));
+        }
+
+        return true;
     }
 
     /*!

include/nlohmann/detail/output/binary_writer.hpp

@@ -115,8 +115,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     @throw type_error.317 if @a j is not an object
     */
     void write_bson(const BasicJsonType& j)
@@ -147,8 +145,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     */
     void write_cbor(const BasicJsonType& j)
     {
@@ -215,8 +211,6 @@ class binary_writer
 
             case value_t::string:
             {
-                check_utf8(*j.m_data.m_value.string, j);
-
                 // step 1: write control byte and the string length
                 write_cbor_head(0x60, j.m_data.m_value.string->size());
 
@@ -293,11 +287,6 @@ class binary_writer
                 // step 2: write each element
                 for (const auto& el : *j.m_data.m_value.object)
                 {
-                    // el.first is checked here, against the object as
-                    // diagnostics context, because write_cbor(el.first)
-                    // converts it to a temporary basic_json that would be
-                    // used as the context instead
-                    check_utf8(el.first, j);
                     write_cbor(el.first);
                     write_cbor(el.second);
                 }
@@ -640,8 +629,6 @@ class binary_writer
     @param[in] add_prefix  whether prefixes need to be used for this value
     @param[in] use_bjdata  whether write in BJData format, default is false
     @param[in] bjdata_version  which BJData version to use, default is draft2
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     */
     void write_ubjson(const BasicJsonType& j, const bool use_count,
                       const bool use_type, const bool add_prefix = true,
@@ -691,8 +678,6 @@ class binary_writer
 
             case value_t::string:
             {
-                check_utf8(*j.m_data.m_value.string, j);
-
                 if (add_prefix)
                 {
                     oa.write_character(to_char_type('S'));
@@ -855,7 +840,6 @@ class binary_writer
 
                 for (const auto& el : *j.m_data.m_value.object)
                 {
-                    check_utf8(el.first, j);
                     write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);
                     oa.write_characters(
                           reinterpret_cast<const CharType*>(el.first.data()),
@@ -900,10 +884,6 @@ class binary_writer
     /*!
     @return The size of a BSON document entry header, including the id marker
             and the entry name size (and its null-terminator).
-    @throw out_of_range.409 if @a name contains U+0000, before anything is
-           written
-    @throw type_error.316 if @a name is not valid UTF-8, before anything is
-           written
     */
     static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)
     {
@@ -913,8 +893,7 @@ class binary_writer
             JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j));
         }
 
-        check_utf8(name, j);
-
+        static_cast<void>(j);
         return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
     }
 
@@ -970,21 +949,9 @@ class binary_writer
 
     /*!
     @return The size of the BSON-encoded string in @a value
-    @throw type_error.316 if @a value is not valid UTF-8, before anything is
-           written
-
-    @note The UTF-8 check is skipped if @a value is already too long for the
-          32-bit BSON length field (@ref to_bson_length rejects it later, once
-          the size of the whole document is known); this also keeps the check
-          from reading past a StringType that reports a size larger than what
-          it actually holds.
     */
-    static std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j)
+    static std::size_t calc_bson_string_size(const string_t& value)
     {
-        if (JSON_HEDLEY_LIKELY(value_in_range_of<std::int32_t>(value.size())))
-        {
-            check_utf8(value, j);
-        }
         return sizeof(std::int32_t) + value.size() + 1ul;
     }
 
@@ -1113,8 +1080,6 @@ class binary_writer
             is neither an object nor an array
     @throw out_of_range.415 if @a j is binary with a subtype that does not fit
            into a byte, before anything is written
-    @throw type_error.316 if @a j is a string that is not valid UTF-8, before
-           anything is written
     */
     static std::size_t calc_bson_value_size(const BasicJsonType& j)
     {
@@ -1136,7 +1101,7 @@ class binary_writer
                 return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned);
 
             case value_t::string:
-                return calc_bson_string_size(*j.m_data.m_value.string, j);
+                return calc_bson_string_size(*j.m_data.m_value.string);
 
             case value_t::null:
                 return 0ul;
@@ -1249,8 +1214,6 @@ class binary_writer
            written
     @throw out_of_range.415 if a binary value's subtype does not fit into a
            byte, before anything is written
-    @throw type_error.316 if a string value or a key is not valid UTF-8,
-           before anything is written
     */
     static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector<std::size_t>& nested_sizes)
     {
@@ -2129,7 +2092,7 @@ class binary_writer
     */
     void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context)
     {
-        check_utf8(s, context);
+        check_bon8_utf8(s, context);
 
         // a string that follows another string terminates it
         if (string_open)
@@ -2159,7 +2122,7 @@ class binary_writer
     @throw type_error.316 if @a s is not valid UTF-8; the message names the
            first byte of the first invalid or incomplete sequence
     */
-    static void check_utf8(const string_t& s, const BasicJsonType& context)
+    static void check_bon8_utf8(const string_t& s, const BasicJsonType& context)
     {
         static_cast<void>(context); // only used when exceptions are enabled
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());

include/nlohmann/detail/string_utils.hpp

@@ -117,14 +117,13 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally
 written by Björn Hoehrmann. See
 http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
 
-The library checks UTF-8 well-formedness (RFC 3629, section 4) in three
+The library checks UTF-8 well-formedness (RFC 3629, section 4) in four
 places, which differ in speed, diagnostics, and how they read the input:
 
-- decode() below: the serializer, to escape and, in strict mode, reject
-  ill-formed UTF-8 when dumping a string. The CBOR, MessagePack, BSON,
-  UBJSON and BJData readers do not use it: none of those specs requires a
-  decoder to reject ill-formed UTF-8 in text strings, so the readers keep
-  the bytes as is and leave the check to dump() and the binary writers.
+- decode() and @ref is_valid_utf8 below: the serializer (to escape and, in
+  strict mode, reject ill-formed UTF-8 when dumping a string) and the CBOR,
+  MessagePack, BSON, UBJSON and BJData readers (to reject ill-formed UTF-8 in
+  text strings at decode time).
 - the per-lead-byte switch in lexer::scan_string(): JSON text, with a
   diagnostic for each kind of error.
 - validate_one_utf8() and valid_utf8_prefix() in string_scan.hpp: the lexer's
@@ -179,5 +178,38 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std:
     return state;
 }
 
+/*!
+@brief check whether a string consists solely of valid UTF-8
+
+Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text
+strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the
+MessagePack/BSON specifications all require text strings to be UTF-8), so
+that malformed input is caught immediately instead of only surfacing later
+as a type_error.316 when the resulting value is dumped.
+
+@param[in] s      the string to check
+@param[in] first  index of the first byte to check; the bytes before it are
+                  assumed to have been validated already and to end on a
+                  code point boundary
+@return whether @a s (from index @a first on) is valid UTF-8
+*/
+template<typename StringType>
+inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept
+{
+    std::uint8_t state = UTF8_ACCEPT;
+    std::uint32_t codepoint = 0;
+
+    for (std::size_t i = first; i < s.size(); ++i)
+    {
+        decode(state, codepoint, static_cast<std::uint8_t>(s[i]));
+        if (state == UTF8_REJECT)
+        {
+            return false;
+        }
+    }
+
+    return state == UTF8_ACCEPT;
+}
+
 }  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END

single_include/nlohmann/json.hpp

@@ -6321,14 +6321,13 @@ This is a single-byte step of a "shift-based" UTF-8 decoder originally
 written by Björn Hoehrmann. See
 http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
 
-The library checks UTF-8 well-formedness (RFC 3629, section 4) in three
+The library checks UTF-8 well-formedness (RFC 3629, section 4) in four
 places, which differ in speed, diagnostics, and how they read the input:
 
-- decode() below: the serializer, to escape and, in strict mode, reject
-  ill-formed UTF-8 when dumping a string. The CBOR, MessagePack, BSON,
-  UBJSON and BJData readers do not use it: none of those specs requires a
-  decoder to reject ill-formed UTF-8 in text strings, so the readers keep
-  the bytes as is and leave the check to dump() and the binary writers.
+- decode() and @ref is_valid_utf8 below: the serializer (to escape and, in
+  strict mode, reject ill-formed UTF-8 when dumping a string) and the CBOR,
+  MessagePack, BSON, UBJSON and BJData readers (to reject ill-formed UTF-8 in
+  text strings at decode time).
 - the per-lead-byte switch in lexer::scan_string(): JSON text, with a
   diagnostic for each kind of error.
 - validate_one_utf8() and valid_utf8_prefix() in string_scan.hpp: the lexer's
@@ -6383,6 +6382,39 @@ inline std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std:
     return state;
 }
 
+/*!
+@brief check whether a string consists solely of valid UTF-8
+
+Used by the CBOR/MessagePack/BSON/UBJSON binary readers to reject text
+strings that are not valid UTF-8 at decode time (RFC 8949 §3.1 and the
+MessagePack/BSON specifications all require text strings to be UTF-8), so
+that malformed input is caught immediately instead of only surfacing later
+as a type_error.316 when the resulting value is dumped.
+
+@param[in] s      the string to check
+@param[in] first  index of the first byte to check; the bytes before it are
+                  assumed to have been validated already and to end on a
+                  code point boundary
+@return whether @a s (from index @a first on) is valid UTF-8
+*/
+template<typename StringType>
+inline bool is_valid_utf8(const StringType& s, const std::size_t first = 0) noexcept
+{
+    std::uint8_t state = UTF8_ACCEPT;
+    std::uint32_t codepoint = 0;
+
+    for (std::size_t i = first; i < s.size(); ++i)
+    {
+        decode(state, codepoint, static_cast<std::uint8_t>(s[i]));
+        if (state == UTF8_REJECT)
+        {
+            return false;
+        }
+    }
+
+    return state == UTF8_ACCEPT;
+}
+
 }  // namespace detail
 NLOHMANN_JSON_NAMESPACE_END
 
@@ -17512,13 +17544,28 @@ class binary_reader
                     const NumberType len,
                     string_t& result)
     {
-        // Strings are taken as is: none of CBOR (RFC 8949 §3.1 leaves the
-        // choice to the decoder), MessagePack (whose spec explicitly allows
-        // a str object to contain an invalid byte sequence), UBJSON, BJData,
-        // or BSON requires a decoder to reject ill-formed UTF-8. The bytes
-        // are kept unchanged; dump() and the binary writers are the ones
-        // that check them and report type_error.316 if they are not valid.
-        return get_bytes(format, len, "string", result);
+        // get_bytes() appends to result, and CBOR indefinite-length strings
+        // collect all their chunks in the same result; validating only the
+        // newly read bytes keeps the check linear in the input size
+        const std::size_t old_size = result.size();
+        if (JSON_HEDLEY_UNLIKELY(!get_bytes(format, len, "string", result)))
+        {
+            return false;
+        }
+
+        // RFC 8949 (CBOR) §3.1 and the MessagePack/BSON/UBJSON specifications
+        // all require text strings to be valid UTF-8; reject anything else
+        // right here so malformed input is caught at decode time instead of
+        // only surfacing later as a type_error.316 when the value is dumped
+        // (which would defeat allow_exceptions=false / strict discarding).
+        if (JSON_HEDLEY_UNLIKELY(!is_valid_utf8(result, old_size)))
+        {
+            return sax->parse_error(chars_read, get_token_string(),
+                                    parse_error::create(113, chars_read,
+                                            exception_message(format, "invalid string: ill-formed UTF-8 byte", "string"), nullptr));
+        }
+
+        return true;
     }
 
     /*!
@@ -21193,8 +21240,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     @throw type_error.317 if @a j is not an object
     */
     void write_bson(const BasicJsonType& j)
@@ -21225,8 +21270,6 @@ class binary_writer
 
     /*!
     @param[in] j  JSON value to serialize
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     */
     void write_cbor(const BasicJsonType& j)
     {
@@ -21293,8 +21336,6 @@ class binary_writer
 
             case value_t::string:
             {
-                check_utf8(*j.m_data.m_value.string, j);
-
                 // step 1: write control byte and the string length
                 write_cbor_head(0x60, j.m_data.m_value.string->size());
 
@@ -21371,11 +21412,6 @@ class binary_writer
                 // step 2: write each element
                 for (const auto& el : *j.m_data.m_value.object)
                 {
-                    // el.first is checked here, against the object as
-                    // diagnostics context, because write_cbor(el.first)
-                    // converts it to a temporary basic_json that would be
-                    // used as the context instead
-                    check_utf8(el.first, j);
                     write_cbor(el.first);
                     write_cbor(el.second);
                 }
@@ -21718,8 +21754,6 @@ class binary_writer
     @param[in] add_prefix  whether prefixes need to be used for this value
     @param[in] use_bjdata  whether write in BJData format, default is false
     @param[in] bjdata_version  which BJData version to use, default is draft2
-    @throw type_error.316 if a string value or an object key is not valid
-           UTF-8
     */
     void write_ubjson(const BasicJsonType& j, const bool use_count,
                       const bool use_type, const bool add_prefix = true,
@@ -21769,8 +21803,6 @@ class binary_writer
 
             case value_t::string:
             {
-                check_utf8(*j.m_data.m_value.string, j);
-
                 if (add_prefix)
                 {
                     oa.write_character(to_char_type('S'));
@@ -21933,7 +21965,6 @@ class binary_writer
 
                 for (const auto& el : *j.m_data.m_value.object)
                 {
-                    check_utf8(el.first, j);
                     write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata);
                     oa.write_characters(
                           reinterpret_cast<const CharType*>(el.first.data()),
@@ -21978,10 +22009,6 @@ class binary_writer
     /*!
     @return The size of a BSON document entry header, including the id marker
             and the entry name size (and its null-terminator).
-    @throw out_of_range.409 if @a name contains U+0000, before anything is
-           written
-    @throw type_error.316 if @a name is not valid UTF-8, before anything is
-           written
     */
     static std::size_t calc_bson_entry_header_size(const string_t& name, const BasicJsonType& j)
     {
@@ -21991,8 +22018,7 @@ class binary_writer
             JSON_THROW(out_of_range::create(409, concat("BSON key cannot contain code point U+0000 (at byte ", std::to_string(it), ")"), &j));
         }
 
-        check_utf8(name, j);
-
+        static_cast<void>(j);
         return /*id*/ 1ul + name.size() + /*zero-terminator*/1u;
     }
 
@@ -22048,21 +22074,9 @@ class binary_writer
 
     /*!
     @return The size of the BSON-encoded string in @a value
-    @throw type_error.316 if @a value is not valid UTF-8, before anything is
-           written
-
-    @note The UTF-8 check is skipped if @a value is already too long for the
-          32-bit BSON length field (@ref to_bson_length rejects it later, once
-          the size of the whole document is known); this also keeps the check
-          from reading past a StringType that reports a size larger than what
-          it actually holds.
     */
-    static std::size_t calc_bson_string_size(const string_t& value, const BasicJsonType& j)
+    static std::size_t calc_bson_string_size(const string_t& value)
     {
-        if (JSON_HEDLEY_LIKELY(value_in_range_of<std::int32_t>(value.size())))
-        {
-            check_utf8(value, j);
-        }
         return sizeof(std::int32_t) + value.size() + 1ul;
     }
 
@@ -22191,8 +22205,6 @@ class binary_writer
             is neither an object nor an array
     @throw out_of_range.415 if @a j is binary with a subtype that does not fit
            into a byte, before anything is written
-    @throw type_error.316 if @a j is a string that is not valid UTF-8, before
-           anything is written
     */
     static std::size_t calc_bson_value_size(const BasicJsonType& j)
     {
@@ -22214,7 +22226,7 @@ class binary_writer
                 return calc_bson_unsigned_size(j.m_data.m_value.number_unsigned);
 
             case value_t::string:
-                return calc_bson_string_size(*j.m_data.m_value.string, j);
+                return calc_bson_string_size(*j.m_data.m_value.string);
 
             case value_t::null:
                 return 0ul;
@@ -22327,8 +22339,6 @@ class binary_writer
            written
     @throw out_of_range.415 if a binary value's subtype does not fit into a
            byte, before anything is written
-    @throw type_error.316 if a string value or a key is not valid UTF-8,
-           before anything is written
     */
     static std::size_t calc_bson_sizes(const BasicJsonType& document, std::vector<std::size_t>& nested_sizes)
     {
@@ -23207,7 +23217,7 @@ class binary_writer
     */
     void write_bon8_string(const string_t& s, bool& string_open, const BasicJsonType& context)
     {
-        check_utf8(s, context);
+        check_bon8_utf8(s, context);
 
         // a string that follows another string terminates it
         if (string_open)
@@ -23237,7 +23247,7 @@ class binary_writer
     @throw type_error.316 if @a s is not valid UTF-8; the message names the
            first byte of the first invalid or incomplete sequence
     */
-    static void check_utf8(const string_t& s, const BasicJsonType& context)
+    static void check_bon8_utf8(const string_t& s, const BasicJsonType& context)
     {
         static_cast<void>(context); // only used when exceptions are enabled
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());

tests/src/unit-bjdata.cpp

@@ -3907,41 +3907,6 @@ TEST_CASE("Universal Binary JSON Specification Examples 1")
             CHECK(json::to_bjdata(j) == v);
             CHECK(json::from_bjdata(v) == j);
         }
-
-        SECTION("ill-formed UTF-8 (see #5529, #5651)")
-        {
-            // none of the binary format specs requires a decoder to reject
-            // ill-formed UTF-8 in a text string, so a value whose bytes are
-            // not valid UTF-8 (0xC0 0xAE is an overlong encoding of '.')
-            // round-trips byte for byte as a string value; to_bjdata() is
-            // strict, so such a value cannot be written back
-            const std::vector<uint8_t> v = {'S', 'i', 2, 0xc0, 0xae};
-            json j;
-            CHECK_NOTHROW(j = json::from_bjdata(v));
-            REQUIRE(j.is_string());
-            CHECK(j.get_ref<const json::string_t&>() == std::string("\xc0\xae"));
-            CHECK_THROWS_AS(j.dump(), json::type_error&);
-            CHECK_THROWS_AS(json::to_bjdata(j), json::type_error&);
-
-            // the same bytes as an object key round-trip as well
-            const std::vector<uint8_t> v_key = {'{', 'i', 2, 0xc0, 0xae, 'i', 1, '}'};
-            json j_key;
-            CHECK_NOTHROW(j_key = json::from_bjdata(v_key));
-            REQUIRE(j_key.is_object());
-            CHECK(j_key.contains(std::string("\xc0\xae")));
-            CHECK_THROWS_AS(json::to_bjdata(j_key), json::type_error&);
-
-            CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-            // a truncated multi-byte sequence
-            CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&);
-            // an encoded surrogate half (U+D800)
-            CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&);
-            // an overlong encoding of '.'
-            CHECK_THROWS_WITH_AS(json::to_bjdata(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&);
-
-            // an object key with ill-formed UTF-8 is rejected the same way
-            CHECK_THROWS_WITH_AS(json::to_bjdata(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-        }
     }
 
     SECTION("Array Type")

tests/src/unit-bson.cpp

@@ -154,51 +154,6 @@ TEST_CASE("BSON")
 #endif
     }
 
-    SECTION("ill-formed UTF-8 (see #5529, #5651)")
-    {
-        // a BSON document {"s": "\xC0\xAE"} (0xC0 0xAE is an overlong
-        // encoding of '.'); the BSON spec does not require a decoder to
-        // reject ill-formed UTF-8 in a string value, so the reader hands the
-        // bytes back unchanged
-        const std::vector<uint8_t> v =
-        {
-            0x0F, 0x00, 0x00, 0x00, // document length
-            0x02, 's', 0x00,        // type 0x02 (string), key "s"
-            0x03, 0x00, 0x00, 0x00, // string length (including null)
-            0xc0, 0xae, 0x00,       // string content and its null terminator
-            0x00                    // document terminator
-        };
-        json j;
-        CHECK_NOTHROW(j = json::from_bson(v));
-        REQUIRE(j.is_object());
-        REQUIRE(j.contains("s"));
-        CHECK(j["s"].get_ref<const json::string_t&>() == std::string("\xc0\xae"));
-        // dump() still requires valid UTF-8 and throws for such a value
-        CHECK_THROWS_AS(j.dump(), json::type_error&);
-        // to_bson() is strict as well, so the value cannot be written back
-        CHECK_THROWS_AS(json::to_bson(j), json::type_error&);
-
-        // to_bson() rejects the same kind of ill-formed string value, before
-        // any bytes reach the output adapter (the BSON document length
-        // prefix must be known up front, so nothing is written incrementally)
-        std::vector<std::uint8_t> out{0x42}; // a sentinel byte the writer must not touch
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}, nlohmann::detail::output_adapter<std::uint8_t>(out)), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-        CHECK(out == std::vector<std::uint8_t> {0x42});
-
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xFF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-        // a truncated multi-byte sequence
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC3"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&);
-        // an encoded surrogate half (U+D800)
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xED\xA0\x80"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&);
-        // an overlong encoding of '.'
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"s", "\xC0\xAF"}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&);
-
-        // an object key with ill-formed UTF-8 is rejected as well; unlike
-        // the reader (which never validates element names), the writer
-        // checks both string values and object keys
-        CHECK_THROWS_WITH_AS(json::to_bson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-    }
-
     SECTION("lengths exceeding INT32_MAX cannot be serialized to BSON")
     {
         // out_of_range.412 is thrown from a single shared helper

tests/src/unit-cbor.cpp

@@ -1801,40 +1801,19 @@ TEST_CASE("CBOR")
             CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0xA1, 0x7C, 0x01})), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing CBOR string: expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x7C", json::parse_error&);
         }
 
-        SECTION("ill-formed UTF-8 in string (see #5529, #5651)")
+        SECTION("invalid UTF-8 in string (see #5529)")
         {
-            // RFC 8949 §3.1 leaves it up to the decoder whether to reject
-            // ill-formed UTF-8 in a text string; this library does not, and
-            // hands the original bytes back unchanged, matching the
-            // MessagePack reader and the behavior before #5185/#5531 (not in
-            // any release)
-
             // a two-character text string (major type 3) whose bytes are not
-            // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') round-trips
-            // byte for byte as a string value
-            const std::vector<uint8_t> ill_formed_value = {0x62, 0xc0, 0xae};
-            json j_value;
-            CHECK_NOTHROW(j_value = json::from_cbor(ill_formed_value));
-            REQUIRE(j_value.is_string());
-            CHECK(j_value.get_ref<const json::string_t&>() == std::string("\xc0\xae"));
-            // dump() still requires valid UTF-8 and throws for such a value,
-            // unless an error handler that replaces or ignores the bytes is
-            // passed
-            CHECK_THROWS_AS(j_value.dump(), json::type_error&);
-            // to_cbor() is strict as well, so the value cannot be written back
-            CHECK_THROWS_AS(json::to_cbor(j_value), json::type_error&);
-
-            // the same bytes as an object key round-trip as well
-            const std::vector<uint8_t> ill_formed_key = {0xa1, 0x62, 0xc0, 0xae, 0x01};
-            json j_key;
-            CHECK_NOTHROW(j_key = json::from_cbor(ill_formed_key));
-            REQUIRE(j_key.is_object());
-            CHECK(j_key.contains(std::string("\xc0\xae")));
-            CHECK_THROWS_AS(json::to_cbor(j_key), json::type_error&);
+            // valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be
+            // rejected at decode time, matching every other kind of
+            // malformed binary input, rather than only failing later when
+            // the resulting value is dumped
+            json _;
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x62, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
+            CHECK(json::from_cbor(std::vector<uint8_t>({0x62, 0xc0, 0xae}), true, false).is_discarded());
 
             // a CBOR byte string (major type 2) with the very same bytes is
             // NOT text and must still be accepted as-is
-            json _;
             CHECK_NOTHROW(_ = json::from_cbor(std::vector<uint8_t>({0x42, 0xc0, 0xae})));
             CHECK(_ == json::binary(std::vector<std::uint8_t>({0xc0, 0xae})));
 
@@ -1843,46 +1822,17 @@ TEST_CASE("CBOR")
             CHECK(json::from_cbor(json::to_cbor(j)) == j);
         }
 
-        SECTION("to_cbor rejects ill-formed UTF-8 (see #5651)")
-        {
-            // to_cbor() must reject the same ill-formed strings from_cbor()
-            // rejects, so a value it accepts can always be read back
-            CHECK_THROWS_WITH_AS(json::to_cbor(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-            // a truncated multi-byte sequence
-            CHECK_THROWS_WITH_AS(json::to_cbor(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&);
-            // an encoded surrogate half (U+D800)
-            CHECK_THROWS_WITH_AS(json::to_cbor(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&);
-            // an overlong encoding of '.'
-            CHECK_THROWS_WITH_AS(json::to_cbor(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&);
-
-            // an object key with ill-formed UTF-8 is rejected the same way
-            CHECK_THROWS_WITH_AS(json::to_cbor(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-
-            // binary values are not text and are unaffected
-            CHECK_NOTHROW(json::to_cbor(json::binary(std::vector<std::uint8_t>({0xFF}))));
-        }
-
-        SECTION("ill-formed UTF-8 in indefinite-length string")
+        SECTION("invalid UTF-8 in indefinite-length string")
         {
             json _;
 
-            // the chunks are concatenated as is, without checking that each
-            // chunk is valid UTF-8 on its own (RFC 8949, Section 3.2.3), so
-            // a code point split across two chunks yields a valid string
-            CHECK_NOTHROW(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})));
-            CHECK(_ == "\xc3\xa9");
-            CHECK(_.dump() == "\"\xc3\xa9\"");
+            // every chunk must be valid UTF-8 on its own (RFC 8949, Section
+            // 3.2.3), so a code point split across two chunks is rejected
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
+            CHECK(json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}), true, false).is_discarded());
 
-            // a truncated code point is kept as is
-            CHECK_NOTHROW(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0xff})));
-            CHECK(_ == "\xc3");
-            CHECK_THROWS_AS(_.dump(), json::type_error&);
-            CHECK_THROWS_AS(json::to_cbor(_), json::type_error&);
-
-            // an ill-formed later chunk is kept after valid ones
-            CHECK_NOTHROW(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})));
-            CHECK(_ == "\xc3\xa9\xc0\xae");
-            CHECK_THROWS_AS(_.dump(), json::type_error&);
+            // an ill-formed later chunk is rejected after valid ones
+            CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
 
             // valid multi-byte chunks are accepted
             CHECK(json::from_cbor(std::vector<uint8_t>({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6");
@@ -1890,6 +1840,9 @@ TEST_CASE("CBOR")
 
         SECTION("many chunks in indefinite-length string")
         {
+            // only the newly read chunk is validated, not the whole string
+            // collected so far; validating the latter made this input take
+            // quadratic time (about ten seconds for 100000 chunks)
             constexpr std::size_t chunks = 100000;
             std::vector<uint8_t> v{0x7f};
             for (std::size_t i = 0; i < chunks; ++i)

tests/src/unit-msgpack.cpp

@@ -1540,39 +1540,19 @@ TEST_CASE("MessagePack")
             CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector<uint8_t>({0x81})), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing MessagePack string: unexpected end of input", json::parse_error&);
         }
 
-        SECTION("ill-formed UTF-8 in string (see #5529, #5651)")
+        SECTION("invalid UTF-8 in string (see #5529)")
         {
-            // the MessagePack specification explicitly allows a str object to
-            // contain a byte sequence that is not valid UTF-8 and expects a
-            // deserializer to hand the original bytes back unchanged; this
-            // library follows that, unlike CBOR/UBJSON/BJData/BSON, whose
-            // specifications require text strings to be valid UTF-8
-
             // a fixstr of length 2 (0xA0 | 2) whose bytes are not valid UTF-8
-            // (0xC0 0xAE is an overlong encoding of '.') round-trips byte for
-            // byte as a string value
-            const std::vector<uint8_t> ill_formed_value = {0xa2, 0xc0, 0xae};
-            json j_value;
-            CHECK_NOTHROW(j_value = json::from_msgpack(ill_formed_value));
-            REQUIRE(j_value.is_string());
-            CHECK(j_value.get_ref<const json::string_t&>() == std::string("\xc0\xae"));
-            CHECK(json::from_msgpack(json::to_msgpack(j_value)) == j_value);
-            // dump() still requires valid UTF-8 and throws for such a value,
-            // unless an error handler that replaces or ignores the bytes is
-            // passed
-            CHECK_THROWS_AS(j_value.dump(), json::type_error&);
-
-            // the same bytes as an object key round-trip as well
-            const std::vector<uint8_t> ill_formed_key = {0x81, 0xa2, 0xc0, 0xae, 0x01};
-            json j_key;
-            CHECK_NOTHROW(j_key = json::from_msgpack(ill_formed_key));
-            REQUIRE(j_key.is_object());
-            CHECK(j_key.contains(std::string("\xc0\xae")));
-            CHECK(json::from_msgpack(json::to_msgpack(j_key)) == j_key);
+            // (0xC0 0xAE is an overlong encoding of '.') must be rejected at
+            // decode time, matching every other kind of malformed binary
+            // input, rather than only failing later when the resulting
+            // value is dumped
+            json _;
+            CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector<uint8_t>({0xa2, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing MessagePack string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
+            CHECK(json::from_msgpack(std::vector<uint8_t>({0xa2, 0xc0, 0xae}), true, false).is_discarded());
 
             // a MessagePack bin8 blob with the very same bytes is NOT text
             // and must still be accepted as-is
-            json _;
             CHECK_NOTHROW(_ = json::from_msgpack(std::vector<uint8_t>({0xc4, 0x02, 0xc0, 0xae})));
             CHECK(_ == json::binary(std::vector<std::uint8_t>({0xc0, 0xae})));
 

tests/src/unit-ubjson.cpp

@@ -2505,41 +2505,6 @@ TEST_CASE("Universal Binary JSON Specification Examples 1")
             CHECK(json::to_ubjson(j) == v);
             CHECK(json::from_ubjson(v) == j);
         }
-
-        SECTION("ill-formed UTF-8 (see #5529, #5651)")
-        {
-            // none of the binary format specs requires a decoder to reject
-            // ill-formed UTF-8 in a text string, so a value whose bytes are
-            // not valid UTF-8 (0xC0 0xAE is an overlong encoding of '.')
-            // round-trips byte for byte as a string value; to_ubjson() is
-            // strict, so such a value cannot be written back
-            const std::vector<uint8_t> v = {'S', 'i', 2, 0xc0, 0xae};
-            json j;
-            CHECK_NOTHROW(j = json::from_ubjson(v));
-            REQUIRE(j.is_string());
-            CHECK(j.get_ref<const json::string_t&>() == std::string("\xc0\xae"));
-            CHECK_THROWS_AS(j.dump(), json::type_error&);
-            CHECK_THROWS_AS(json::to_ubjson(j), json::type_error&);
-
-            // the same bytes as an object key round-trip as well
-            const std::vector<uint8_t> v_key = {'{', 'i', 2, 0xc0, 0xae, 'i', 1, '}'};
-            json j_key;
-            CHECK_NOTHROW(j_key = json::from_ubjson(v_key));
-            REQUIRE(j_key.is_object());
-            CHECK(j_key.contains(std::string("\xc0\xae")));
-            CHECK_THROWS_AS(json::to_ubjson(j_key), json::type_error&);
-
-            CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xFF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-            // a truncated multi-byte sequence
-            CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xC3")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC3", json::type_error&);
-            // an encoded surrogate half (U+D800)
-            CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xED\xA0\x80")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xED", json::type_error&);
-            // an overlong encoding of '.'
-            CHECK_THROWS_WITH_AS(json::to_ubjson(json("\xC0\xAF")), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xC0", json::type_error&);
-
-            // an object key with ill-formed UTF-8 is rejected the same way
-            CHECK_THROWS_WITH_AS(json::to_ubjson(json{{"\xFF", 1}}), "[json.exception.type_error.316] invalid UTF-8 byte at index 0: 0xFF", json::type_error&);
-        }
     }
 
     SECTION("Array Type")