BSON writer: write to_bson(j) in one pass, with sizes written afterwards

BSON prefixes every document and array with its size. to_bson() therefore
walked the value twice: once to compute every size, once to write. When
the output is the vector that to_bson(j) returns, the writer now puts a
placeholder in front of each document and array and writes its size there
once the document is complete, so one walk is enough.

Every key and value is still checked as the size pass checked it, in the
same order, so the same exception is thrown; the partial output is
discarded with the vector. If a length would not fit BSON's 32-bit size
fields, the writer drops what it has written and writes the document in
two passes, which reports that error before writing anything, as before.

The two passes remain for output adapters (streams, strings, vectors
passed by reference), for error_handler_t::replace and ignore (which
change sizes), and for integer types wider than 64 bits.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -4293,19 +4293,6 @@ class binary_reader
         return get_bytes(len, "string", result);
     }
 
-    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
-    /// sequence by sequence (as the binary writers check strings)
-    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
-    {
-        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
-    }
-
-    /// whether @a s, of a character type wider than a byte, is valid UTF-8
-    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
-    {
-        return is_valid_utf8(s);
-    }
-
     /*!
     @brief validate a decoded text string (value or object key) against @ref error_handler
 
@@ -4325,16 +4312,7 @@ class binary_reader
     */
     bool check_string_utf8(string_t& result, const char* context)
     {
-        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
-    }
-
-    /// @ref check_string_utf8 for an error handler other than @ref
-    /// error_handler_t::keep; out of line, so that reading strings without a
-    /// check does not pay for the code of the check
-    JSON_HEDLEY_NEVER_INLINE
-    bool check_string_utf8_handler(string_t& result, const char* context)
-    {
-        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
         {
             return true;
         }

include/nlohmann/detail/input/simd_scan.hpp

@@ -322,78 +322,6 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
     block_out = block;
     return false;
 }
-
-/*!
-@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
-
-Like vector_utf8_run, but quotes, backslashes, and control characters are
-ordinary characters. Returns the start of the first block in which the check
-found an error (which may belong to a sequence begun in the block before) or
-where fewer than 16 bytes are left; a sequence may be cut at that point, so the
-caller resumes at its start, one sequence at a time.
-*/
-JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
-{
-    using lookup = utf8_lookup4<>;
-#if JSON_SCAN_NEON
-    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
-    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
-    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
-    uint8x16_t prev = vdupq_n_u8(0);
-    while (e - block >= 16)
-    {
-        const uint8x16_t in = vld1q_u8(block);
-        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
-        {
-            // ASCII here, and no sequence from the block before to finish
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
-        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
-        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
-        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
-        if (vmaxvq_u8(err) != 0)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#else
-    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i nibble = _mm_set1_epi8(0x0F);
-    const __m128i zero = _mm_setzero_si128();
-    __m128i prev = zero;
-    while (e - block >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
-        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
-        {
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
-        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
-                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
-                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
-        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
-                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
-        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
-        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#endif
-    return block;
-}
 #endif
 
 }  // namespace detail

include/nlohmann/detail/input/string_scan.hpp

@@ -236,64 +236,6 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
     return pos;
 }
 
-/*!
-@brief whether [data, data+n) is valid UTF-8, for checking whole strings
-
-The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
-most common keys and values, are decided in one test, and long strings are
-checked 16 bytes at a time where the vector check is available.
-*/
-inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    if (n <= 16)
-    {
-        // two overlapping words cover 8 to 16 bytes, one partial word fewer
-        std::uint64_t first = 0;
-        std::uint64_t last = 0;
-        if (n >= 8)
-        {
-            std::memcpy(&first, data, sizeof(first));
-            std::memcpy(&last, data + n - 8, sizeof(last));
-        }
-        else if (n != 0)
-        {
-            std::memcpy(&first, data, n);
-        }
-        if (((first | last) & high) == 0)
-        {
-            return true;
-        }
-    }
-#if JSON_SCAN_VECTOR_UTF8
-    else if (n >= 32
-#if JSON_SCAN_SSSE3_DISPATCH
-             && cpu_has_ssse3()
-#endif
-            )
-    {
-        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
-        // the rest from the start of a sequence that may be cut at block
-        for (int i = 1; i <= 3 && block - i >= data; ++i)
-        {
-            const unsigned char c = block[-i];
-            if (c < 0x80)
-            {
-                break;
-            }
-            if (c >= 0xC0)
-            {
-                block -= i;
-                break;
-            }
-        }
-        const auto checked = static_cast<std::size_t>(block - data);
-        return checked + valid_utf8_prefix(block, n - checked) == n;
-    }
-#endif
-    return valid_utf8_prefix(data, n) == n;
-}
-
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a

include/nlohmann/detail/output/binary_writer.hpp

@@ -2088,7 +2088,8 @@ class binary_writer
         /// arrays: the first index with one more digit; only maintained by
         /// @ref calc_bson_sizes
         std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes only: where its size goes in the table
+        /// @ref calc_bson_sizes: where its size goes in the table; @ref
+        /// write_bson_document_single_pass: the position of its size field
         std::size_t size_slot;
         /// @ref calc_bson_sizes only: the size of its entries seen so far
         std::size_t entries_size = 0;
@@ -2339,6 +2340,197 @@ class binary_writer
         }
     }
 
+    template<typename SinkType>
+    using sink_written_t = decltype(std::declval<const SinkType&>().written());
+
+    /*!
+    @brief whether BSON documents can be written in a single pass: the sink
+           can overwrite what it has written (output_vector_sink), and every
+           integer fits the 64 bits BSON has, so that writing an array of
+           numbers needs no range check
+    */
+    using bson_single_pass_sink = std::integral_constant < bool,
+          is_detected<sink_written_t, OutputSinkType>::value
+          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
+          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
+          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
+          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
+
+    /// write a placeholder for the size of an object or array whose entries
+    /// follow; @return its position, for @ref patch_bson_size
+    std::size_t write_bson_size_placeholder()
+    {
+        const std::size_t pos = oa.written();
+        write_number<std::int32_t>(0, true);
+        return pos;
+    }
+
+    /// end the object or array whose size is at @a pos and write its size
+    /// there: everything from that field to the terminating null byte
+    /// @return false (and the size is not written) if it does not fit
+    bool close_bson_container(const std::size_t pos)
+    {
+        oa.write_character(to_char_type(0x00));
+        const std::size_t size = oa.written() - pos;
+        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
+        {
+            return false;
+        }
+        std::array<CharType, sizeof(std::int32_t)> bytes{};
+        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
+        oa.overwrite(pos, bytes.data(), bytes.size());
+        return true;
+    }
+
+    /*!
+    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
+           does, for @ref write_bson_document_single_pass
+    @return at least the size of the value
+    A number, boolean, or null needs no check there (its integers have at most
+    64 bits) and has at most 8 characters; a discarded value is rejected with
+    the same exception when it is written.
+    */
+    std::size_t check_bson_value(const BasicJsonType& j)
+    {
+        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
+    }
+
+    /// whether an entry of @a entry_size characters keeps the document that
+    /// begins at @a start, closed by @a open_containers null bytes, within
+    /// the 32-bit size of BSON
+    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
+    {
+        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
+        const std::size_t written = oa.written() - start;
+        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
+    }
+
+    /*!
+    @brief Serializes the JSON object @a document as a BSON document in a
+           single pass, writing the size of each object and array once it is
+           complete
+
+    Checks every key and value as @ref calc_bson_sizes does, in the same
+    order, so the same exception is thrown; the output written up to then is
+    discarded with the vector it was written to.
+
+    @return false as soon as the document or a string, binary value, object,
+            or array in it would be larger than the 32-bit BSON sizes allow;
+            nothing large has been written then, and the caller writes the
+            document in two passes, which reports that error
+    */
+    template < typename SinkType = OutputSinkType,
+               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
+    bool write_bson_document_single_pass(const BasicJsonType& document)
+    {
+        const std::size_t start = oa.written();
+        // the object or array whose entries are being written (with the
+        // position of its size), and the ones it is in
+        bson_frame current(&document, write_bson_size_placeholder());
+        std::vector<bson_frame> parents;
+
+        while (true)
+        {
+            const BasicJsonType* nested = nullptr;
+            std::size_t nested_size_pos = 0;
+            if (current.value->is_object())
+            {
+                const auto& object = *current.value->m_data.m_value.object;
+                while (nested == nullptr && current.member != object.cend())
+                {
+                    const auto& el = *current.member;
+                    ++current.member;
+                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
+                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
+                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
+                    {
+                        return false;
+                    }
+                    if (el.second.is_structured())
+                    {
+                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
+                        nested_size_pos = write_bson_size_placeholder();
+                        nested = &el.second;
+                    }
+                    else
+                    {
+                        write_bson_value(el.first, el.second);
+                    }
+                }
+            }
+            else
+            {
+                const auto& array = *current.value->m_data.m_value.array;
+                while (nested == nullptr && current.index < array.size())
+                {
+                    if (is_bson_fixed_size(array[current.index]))
+                    {
+                        write_bson_array_fixed_entries(current);
+                        if (current.index == array.size())
+                        {
+                            break;
+                        }
+                    }
+
+                    // a string, binary value, object or array
+                    const BasicJsonType& el = array[current.index];
+                    const string_t index_name(current.index_name.data(), current.index_name_size);
+                    current.next_index();
+                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
+                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
+                    {
+                        return false;
+                    }
+                    if (el.is_structured())
+                    {
+                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
+                        nested_size_pos = write_bson_size_placeholder();
+                        nested = &el;
+                    }
+                    else
+                    {
+                        write_bson_value(index_name, el);
+                    }
+                }
+            }
+
+            if (nested != nullptr)
+            {
+                if (nested->is_array())
+                {
+                    // an array of booleans, numbers and null is written without a frame on the stack
+                    bson_frame leaf(nested, nested_size_pos);
+                    write_bson_array_fixed_entries(leaf);
+                    if (leaf.index == nested->m_data.m_value.array->size())
+                    {
+                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
+                        {
+                            return false;
+                        }
+                        continue;
+                    }
+                    parents.push_back(std::move(current));
+                    current = std::move(leaf);
+                    continue;
+                }
+                parents.push_back(std::move(current));
+                current = bson_frame(nested, nested_size_pos);
+                continue;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
+            {
+                return false;
+            }
+            if (parents.empty())
+            {
+                return true;
+            }
+            current = std::move(parents.back());
+            parents.pop_back();
+        }
+    }
+
     /*!
     @brief Serializes the JSON object @a document as a BSON document
 
@@ -2350,6 +2542,39 @@ class binary_writer
     @pre       document.type() == value_t::object
     */
     void write_bson_document(const BasicJsonType& document)
+    {
+        write_bson_document(document, bson_single_pass_sink());
+    }
+
+    /// the sink can overwrite: one pass, unless the error handler replaces or
+    /// drops ill-formed UTF-8, which changes the sizes the checks compute
+    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
+    {
+        const std::size_t start = oa.written();
+        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
+                && write_bson_document_single_pass(document))
+        {
+            return;
+        }
+        // a length does not fit: start over, so that it is reported as the
+        // two passes report it, before anything is written
+        oa.rewind(start);
+        write_bson_document_two_pass(document);
+    }
+
+    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
+    {
+        write_bson_document_two_pass(document);
+    }
+
+    /*!
+    @brief Serializes the JSON object @a document as a BSON document: the
+           sizes of all objects and arrays first, then the entries
+
+    @param[in] document  the JSON object to serialize
+    @pre       document.type() == value_t::object
+    */
+    void write_bson_document_two_pass(const BasicJsonType& document)
     {
         std::vector<std::size_t> nested_sizes;
         const std::size_t document_size = calc_bson_sizes(document, nested_sizes);
@@ -3080,25 +3305,6 @@ class binary_writer
         }
     }
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 0x17)
-        {
-            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-        }
-        else
-        {
-            *p++ = to_char_type(0x78);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a CBOR text string
 
@@ -3120,7 +3326,19 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_cbor_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -3134,13 +3352,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_cbor_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -3155,27 +3366,6 @@ class binary_writer
     // MsgPack //
     /////////////
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 31)
-        {
-            // fixstr
-            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-        }
-        else
-        {
-            // str 8
-            *p++ = to_char_type(0xD9);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a MessagePack str
 
@@ -3193,7 +3383,21 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_msgpack_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -3206,13 +3410,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_msgpack_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -4822,9 +5019,10 @@ class binary_writer
     static void check_utf8(const string_t& s, const BasicJsonType& context)
     {
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
+        const std::size_t valid = valid_utf8_prefix(data, s.size());
+        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
         {
-            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
+            throw_invalid_utf8(data, valid, context);
         }
     }
 

include/nlohmann/detail/output/output_adapters.hpp

@@ -227,6 +227,29 @@ class output_vector_sink
     void flush() noexcept
     {}
 
+    /// the number of characters written so far, the position of the next one
+    std::size_t written() const noexcept
+    {
+        return position();
+    }
+
+    /// @brief replace @a length characters written before, from position @a pos
+    ///
+    /// Lets a writer put a length in front of data it has written since, such
+    /// as the size of a BSON document, without computing it in advance.
+    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
+    {
+        JSON_ASSERT(pos + length <= position());
+        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
+    }
+
+    /// drop the characters written from position @a pos on
+    void rewind(const std::size_t pos) noexcept
+    {
+        JSON_ASSERT(pos <= position());
+        cursor = v->data() + pos;
+    }
+
   private:
     /// number of characters of *v written so far
     std::size_t position() const noexcept

single_include/nlohmann/json.hpp

@@ -12377,78 +12377,6 @@ JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline bool vector_utf8_run(cons
     block_out = block;
     return false;
 }
-
-/*!
-@brief skip 16-byte blocks of well-formed UTF-8 at p (a character boundary)
-
-Like vector_utf8_run, but quotes, backslashes, and control characters are
-ordinary characters. Returns the start of the first block in which the check
-found an error (which may belong to a sequence begun in the block before) or
-where fewer than 16 bytes are left; a sequence may be cut at that point, so the
-caller resumes at its start, one sequence at a time.
-*/
-JSON_SCAN_SSSE3_TARGET JSON_HEDLEY_NEVER_INLINE inline const unsigned char* vector_utf8_valid_blocks(const unsigned char* block, const unsigned char* e) noexcept
-{
-    using lookup = utf8_lookup4<>;
-#if JSON_SCAN_NEON
-    const uint8x16_t t1h = vld1q_u8(lookup::byte_1_high.data());
-    const uint8x16_t t1l = vld1q_u8(lookup::byte_1_low.data());
-    const uint8x16_t t2h = vld1q_u8(lookup::byte_2_high.data());
-    uint8x16_t prev = vdupq_n_u8(0);
-    while (e - block >= 16)
-    {
-        const uint8x16_t in = vld1q_u8(block);
-        if (vmaxvq_u8(vorrq_u8(in, prev)) < 0x80)
-        {
-            // ASCII here, and no sequence from the block before to finish
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const uint8x16_t prev1 = vextq_u8(prev, in, 15);
-        const uint8x16_t sc = vandq_u8(vandq_u8(vqtbl1q_u8(t1h, vshrq_n_u8(prev1, 4)), vqtbl1q_u8(t1l, vandq_u8(prev1, vdupq_n_u8(0x0F)))), vqtbl1q_u8(t2h, vshrq_n_u8(in, 4)));
-        const uint8x16_t must23 = vorrq_u8(vqsubq_u8(vextq_u8(prev, in, 14), vdupq_n_u8(0xE0 - 0x80)), vqsubq_u8(vextq_u8(prev, in, 13), vdupq_n_u8(0xF0 - 0x80)));
-        const uint8x16_t err = veorq_u8(vandq_u8(must23, vdupq_n_u8(0x80)), sc);
-        if (vmaxvq_u8(err) != 0)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#else
-    const __m128i t1h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t1l = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_1_low.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i t2h = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(lookup::byte_2_high.data()))); // NOLINT(bugprone-casting-through-void)
-    const __m128i nibble = _mm_set1_epi8(0x0F);
-    const __m128i zero = _mm_setzero_si128();
-    __m128i prev = zero;
-    while (e - block >= 16)
-    {
-        const __m128i in = _mm_loadu_si128(static_cast<const __m128i*>(static_cast<const void*>(block))); // NOLINT(bugprone-casting-through-void)
-        if (_mm_movemask_epi8(_mm_or_si128(in, prev)) == 0)
-        {
-            prev = in;
-            block += 16;
-            continue;
-        }
-        const __m128i prev1 = _mm_alignr_epi8(in, prev, 15);
-        const __m128i sc = _mm_and_si128(_mm_and_si128(_mm_shuffle_epi8(t1h, _mm_and_si128(_mm_srli_epi16(prev1, 4), nibble)),
-                                         _mm_shuffle_epi8(t1l, _mm_and_si128(prev1, nibble))),
-                                         _mm_shuffle_epi8(t2h, _mm_and_si128(_mm_srli_epi16(in, 4), nibble)));
-        const __m128i must23 = _mm_or_si128(_mm_subs_epu8(_mm_alignr_epi8(in, prev, 14), _mm_set1_epi8(0xE0 - 0x80)),
-                                            _mm_subs_epu8(_mm_alignr_epi8(in, prev, 13), _mm_set1_epi8(0xF0 - 0x80)));
-        const __m128i err = _mm_xor_si128(_mm_and_si128(must23, _mm_set1_epi8(static_cast<char>(-128))), sc);
-        if (_mm_movemask_epi8(_mm_cmpeq_epi8(err, zero)) != 0xFFFF)
-        {
-            break;
-        }
-        prev = in;
-        block += 16;
-    }
-#endif
-    return block;
-}
 #endif
 
 }  // namespace detail
@@ -12676,64 +12604,6 @@ inline std::size_t valid_utf8_prefix(const unsigned char* data, std::size_t n) n
     return pos;
 }
 
-/*!
-@brief whether [data, data+n) is valid UTF-8, for checking whole strings
-
-The same answer as valid_utf8_prefix(data, n) == n. Short ASCII strings, the
-most common keys and values, are decided in one test, and long strings are
-checked 16 bytes at a time where the vector check is available.
-*/
-inline bool valid_utf8_string(const unsigned char* data, std::size_t n) noexcept
-{
-    constexpr std::uint64_t high = 0x8080808080808080ull;
-    if (n <= 16)
-    {
-        // two overlapping words cover 8 to 16 bytes, one partial word fewer
-        std::uint64_t first = 0;
-        std::uint64_t last = 0;
-        if (n >= 8)
-        {
-            std::memcpy(&first, data, sizeof(first));
-            std::memcpy(&last, data + n - 8, sizeof(last));
-        }
-        else if (n != 0)
-        {
-            std::memcpy(&first, data, n);
-        }
-        if (((first | last) & high) == 0)
-        {
-            return true;
-        }
-    }
-#if JSON_SCAN_VECTOR_UTF8
-    else if (n >= 32
-#if JSON_SCAN_SSSE3_DISPATCH
-             && cpu_has_ssse3()
-#endif
-            )
-    {
-        const unsigned char* block = vector_utf8_valid_blocks(data, data + n);
-        // the rest from the start of a sequence that may be cut at block
-        for (int i = 1; i <= 3 && block - i >= data; ++i)
-        {
-            const unsigned char c = block[-i];
-            if (c < 0x80)
-            {
-                break;
-            }
-            if (c >= 0xC0)
-            {
-                block -= i;
-                break;
-            }
-        }
-        const auto checked = static_cast<std::size_t>(block - data);
-        return checked + valid_utf8_prefix(block, n - checked) == n;
-    }
-#endif
-    return valid_utf8_prefix(data, n) == n;
-}
-
 #if JSON_SCAN_VECTOR_UTF8
 /// the end of scan_string_vector from block, where the vector loop stopped
 /// (ill-formed UTF-8, or fewer than 16 bytes left): one byte or sequence at a
@@ -22199,19 +22069,6 @@ class binary_reader
         return get_bytes(len, "string", result);
     }
 
-    /// whether @a s is valid UTF-8: ASCII eight bytes at a time, the rest
-    /// sequence by sequence (as the binary writers check strings)
-    static bool is_valid_utf8_string(const string_t& s, std::true_type /*is_byte*/) noexcept
-    {
-        return valid_utf8_string(reinterpret_cast<const unsigned char*>(s.data()), s.size());
-    }
-
-    /// whether @a s, of a character type wider than a byte, is valid UTF-8
-    static bool is_valid_utf8_string(const string_t& s, std::false_type /*is_byte*/) noexcept
-    {
-        return is_valid_utf8(s);
-    }
-
     /*!
     @brief validate a decoded text string (value or object key) against @ref error_handler
 
@@ -22231,16 +22088,7 @@ class binary_reader
     */
     bool check_string_utf8(string_t& result, const char* context)
     {
-        return error_handler == error_handler_t::keep || check_string_utf8_handler(result, context);
+        if (error_handler == error_handler_t::keep || is_valid_utf8(result))
-    }
-
-    /// @ref check_string_utf8 for an error handler other than @ref
-    /// error_handler_t::keep; out of line, so that reading strings without a
-    /// check does not pay for the code of the check
-    JSON_HEDLEY_NEVER_INLINE
-    bool check_string_utf8_handler(string_t& result, const char* context)
-    {
-        if (is_valid_utf8_string(result, std::integral_constant<bool, sizeof(typename string_t::value_type) == 1>()))
         {
             return true;
         }
@@ -27022,6 +26870,29 @@ class output_vector_sink
     void flush() noexcept
     {}
 
+    /// the number of characters written so far, the position of the next one
+    std::size_t written() const noexcept
+    {
+        return position();
+    }
+
+    /// @brief replace @a length characters written before, from position @a pos
+    ///
+    /// Lets a writer put a length in front of data it has written since, such
+    /// as the size of a BSON document, without computing it in advance.
+    void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept
+    {
+        JSON_ASSERT(pos + length <= position());
+        std::memcpy(v->data() + pos, s, length * sizeof(CharType));
+    }
+
+    /// drop the characters written from position @a pos on
+    void rewind(const std::size_t pos) noexcept
+    {
+        JSON_ASSERT(pos <= position());
+        cursor = v->data() + pos;
+    }
+
   private:
     /// number of characters of *v written so far
     std::size_t position() const noexcept
@@ -29466,7 +29337,8 @@ class binary_writer
         /// arrays: the first index with one more digit; only maintained by
         /// @ref calc_bson_sizes
         std::uint64_t index_boundary = 10;
-        /// @ref calc_bson_sizes only: where its size goes in the table
+        /// @ref calc_bson_sizes: where its size goes in the table; @ref
+        /// write_bson_document_single_pass: the position of its size field
         std::size_t size_slot;
         /// @ref calc_bson_sizes only: the size of its entries seen so far
         std::size_t entries_size = 0;
@@ -29717,6 +29589,197 @@ class binary_writer
         }
     }
 
+    template<typename SinkType>
+    using sink_written_t = decltype(std::declval<const SinkType&>().written());
+
+    /*!
+    @brief whether BSON documents can be written in a single pass: the sink
+           can overwrite what it has written (output_vector_sink), and every
+           integer fits the 64 bits BSON has, so that writing an array of
+           numbers needs no range check
+    */
+    using bson_single_pass_sink = std::integral_constant < bool,
+          is_detected<sink_written_t, OutputSinkType>::value
+          && std::numeric_limits<typename BasicJsonType::number_integer_t>::is_specialized
+          && std::numeric_limits<typename BasicJsonType::number_integer_t>::digits <= std::numeric_limits<std::int64_t>::digits
+          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::is_specialized
+          && std::numeric_limits<typename BasicJsonType::number_unsigned_t>::digits <= std::numeric_limits<std::uint64_t>::digits >;
+
+    /// write a placeholder for the size of an object or array whose entries
+    /// follow; @return its position, for @ref patch_bson_size
+    std::size_t write_bson_size_placeholder()
+    {
+        const std::size_t pos = oa.written();
+        write_number<std::int32_t>(0, true);
+        return pos;
+    }
+
+    /// end the object or array whose size is at @a pos and write its size
+    /// there: everything from that field to the terminating null byte
+    /// @return false (and the size is not written) if it does not fit
+    bool close_bson_container(const std::size_t pos)
+    {
+        oa.write_character(to_char_type(0x00));
+        const std::size_t size = oa.written() - pos;
+        if (JSON_HEDLEY_UNLIKELY(!value_in_range_of<std::int32_t>(size)))
+        {
+            return false;
+        }
+        std::array<CharType, sizeof(std::int32_t)> bytes{};
+        store_little_endian<std::int32_t>(bytes.data(), static_cast<std::int32_t>(size));
+        oa.overwrite(pos, bytes.data(), bytes.size());
+        return true;
+    }
+
+    /*!
+    @brief check the value @a j of a BSON entry as @ref calc_bson_value_size
+           does, for @ref write_bson_document_single_pass
+    @return at least the size of the value
+    A number, boolean, or null needs no check there (its integers have at most
+    64 bits) and has at most 8 characters; a discarded value is rejected with
+    the same exception when it is written.
+    */
+    std::size_t check_bson_value(const BasicJsonType& j)
+    {
+        return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8;
+    }
+
+    /// whether an entry of @a entry_size characters keeps the document that
+    /// begins at @a start, closed by @a open_containers null bytes, within
+    /// the 32-bit size of BSON
+    bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept
+    {
+        const std::size_t limit = static_cast<std::size_t>((std::numeric_limits<std::int32_t>::max)());
+        const std::size_t written = oa.written() - start;
+        return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size;
+    }
+
+    /*!
+    @brief Serializes the JSON object @a document as a BSON document in a
+           single pass, writing the size of each object and array once it is
+           complete
+
+    Checks every key and value as @ref calc_bson_sizes does, in the same
+    order, so the same exception is thrown; the output written up to then is
+    discarded with the vector it was written to.
+
+    @return false as soon as the document or a string, binary value, object,
+            or array in it would be larger than the 32-bit BSON sizes allow;
+            nothing large has been written then, and the caller writes the
+            document in two passes, which reports that error
+    */
+    template < typename SinkType = OutputSinkType,
+               typename std::enable_if < is_detected<sink_written_t, SinkType>::value, int >::type = 0 >
+    bool write_bson_document_single_pass(const BasicJsonType& document)
+    {
+        const std::size_t start = oa.written();
+        // the object or array whose entries are being written (with the
+        // position of its size), and the ones it is in
+        bson_frame current(&document, write_bson_size_placeholder());
+        std::vector<bson_frame> parents;
+
+        while (true)
+        {
+            const BasicJsonType* nested = nullptr;
+            std::size_t nested_size_pos = 0;
+            if (current.value->is_object())
+            {
+                const auto& object = *current.value->m_data.m_value.object;
+                while (nested == nullptr && current.member != object.cend())
+                {
+                    const auto& el = *current.member;
+                    ++current.member;
+                    const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second);
+                    const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second);
+                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1)))
+                    {
+                        return false;
+                    }
+                    if (el.second.is_structured())
+                    {
+                        write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04);
+                        nested_size_pos = write_bson_size_placeholder();
+                        nested = &el.second;
+                    }
+                    else
+                    {
+                        write_bson_value(el.first, el.second);
+                    }
+                }
+            }
+            else
+            {
+                const auto& array = *current.value->m_data.m_value.array;
+                while (nested == nullptr && current.index < array.size())
+                {
+                    if (is_bson_fixed_size(array[current.index]))
+                    {
+                        write_bson_array_fixed_entries(current);
+                        if (current.index == array.size())
+                        {
+                            break;
+                        }
+                    }
+
+                    // a string, binary value, object or array
+                    const BasicJsonType& el = array[current.index];
+                    const string_t index_name(current.index_name.data(), current.index_name_size);
+                    current.next_index();
+                    const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el);
+                    if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1)))
+                    {
+                        return false;
+                    }
+                    if (el.is_structured())
+                    {
+                        write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04);
+                        nested_size_pos = write_bson_size_placeholder();
+                        nested = &el;
+                    }
+                    else
+                    {
+                        write_bson_value(index_name, el);
+                    }
+                }
+            }
+
+            if (nested != nullptr)
+            {
+                if (nested->is_array())
+                {
+                    // an array of booleans, numbers and null is written without a frame on the stack
+                    bson_frame leaf(nested, nested_size_pos);
+                    write_bson_array_fixed_entries(leaf);
+                    if (leaf.index == nested->m_data.m_value.array->size())
+                    {
+                        if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos)))
+                        {
+                            return false;
+                        }
+                        continue;
+                    }
+                    parents.push_back(std::move(current));
+                    current = std::move(leaf);
+                    continue;
+                }
+                parents.push_back(std::move(current));
+                current = bson_frame(nested, nested_size_pos);
+                continue;
+            }
+
+            if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot)))
+            {
+                return false;
+            }
+            if (parents.empty())
+            {
+                return true;
+            }
+            current = std::move(parents.back());
+            parents.pop_back();
+        }
+    }
+
     /*!
     @brief Serializes the JSON object @a document as a BSON document
 
@@ -29728,6 +29791,39 @@ class binary_writer
     @pre       document.type() == value_t::object
     */
     void write_bson_document(const BasicJsonType& document)
+    {
+        write_bson_document(document, bson_single_pass_sink());
+    }
+
+    /// the sink can overwrite: one pass, unless the error handler replaces or
+    /// drops ill-formed UTF-8, which changes the sizes the checks compute
+    void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/)
+    {
+        const std::size_t start = oa.written();
+        if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict)
+                && write_bson_document_single_pass(document))
+        {
+            return;
+        }
+        // a length does not fit: start over, so that it is reported as the
+        // two passes report it, before anything is written
+        oa.rewind(start);
+        write_bson_document_two_pass(document);
+    }
+
+    void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/)
+    {
+        write_bson_document_two_pass(document);
+    }
+
+    /*!
+    @brief Serializes the JSON object @a document as a BSON document: the
+           sizes of all objects and arrays first, then the entries
+
+    @param[in] document  the JSON object to serialize
+    @pre       document.type() == value_t::object
+    */
+    void write_bson_document_two_pass(const BasicJsonType& document)
     {
         std::vector<std::size_t> nested_sizes;
         const std::size_t document_size = calc_bson_sizes(document, nested_sizes);
@@ -30458,25 +30554,6 @@ class binary_writer
         }
     }
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_cbor_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 0x17)
-        {
-            *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
-        }
-        else
-        {
-            *p++ = to_char_type(0x78);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a CBOR text string
 
@@ -30498,7 +30575,19 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_cbor_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 0x17)
+                {
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0x60 + value.size()));
+                }
+                else
+                {
+                    *p++ = to_char_type(0x78);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -30512,13 +30601,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_cbor_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_cbor_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -30533,27 +30615,6 @@ class binary_writer
     // MsgPack //
     /////////////
 
-    /// write the head and the characters of @a value, at most
-    /// max_text_in_reservation of them, in one reservation
-    JSON_HEDLEY_ALWAYS_INLINE void write_msgpack_string_reserved(const string_t& value)
-    {
-        CharType* const out = oa.reserve(value.size() + 2);
-        CharType* p = out;
-        if (value.size() <= 31)
-        {
-            // fixstr
-            *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
-        }
-        else
-        {
-            // str 8
-            *p++ = to_char_type(0xD9);
-            *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
-        }
-        copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
-        oa.commit(static_cast<std::size_t>(p - out) + value.size());
-    }
-
     /*!
     @brief write a MessagePack str
 
@@ -30571,7 +30632,21 @@ class binary_writer
             // the string is written as it is: head and characters in one reservation
             if (JSON_HEDLEY_LIKELY(value.size() <= max_text_in_reservation))
             {
-                write_msgpack_string_reserved(value);
+                CharType* const out = oa.reserve(value.size() + 2);
+                CharType* p = out;
+                if (value.size() <= 31)
+                {
+                    // fixstr
+                    *p++ = to_char_type(static_cast<std::uint8_t>(0xA0 | value.size()));
+                }
+                else
+                {
+                    // str 8
+                    *p++ = to_char_type(0xD9);
+                    *p++ = to_char_type(static_cast<std::uint8_t>(value.size()));
+                }
+                copy_characters(p, reinterpret_cast<const CharType*>(value.data()), value.size());
+                oa.commit(static_cast<std::size_t>(p - out) + value.size());
                 return;
             }
         }
@@ -30584,13 +30659,6 @@ class binary_writer
     JSON_HEDLEY_NEVER_INLINE
     void write_msgpack_checked_string(const string_t& value, const BasicJsonType& context)
     {
-        if (error_handler == error_handler_t::strict && value.size() <= max_text_in_reservation)
-        {
-            check_utf8(value, context);
-            write_msgpack_string_reserved(value);
-            return;
-        }
-
         string_t storage;
         const string_t& sanitized = sanitize_utf8_for_write(value, context, storage);
 
@@ -32200,9 +32268,10 @@ class binary_writer
     static void check_utf8(const string_t& s, const BasicJsonType& context)
     {
         const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        if (JSON_HEDLEY_UNLIKELY(!valid_utf8_string(data, s.size())))
+        const std::size_t valid = valid_utf8_prefix(data, s.size());
+        if (JSON_HEDLEY_UNLIKELY(valid != s.size()))
         {
-            throw_invalid_utf8(data, valid_utf8_prefix(data, s.size()), context);
+            throw_invalid_utf8(data, valid, context);
         }
     }
 

tests/src/unit-bson.cpp

@@ -2267,3 +2267,132 @@ TEST_CASE("BSON large strings and binaries (chunked reader)")
         }
     }
 }
+
+TEST_CASE("BSON written in one pass and in two passes")
+{
+    // to_bson() into a returned vector writes each size once the object or
+    // array is complete; through an output adapter, all sizes are computed
+    // first. Both must produce the same bytes and throw the same exceptions.
+    const auto two_pass = [](const json & j, const json::error_handler_t handler)
+    {
+        std::vector<std::uint8_t> result;
+        json::to_bson(j, result, handler);
+        return result;
+    };
+
+    const auto check_same = [&](const json & j, const json::error_handler_t handler)
+    {
+        CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace))
+        CAPTURE(static_cast<int>(handler))
+#if !defined(JSON_NOEXCEPTION)
+        std::string one_pass_error;
+        std::string two_pass_error;
+        std::vector<std::uint8_t> one;
+        std::vector<std::uint8_t> two;
+        try
+        {
+            one = json::to_bson(j, handler);
+        }
+        catch (const json::exception& e)
+        {
+            one_pass_error = e.what();
+        }
+        try
+        {
+            two = two_pass(j, handler);
+        }
+        catch (const json::exception& e)
+        {
+            two_pass_error = e.what();
+        }
+        CHECK(one_pass_error == two_pass_error);
+        CHECK(one == two);
+#else
+        CHECK(json::to_bson(j, handler) == two_pass(j, handler));
+#endif
+    };
+
+    const json::error_handler_t handlers[] = {json::error_handler_t::keep, json::error_handler_t::strict,
+                                              json::error_handler_t::replace, json::error_handler_t::ignore
+                                             };
+
+    SECTION("corpus")
+    {
+        for (const auto& value : utils::round_trip_corpus::values())
+        {
+            const json document = value.is_object() ? value : json{{"value", value}};
+            for (const auto handler : handlers)
+            {
+                check_same(document, handler);
+            }
+        }
+    }
+
+    SECTION("nesting and arrays of every length class")
+    {
+        json document = json::object();
+        document["empty object"] = json::object();
+        document["empty array"] = json::array();
+        for (const std::size_t n :
+                {
+                    std::size_t(1), std::size_t(9), std::size_t(10), std::size_t(11), std::size_t(99), std::size_t(100), std::size_t(101), std::size_t(1000)
+                })
+        {
+            json numbers = json::array();
+            json mixed = json::array();
+            for (std::size_t i = 0; i < n; ++i)
+            {
+                numbers.push_back(i % 3 == 0 ? json(static_cast<double>(i) / 3) : json(static_cast<std::int64_t>(i) * 1000003));
+                mixed.push_back(i % 4 == 0 ? json("s" + std::to_string(i)) : i % 4 == 1 ? json::array({i, json::object({{"k", i}})}) : i % 4 == 2 ? json(nullptr) : json(i % 2 == 0));
+            }
+            document["numbers " + std::to_string(n)] = numbers;
+            document["mixed " + std::to_string(n)] = mixed;
+            document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}});
+        }
+        document["binary"] = json::binary({1, 2, 3}, 0x80);
+        document["binary old"] = json::binary({1, 2, 3}, 0x02);
+        for (const auto handler : handlers)
+        {
+            check_same(document, handler);
+        }
+    }
+
+    SECTION("errors at every position")
+    {
+        // an entry that cannot be written, at each place among valid ones
+        const std::string bad_utf8 = "\xC3\x28";
+        const json bad_entries[] =
+        {
+            json::object({{std::string("a\0b", 3), 1}}),
+            json::object({{"s", bad_utf8}}),
+            json::object({{bad_utf8, 1}}),
+            json::array({1, 2, bad_utf8}),
+            json::object({{"b", json::binary({1}, 0x100)}}),
+            json::object({{"d", json(json::value_t::discarded)}}),
+            json::array({json(json::value_t::discarded)}),
+        };
+        for (const auto& bad : bad_entries)
+        {
+            for (std::size_t pos = 0; pos < 4; ++pos)
+            {
+                json document = json::object();
+                json list = json::array();
+                for (std::size_t i = 0; i < 4; ++i)
+                {
+                    if (i == pos)
+                    {
+                        document["x" + std::to_string(i)] = bad;
+                        list.push_back(bad);
+                    }
+                    document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})});
+                    list.push_back(i);
+                }
+                document["list"] = list;
+                for (const auto handler : handlers)
+                {
+                    check_same(document, handler);
+                }
+            }
+        }
+    }
+}

tests/src/unit-guard-page.cpp

@@ -403,7 +403,6 @@ TEST_CASE("guard page")
                     CHECK(nlohmann::detail::find_string_special(p, n) <= n);
                     CHECK(nlohmann::detail::find_ascii_copyable_run(p, n) <= n);
                     CHECK(nlohmann::detail::valid_utf8_prefix(p, n) <= n);
-                    CHECK(nlohmann::detail::valid_utf8_string(p, n) == (nlohmann::detail::valid_utf8_prefix(p, n) == n));
 #if defined(JSON_USE_SIMDUTF) && defined(JSON_HAS_CPP_17)
                     CHECK(nlohmann::detail::find_string_delimiter(p, n) <= n);
 #endif

tests/src/unit-unicode.cpp

@@ -13,7 +13,6 @@ using nlohmann::json;
 
 #include <algorithm>
 #include <fstream>
-#include <random>
 #include <string>
 #include "make_test_data_available.hpp"
 #include "test_utils.hpp"
@@ -1632,40 +1631,3 @@ TEST_CASE("Unicode RFC 3629")
 }
 
 DOCTEST_CLANG_SUPPRESS_WARNING_POP
-
-TEST_CASE("UTF-8 check of whole strings")
-{
-    // valid_utf8_string() decides short strings with a word test and long ones
-    // 16 bytes at a time where a vector check is available; it must agree with
-    // the byte-wise check for every mix of ASCII, sequences of each length,
-    // ill-formed and truncated sequences, at every length and offset
-    const char* const pieces[] =
-    {
-        "a", "\x7F", "\xC3\xA9", "\xE2\x82\xAC", "\xE0\xA0\x80", "\xEF\xBF\xBF", "\xF0\x9F\x98\x80", "\xF4\x8F\xBF\xBF",
-        "\x80", "\xC0\xAF", "\xC3", "\xE2\x82", "\xED\xA0\x80", "\xF0\x9F", "\xF4\x90\x80\x80", "\xFF"
-    };
-    // std::mt19937's sequence is fixed by the standard; no distribution is used
-    std::mt19937 rng(5);
-    std::size_t valid = 0;
-    for (int i = 0; i < 100000; ++i)
-    {
-        std::string s;
-        const std::size_t length = rng() % 100;
-        const std::uint32_t ascii = rng() % 101;
-        while (s.size() < length)
-        {
-            // mostly well-formed pieces (the first eight), so that long valid
-            // strings are common
-            s += (rng() % 100 < ascii) ? std::string("x") : std::string(pieces[rng() % (rng() % 8 == 0 ? 16 : 8)]);
-        }
-        const auto* data = reinterpret_cast<const unsigned char*>(s.data());
-        const bool expected = nlohmann::detail::is_valid_utf8(s);
-        valid += expected ? 1 : 0;
-        CAPTURE(s)
-        CHECK(nlohmann::detail::valid_utf8_string(data, s.size()) == expected);
-        CHECK((nlohmann::detail::valid_utf8_prefix(data, s.size()) == s.size()) == expected);
-    }
-    // both outcomes are common enough to be tested
-    CHECK(valid > 20000);
-    CHECK(valid < 80000);
-}