Merge branch 'perf/binary-reader-byteswap' into perf/msgpack-scalar-runs

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

include/nlohmann/detail/input/binary_reader.hpp

@@ -22,6 +22,7 @@
     #include <bit>  //byteswap
 #endif
 
+#include <nlohmann/detail/bit_ops.hpp>
 #include <nlohmann/detail/exceptions.hpp>
 #include <nlohmann/detail/input/input_adapters.hpp>
 #include <nlohmann/detail/input/json_sax.hpp>
@@ -741,11 +742,13 @@ class binary_reader
     bool get_cbor_negative_integer()
     {
         NumberType number{};
-        if (JSON_HEDLEY_UNLIKELY(!get_number(number)))
+        return get_number(number) && emit_cbor_negative_integer(number);
-        {
+    }
-            return false;
-        }
 
+    /// pass the CBOR negative integer -1 - @a number to the SAX parser
+    template<typename NumberType>
+    bool emit_cbor_negative_integer(const NumberType number)
+    {
         // the value is -1 - number, which fits into number_integer_t
         // whenever number does; the outer cast undoes the integral promotion
         // for number_integer_t types narrower than int
@@ -970,7 +973,7 @@ class binary_reader
             case 0x95:
             case 0x96:
             case 0x97:
-                return enter_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
+                return enter_cbor_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x1Fu));
 
             case 0x98: // array (one-byte uint8_t for n follows)
             case 0x99: // array (two-byte uint16_t for n follow)
@@ -979,7 +982,7 @@ class binary_reader
             {
                 std::uint64_t len{};
                 std::size_t size{};
-                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size);
+                return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size);
             }
 
             case 0x9F: // array (indefinite length)
@@ -1133,16 +1136,10 @@ class binary_reader
                 return get_half_float(false);
 
             case 0xFA: // Single-Precision Float (four-byte IEEE 754)
-            {
+                return get_cbor_float<float>();
-                float number{};
-                return get_number(number) && emit_float(number);
-            }
 
             case 0xFB: // Double-Precision Float (eight-byte IEEE 754)
-            {
+                return get_cbor_float<double>();
-                double number{};
-                return get_number(number) && emit_float(number);
-            }
 
             default: // anything else (0xFF is handled inside the other types)
             {
@@ -1151,6 +1148,14 @@ class binary_reader
         }
     }
 
+    /// read a single- or double-precision float
+    template<typename NumberType>
+    bool get_cbor_float()
+    {
+        NumberType number{};
+        return get_number(number) && emit_float(number);
+    }
+
     /*!
     @brief reports a nested indefinite-length CBOR string or byte array
     @param[in] type_name  name of the rejected string type
@@ -1676,6 +1681,183 @@ class binary_reader
         return true;
     }
 
+    /*!
+    @brief read a big-endian number of type @a NumberType at @a data
+    */
+    template<typename NumberType>
+    NumberType load_big_endian(const unsigned char* data) const noexcept
+    {
+        NumberType result{};
+        std::memcpy(&result, data, sizeof(NumberType));
+        if (is_little_endian)
+        {
+            byte_swap(result);
+        }
+        return result;
+    }
+
+    /*!
+    @brief read the integers, floats, booleans, and nulls that follow in a
+           definite-length array, without the per-element bookkeeping of
+           @ref parse_cbor_internal
+
+    Stops at the end of the array, at any other kind of value, and when fewer
+    than 9 bytes (the longest of these values) are left in the input, leaving
+    those to @ref parse_cbor_value, which reads them the same way.
+
+    The caller has checked that an element remains, that at least 9 bytes are
+    left, and that the next byte begins a scalar.
+
+    @param[in,out] remaining  the number of elements of the array not yet read
+    @return whether the SAX parser accepted the values
+    */
+    bool parse_cbor_scalar_run(std::size_t& remaining)
+    {
+        do
+        {
+            const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
+            const unsigned char head = data[0];
+            std::size_t size = 0;
+            // false positive: every case assigns accepted before it is read
+            // @infer-ignore DEAD_STORE
+            bool accepted = true;
+            switch (head)
+            {
+                case 0x18:
+                    size = 2;
+                    accepted = emit_unsigned(data[1]);
+                    break;
+                case 0x19:
+                    size = 3;
+                    accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
+                    break;
+                case 0x1A:
+                    size = 5;
+                    accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
+                    break;
+                case 0x1B:
+                    size = 9;
+                    accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
+                    break;
+                case 0x38:
+                    size = 2;
+                    accepted = emit_cbor_negative_integer(data[1]);
+                    break;
+                case 0x39:
+                    size = 3;
+                    accepted = emit_cbor_negative_integer(load_big_endian<std::uint16_t>(data + 1));
+                    break;
+                case 0x3A:
+                    size = 5;
+                    accepted = emit_cbor_negative_integer(load_big_endian<std::uint32_t>(data + 1));
+                    break;
+                case 0x3B:
+                    size = 9;
+                    accepted = emit_cbor_negative_integer(load_big_endian<std::uint64_t>(data + 1));
+                    break;
+                case 0xF4:
+                    size = 1;
+                    accepted = sax->boolean(false);
+                    break;
+                case 0xF5:
+                    size = 1;
+                    accepted = sax->boolean(true);
+                    break;
+                case 0xF6:
+                    size = 1;
+                    accepted = sax->null();
+                    break;
+                case 0xFA:
+                    size = 5;
+                    accepted = emit_float(load_big_endian<float>(data + 1));
+                    break;
+                case 0xFB:
+                    size = 9;
+                    accepted = emit_float(load_big_endian<double>(data + 1));
+                    break;
+                default:
+                    if (head <= 0x17)
+                    {
+                        size = 1;
+                        accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
+                    }
+                    else if (head >= 0x20 && head <= 0x37)
+                    {
+                        size = 1;
+                        accepted = sax->number_integer(static_cast<std::int8_t>(0x20 - 1 - head));
+                    }
+                    else
+                    {
+                        // any other value is read by parse_cbor_value
+                        return true;
+                    }
+                    break;
+            }
+            // as if read by get() and get_number(): current is the head byte
+            ia.bulk_skip(size);
+            chars_read += size;
+            current = head;
+            --remaining;
+            if (JSON_HEDLEY_UNLIKELY(!accepted))
+            {
+                return false;
+            }
+        }
+        while (remaining != 0 && ia.bulk_remaining() >= 9);
+        return true;
+    }
+
+    /*!
+    @brief open a definite-length array and read the scalars it begins with
+           (see @ref parse_cbor_scalar_run)
+
+    Only the start of an array is tried, which covers arrays of numbers
+    (e.g. coordinates) but costs arrays of other values only one test, not one
+    per element.
+
+    @param[in] len  the number of elements
+    @return whether the SAX parser accepted the array and the values read
+    */
+    bool enter_cbor_array(const std::size_t len)
+    {
+        return enter_array(len) && (len == 0 || !cbor_scalar_follows(std::integral_constant<bool, bulk_scan> {})
+                                    || parse_cbor_array_start(std::integral_constant<bool, bulk_scan> {}));
+    }
+
+    /// whether a run of scalars can start at the next byte: at least 9 bytes
+    /// are left and the byte begins an integer, a float, a boolean, or null;
+    /// one test, so that an array of containers or strings makes no call
+    bool cbor_scalar_follows(std::true_type /*bulk*/) const
+    {
+        if (ia.bulk_remaining() < 9)
+        {
+            return false;
+        }
+        const auto head = static_cast<unsigned char>(*ia.bulk_data());
+        return head < 0x40 || head >= 0xF4;
+    }
+
+    /// input that is not contiguous: every value is read by parse_cbor_value
+    bool cbor_scalar_follows(std::false_type /*bulk*/) const noexcept
+    {
+        return false;
+    }
+
+    /// read the scalars the array just opened begins with; out of line, so
+    /// that @ref parse_cbor_value stays small enough to be inlined into
+    /// @ref parse_cbor_internal
+    JSON_HEDLEY_NEVER_INLINE
+    bool parse_cbor_array_start(std::true_type /*bulk*/)
+    {
+        return parse_cbor_scalar_run(container_stack.back().remaining);
+    }
+
+    /// input that is not contiguous: every value is read by parse_cbor_value
+    bool parse_cbor_array_start(std::false_type /*bulk*/) const noexcept
+    {
+        return true;
+    }
+
     /*!
     @brief read a CBOR value and everything nested inside it
 
@@ -1705,6 +1887,7 @@ class binary_reader
                 // pop_back() below, which destroys the container_stack element
                 // it would otherwise alias
                 const container_frame top = container_stack.back();
+                const bool is_object = top.is_object;
                 bool at_end = false;
 
                 if (top.remaining != npos)
@@ -1716,7 +1899,7 @@ class binary_reader
                     {
                         // claim the element about to be read
                         --container_stack.back().remaining;
-                        if (top.is_object)
+                        if (is_object)
                         {
                             get();
                         }
@@ -1729,7 +1912,7 @@ class binary_reader
                     // Testing for it consumes a byte, which is the first byte
                     // of the next element when it is not one.
                     at_end = (get() == 0xFF);
-                    fetch = top.is_object;
+                    fetch = is_object;
                 }
 
                 if (at_end)
@@ -1746,7 +1929,7 @@ class binary_reader
                     continue;
                 }
 
-                if (top.is_object)
+                if (is_object)
                 {
                     if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler)))
                     {
@@ -1971,7 +2154,7 @@ class binary_reader
             case 0x9D:
             case 0x9E:
             case 0x9F:
-                return enter_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
+                return enter_msgpack_array(conditional_static_cast<std::size_t>(static_cast<unsigned int>(current) & 0x0Fu));
 
             // fixstr
             case 0xA0:
@@ -2102,13 +2285,13 @@ class binary_reader
             case 0xDC: // array 16
             {
                 std::uint16_t len{};
-                return get_number(len) && enter_array(static_cast<std::size_t>(len));
+                return get_number(len) && enter_msgpack_array(static_cast<std::size_t>(len));
             }
 
             case 0xDD: // array 32
             {
                 std::uint32_t len{};
-                return get_number(len) && enter_array(conditional_static_cast<std::size_t>(len));
+                return get_number(len) && enter_msgpack_array(conditional_static_cast<std::size_t>(len));
             }
 
             case 0xDE: // map 16
@@ -2450,6 +2633,165 @@ class binary_reader
         }
     }
 
+    /*!
+    @brief read the integers, floats, booleans, and nils that follow in a
+           MessagePack array, without the per-element bookkeeping of
+           @ref parse_msgpack_internal
+
+    Stops at the end of the array, at any other kind of value, and when fewer
+    than 9 bytes (the longest of these values) are left in the input, leaving
+    those to @ref parse_msgpack_value, which reads them the same way.
+
+    The caller has checked that an element remains, that at least 9 bytes are
+    left, and that the next byte begins a scalar.
+
+    @param[in,out] remaining  the number of elements of the array not yet read
+    @return whether the SAX parser accepted the values
+    */
+    bool parse_msgpack_scalar_run(std::size_t& remaining)
+    {
+        do
+        {
+            const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
+            const unsigned char head = data[0];
+            std::size_t size = 0;
+            // false positive: every case assigns accepted before it is read
+            // @infer-ignore DEAD_STORE
+            bool accepted = true;
+            switch (head)
+            {
+                case 0xC0: // nil
+                    size = 1;
+                    accepted = sax->null();
+                    break;
+                case 0xC2: // false
+                    size = 1;
+                    accepted = sax->boolean(false);
+                    break;
+                case 0xC3: // true
+                    size = 1;
+                    accepted = sax->boolean(true);
+                    break;
+                case 0xCA: // float 32
+                    size = 5;
+                    accepted = emit_float(load_big_endian<float>(data + 1));
+                    break;
+                case 0xCB: // float 64
+                    size = 9;
+                    accepted = emit_float(load_big_endian<double>(data + 1));
+                    break;
+                case 0xCC: // uint 8
+                    size = 2;
+                    accepted = emit_unsigned(data[1]);
+                    break;
+                case 0xCD: // uint 16
+                    size = 3;
+                    accepted = emit_unsigned(load_big_endian<std::uint16_t>(data + 1));
+                    break;
+                case 0xCE: // uint 32
+                    size = 5;
+                    accepted = emit_unsigned(load_big_endian<std::uint32_t>(data + 1));
+                    break;
+                case 0xCF: // uint 64
+                    size = 9;
+                    accepted = emit_unsigned(load_big_endian<std::uint64_t>(data + 1));
+                    break;
+                case 0xD0: // int 8
+                    size = 2;
+                    accepted = emit_signed(static_cast<std::int8_t>(data[1]));
+                    break;
+                case 0xD1: // int 16
+                    size = 3;
+                    accepted = emit_signed(load_big_endian<std::int16_t>(data + 1));
+                    break;
+                case 0xD2: // int 32
+                    size = 5;
+                    accepted = emit_signed(load_big_endian<std::int32_t>(data + 1));
+                    break;
+                case 0xD3: // int 64
+                    size = 9;
+                    accepted = emit_signed(load_big_endian<std::int64_t>(data + 1));
+                    break;
+                default:
+                    if (head <= 0x7F)
+                    {
+                        // positive fixint
+                        size = 1;
+                        accepted = sax->number_unsigned(static_cast<number_unsigned_t>(head));
+                    }
+                    else if (head >= 0xE0)
+                    {
+                        // negative fixint
+                        size = 1;
+                        accepted = sax->number_integer(static_cast<std::int8_t>(head));
+                    }
+                    else
+                    {
+                        // any other value is read by parse_msgpack_value
+                        return true;
+                    }
+                    break;
+            }
+            // as if read by get() and get_number(): current is the last byte
+            ia.bulk_skip(size);
+            chars_read += size;
+            current = data[size - 1];
+            --remaining;
+            if (JSON_HEDLEY_UNLIKELY(!accepted))
+            {
+                return false;
+            }
+        }
+        while (remaining != 0 && ia.bulk_remaining() >= 9);
+        return true;
+    }
+
+    /*!
+    @brief open a MessagePack array and read the scalars it begins with (see
+           @ref parse_msgpack_scalar_run)
+    @param[in] len  the number of elements
+    @return whether the SAX parser accepted the array and the values read
+    */
+    bool enter_msgpack_array(const std::size_t len)
+    {
+        return enter_array(len) && (len == 0 || !msgpack_scalar_follows(std::integral_constant<bool, bulk_scan> {})
+                                    || parse_msgpack_array_start(std::integral_constant<bool, bulk_scan> {}));
+    }
+
+    /// whether a run of scalars can start at the next byte: at least 9 bytes
+    /// are left and the byte begins a fixint, nil, a boolean, a float, or a
+    /// sized integer (0xC0-0xD3); one test, so that an array of containers or
+    /// strings makes no call
+    bool msgpack_scalar_follows(std::true_type /*bulk*/) const
+    {
+        if (ia.bulk_remaining() < 9)
+        {
+            return false;
+        }
+        const auto head = static_cast<unsigned char>(*ia.bulk_data());
+        return head < 0x80 || head >= 0xE0 || (head >= 0xC0 && head <= 0xD3);
+    }
+
+    /// input that is not contiguous: every value is read by parse_msgpack_value
+    bool msgpack_scalar_follows(std::false_type /*bulk*/) const noexcept
+    {
+        return false;
+    }
+
+    /// read the scalars the array just opened begins with; out of line, so
+    /// that @ref parse_msgpack_value stays small enough to be inlined
+    JSON_HEDLEY_NEVER_INLINE
+    bool parse_msgpack_array_start(std::true_type /*bulk*/)
+    {
+        return parse_msgpack_scalar_run(container_stack.back().remaining);
+    }
+
+    /// input that is not contiguous: every value is read by parse_msgpack_value
+    bool parse_msgpack_array_start(std::false_type /*bulk*/) const noexcept
+    {
+        return true;
+    }
+
     /*!
     @brief read a MessagePack value and everything nested inside it
 
@@ -3691,13 +4033,13 @@ class binary_reader
                 return sax->null();
 
             case 0xFB:
-                return sax->number_float(static_cast<number_float_t>(-1.0), "");
+                return sax->number_float(static_cast<number_float_t>(-1.0), no_source);
 
             case 0xFC:
-                return sax->number_float(static_cast<number_float_t>(0.0), "");
+                return sax->number_float(static_cast<number_float_t>(0.0), no_source);
 
             case 0xFD:
-                return sax->number_float(static_cast<number_float_t>(1.0), "");
+                return sax->number_float(static_cast<number_float_t>(1.0), no_source);
 
             case 0xFF: // empty string
             {
@@ -4020,7 +4362,12 @@ class binary_reader
     `char_traits<char_type>::eof()` in that case.
 
     @return character read from the input
+
+    @note always inlined: GCC otherwise stops inlining the stream adapter's
+          get_character() into parse_cbor_internal once that function grows
+          (+0.45 % instructions for CBOR from a std::istringstream with GCC 13)
     */
+    JSON_HEDLEY_ALWAYS_INLINE
     char_int_type get()
     {
         ++chars_read;
@@ -4066,6 +4413,27 @@ class binary_reader
         return current;
     }
 
+    /// reverse the byte order of a double with a single byteswap(): GCC does
+    /// not merge the byte-wise swap of the generic version into one
+    static void byte_swap(double& number) noexcept
+    {
+        static_assert(sizeof(double) == sizeof(std::uint64_t), "double must have 64 bits");
+        std::uint64_t bits{};
+        std::memcpy(&bits, &number, sizeof(bits));
+        bits = byteswap(bits);
+        std::memcpy(&number, &bits, sizeof(bits));
+    }
+
+    /// reverse the byte order of a float with a single byteswap()
+    static void byte_swap(float& number) noexcept
+    {
+        static_assert(sizeof(float) == sizeof(std::uint32_t), "float must have 32 bits");
+        std::uint32_t bits{};
+        std::memcpy(&bits, &number, sizeof(bits));
+        bits = byteswap(bits);
+        std::memcpy(&number, &bits, sizeof(bits));
+    }
+
     template<class NumberType>
     static void byte_swap(NumberType& number)
     {
@@ -4201,7 +4569,7 @@ class binary_reader
         {
             return out_of_range_error(exception_id::number_overflow, "number overflow", "value");
         }
-        return sax->number_float(result, "");
+        return sax->number_float(result, no_source);
     }
 
     /*!
@@ -4261,7 +4629,7 @@ class binary_reader
         }();
         return sax->number_float((half & 0x8000u) != 0
                                  ? static_cast<number_float_t>(-val)
-                                 : static_cast<number_float_t>(val), "");
+                                 : static_cast<number_float_t>(val), no_source);
     }
 
     /*!
@@ -4586,6 +4954,10 @@ class binary_reader
     /// the SAX parser
     json_sax_t* sax = nullptr;
 
+    /// the (empty) source text of every number_float event: binary formats
+    /// have none, and a string_t built for each float would cost more
+    const string_t no_source{};
+
     /// the containers that have been opened and not closed yet; see @ref container_frame
     std::vector<container_frame> container_stack{};
 

include/nlohmann/detail/input/lexer.hpp

@@ -209,13 +209,8 @@ constexpr bool input_adapter_bulk_window(std::false_type /*detected*/)
 @brief lexical analysis
 
 This class organizes the lexical analysis during JSON deserialization.
-
-DiscardValues: the caller only needs the token types and never looks at the values of number and string tokens (only
-accept(), which parses through json_sax_acceptor). A template parameter rather than a member, so that the lexer of
-parse() is compiled without the paths of accept(): with both string scans in one scan(), parse() was up to 2 % slower
-on Apple M1.
 */
-template<typename BasicJsonType, typename InputAdapterType, bool DiscardValues = false>
+template<typename BasicJsonType, typename InputAdapterType>
 class lexer : public lexer_base<BasicJsonType>
 {
     using number_integer_t = typename BasicJsonType::number_integer_t;
@@ -273,9 +268,10 @@ class lexer : public lexer_base<BasicJsonType>
   public:
     using token_type = typename lexer_base<BasicJsonType>::token_type;
 
-    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false) noexcept
+    explicit lexer(InputAdapterType&& adapter, bool ignore_comments_ = false, bool discard_number_values_ = false) noexcept
         : ia(std::move(adapter))
         , ignore_comments(ignore_comments_)
+        , discard_number_values(discard_number_values_)
     {
         // the token string of an error before the first token is everything
         // read so far
@@ -392,8 +388,6 @@ class lexer : public lexer_base<BasicJsonType>
     checks if it is inside the range. If a violation was detected, set up an
     error message and return false. Otherwise, return true.
 
-    @param[in] keep    whether the bytes are added to token_buffer (false for
-                       accept(), see scan())
     @param[in] ranges  list of integers; interpreted as list of pairs of
                        inclusive lower and upper bound, respectively
 
@@ -402,18 +396,17 @@ class lexer : public lexer_base<BasicJsonType>
 
     @return true if and only if no range violation was detected
     */
-    template<bool Keep>
+    bool next_byte_in_range(std::initializer_list<char_int_type> ranges)
-    bool next_byte_in_range(std::integral_constant<bool, Keep> keep, std::initializer_list<char_int_type> ranges)
     {
         JSON_ASSERT(ranges.size() == 2 || ranges.size() == 4 || ranges.size() == 6);
-        add_string_char(keep, current);
+        add(current);
 
         for (auto range = ranges.begin(); range != ranges.end(); ++range)
         {
             get();
             if (JSON_HEDLEY_LIKELY(*range <= current && current <= *(++range))) // NOLINT(bugprone-inc-dec-in-conditions)
             {
-                add_string_char(keep, current);
+                add(current);
             }
             else
             {
@@ -426,16 +419,14 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// contiguous input: bulk-append the run of ordinary characters and complete
-    /// well-formed UTF-8 sequences starting at the current read position (only
+    /// well-formed UTF-8 sequences starting at the current read position, then
-    /// skip it if @a keep is false), then
     /// read the first byte that needs individual handling (the closing quote, an
     /// escape, a control character, or an ill-formed UTF-8 byte) like get().
     /// That byte is taken from the buffer that was just scanned. Through get(),
     /// Apple Clang merged this read with the other get() calls of the scan
     /// function, which load the read position again right after it was stored
     /// (3% of json::accept() on jeopardy.json on Apple M1).
-    template<bool Keep>
+    char_int_type get_after_string_run(std::true_type /*bulk*/)
-    char_int_type get_after_string_run(std::true_type /*bulk*/, std::integral_constant<bool, Keep> keep)
     {
         // a pending unget must be consumed through the normal path first
         if (next_unget)
@@ -453,7 +444,7 @@ class lexer : public lexer_base<BasicJsonType>
         const std::size_t pos = lexer_string_bulk_run(data, remaining);
         if (pos != 0)
         {
-            append_string_run(keep, data, pos);
+            token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), pos);
             bulk_consume(reinterpret_cast<const char*>(data), pos);
         }
         if (pos == remaining)
@@ -497,8 +488,7 @@ class lexer : public lexer_base<BasicJsonType>
     }
 
     /// streaming input: no bulk fast path
-    template<bool Keep>
+    char_int_type get_after_string_run(std::false_type /*bulk*/)
-    char_int_type get_after_string_run(std::false_type /*bulk*/, std::integral_constant<bool, Keep> /*keep*/)
     {
         return get();
     }
@@ -512,18 +502,13 @@ class lexer : public lexer_base<BasicJsonType>
     may contain \0 bytes), and token_buffer.size() is the number of bytes in the
     string.
 
-    @param[in] keep  whether the string is copied into token_buffer; if false
-                     (accept(), see scan()), it is only checked, with the same
-                     errors, and token_buffer stays empty
-
     @return token_type::value_string if string could be successfully scanned,
             token_type::parse_error otherwise
 
     @note In case of errors, variable error_message contains a textual
           description.
     */
-    template<bool Keep>
+    token_type scan_string()
-    token_type scan_string(std::integral_constant<bool, Keep> keep)
     {
         // reset token_buffer (ignore opening quote)
         reset();
@@ -535,7 +520,7 @@ class lexer : public lexer_base<BasicJsonType>
         {
             // bulk-consume ordinary characters from contiguous input, then
             // get the next character, the one that ends the run
-            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>(), keep))
+            switch (get_after_string_run(std::integral_constant<bool, bulk_scan>()))
             {
                 // end of file while parsing the string
                 case char_traits<char_type>::eof():
@@ -557,35 +542,35 @@ class lexer : public lexer_base<BasicJsonType>
                     {
                         // quotation mark
                         case '\"':
-                            add_string_char(keep, '\"');
+                            add('\"');
                             break;
                         // reverse solidus
                         case '\\':
-                            add_string_char(keep, '\\');
+                            add('\\');
                             break;
                         // solidus
                         case '/':
-                            add_string_char(keep, '/');
+                            add('/');
                             break;
                         // backspace
                         case 'b':
-                            add_string_char(keep, '\b');
+                            add('\b');
                             break;
                         // form feed
                         case 'f':
-                            add_string_char(keep, '\f');
+                            add('\f');
                             break;
                         // line feed
                         case 'n':
-                            add_string_char(keep, '\n');
+                            add('\n');
                             break;
                         // carriage return
                         case 'r':
-                            add_string_char(keep, '\r');
+                            add('\r');
                             break;
                         // tab
                         case 't':
-                            add_string_char(keep, '\t');
+                            add('\t');
                             break;
 
                         // unicode escapes
@@ -653,7 +638,10 @@ class lexer : public lexer_base<BasicJsonType>
                             JSON_ASSERT(0x00 <= codepoint && codepoint <= 0x10FFFF);
 
                             // translate codepoint into bytes
-                            add_codepoint(keep, codepoint);
+                            encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
+                            {
+                                add(static_cast<char_int_type>(byte));
+                            });
 
                             break;
                         }
@@ -836,7 +824,7 @@ class lexer : public lexer_base<BasicJsonType>
                 case 0x7E:
                 case 0x7F:
                 {
-                    add_string_char(keep, current);
+                    add(current);
                     break;
                 }
 
@@ -872,7 +860,7 @@ class lexer : public lexer_base<BasicJsonType>
                 case 0xDE:
                 case 0xDF:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range(keep, {0x80, 0xBF})))
+                    if (JSON_HEDLEY_UNLIKELY(!next_byte_in_range({0x80, 0xBF})))
                     {
                         return token_type::parse_error;
                     }
@@ -882,7 +870,7 @@ class lexer : public lexer_base<BasicJsonType>
                 // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
                 case 0xE0:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0xA0, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -906,7 +894,7 @@ class lexer : public lexer_base<BasicJsonType>
                 case 0xEE:
                 case 0xEF:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -916,7 +904,7 @@ class lexer : public lexer_base<BasicJsonType>
                 // U+D000..U+D7FF: bytes ED 80..9F 80..BF
                 case 0xED:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x9F, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -926,7 +914,7 @@ class lexer : public lexer_base<BasicJsonType>
                 // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
                 case 0xF0:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -938,7 +926,7 @@ class lexer : public lexer_base<BasicJsonType>
                 case 0xF2:
                 case 0xF3:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -948,7 +936,7 @@ class lexer : public lexer_base<BasicJsonType>
                 // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
                 case 0xF4:
                 {
-                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range(keep, {0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    if (JSON_HEDLEY_UNLIKELY(!(next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
                     {
                         return token_type::parse_error;
                     }
@@ -1439,7 +1427,7 @@ scan_number_done:
     token_type convert_number(token_type number_type, std::size_t mantissa_end)
     {
         // accept() only needs to know whether the input is valid, so it sets
-        // discard_values (see json.hpp), and an integer token whose
+        // discard_number_values (see json.hpp), and an integer token whose
         // digit count shows that it fits is reported without calling
         // convert_integer(). A number with up to 18 digits always fits into
         // both std::uint64_t and std::int64_t (18 nines is about 1e18, below
@@ -1454,7 +1442,7 @@ scan_number_done:
         // json_sax_acceptor, whose number callbacks discard their argument and
         // return true, and the parser rejects neither integers nor finite
         // floats. value_unsigned/value_integer are left unset here, so a caller
-        // that reads the converted value must not set discard_values.
+        // that reads the converted value must not set discard_number_values.
         //
         // On contiguous input, scan_number_bulk_contiguous() converts integer
         // tokens itself and does not pass them to this function, unless
@@ -1462,7 +1450,7 @@ scan_number_done:
         // reached for input without bulk access (e.g. streams), with
         // JSON_DIAGNOSTIC_POSITIONS, or when scan_number_bulk_contiguous()
         // falls back to scan_number().
-        if (discard_values())
+        if (discard_number_values)
         {
             constexpr std::size_t safe_digit_count = 18;
             if (number_type == token_type::value_unsigned && token_buffer.size() <= safe_digit_count)
@@ -1713,7 +1701,7 @@ scan_number_done:
         // w < 10^19, so a value below 10^max_exponent10 cannot overflow, and a
         // value that underflows is a finite number as well. Only a token that
         // may overflow has to be converted to find out.
-        if (JSON_HEDLEY_UNLIKELY(discard_values() && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
+        if (JSON_HEDLEY_UNLIKELY(discard_number_values && s.exponent + 19 <= std::numeric_limits<number_float_t>::max_exponent10))
         {
             value_float = 0;
             return true;
@@ -2045,34 +2033,6 @@ scan_number_done:
         token_buffer.push_back(static_cast<typename string_t::value_type>(c));
     }
 
-    /// add a character of a string token; accept() checks the string but does
-    /// not keep its value (@a keep is false)
-    void add_string_char(std::true_type /*keep*/, char_int_type c)
-    {
-        token_buffer.push_back(static_cast<typename string_t::value_type>(c));
-    }
-
-    static void add_string_char(std::false_type /*keep*/, char_int_type /*c*/) noexcept {}
-
-    /// add the UTF-8 bytes of a code point from a \\u escape to a string token
-    void add_codepoint(std::true_type /*keep*/, int codepoint)
-    {
-        encode_utf8(static_cast<std::uint32_t>(codepoint), [this](std::uint32_t byte)
-        {
-            add(static_cast<char_int_type>(byte));
-        });
-    }
-
-    static void add_codepoint(std::false_type /*keep*/, int /*codepoint*/) noexcept {}
-
-    /// append a run of ordinary string characters (see get_after_string_run())
-    void append_string_run(std::true_type /*keep*/, const unsigned char* data, std::size_t n)
-    {
-        token_buffer.append(reinterpret_cast<const typename string_t::value_type*>(data), n);
-    }
-
-    static void append_string_run(std::false_type /*keep*/, const unsigned char* /*data*/, std::size_t /*n*/) noexcept {}
-
   public:
     /////////////////////
     // value getters
@@ -2467,9 +2427,7 @@ scan_number_done:
 
             // string
             case '\"':
-                // accept() (DiscardValues) checks strings without copying them
+                return scan_string();
-                // into token_buffer
-                return scan_string(keep_strings {});
 
             // number
             case '-':
@@ -2554,19 +2512,11 @@ scan_number_done:
     std::size_t decimal_point_position = std::string::npos;
 
     /// whether the caller only needs the token types and never looks at the
-    /// values of number and string tokens; DiscardValues, set only by accept(), which parses
+    /// converted numeric values; set only by accept(), which parses through
-    /// through json_sax_acceptor. When set, convert_number() skips converting
+    /// json_sax_acceptor. When set, convert_number() skips converting integer
-    /// integer tokens whose digit count guarantees that they fit into 64 bits
+    /// tokens whose digit count guarantees that they fit into 64 bits (see
-    /// (see there), floats are converted only when they may overflow (see
+    /// there)
-    /// convert_accumulated_float()), and strings are checked without being
+    const bool discard_number_values = false;
-    /// copied into token_buffer (see scan())
-    static bool discard_values() noexcept
-    {
-        return DiscardValues;
-    }
-
-    /// the tag of scan_string(): whether string tokens are kept
-    using keep_strings = typename std::conditional<DiscardValues, std::false_type, std::true_type>::type;
 };
 
 }  // namespace detail

include/nlohmann/detail/input/parser.hpp

@@ -72,14 +72,14 @@ This class implements a parser for JSON text. Nested arrays and objects are trac
 stack instead of recursion, so deeply nested input does not exhaust the call stack, and what is read
 is reported as SAX events.
 */
-template<typename BasicJsonType, typename InputAdapterType, bool DiscardValues = false>
+template<typename BasicJsonType, typename InputAdapterType>
 class parser
 {
     using number_integer_t = typename BasicJsonType::number_integer_t;
     using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
     using number_float_t = typename BasicJsonType::number_float_t;
     using string_t = typename BasicJsonType::string_t;
-    using lexer_t = lexer<BasicJsonType, InputAdapterType, DiscardValues>;
+    using lexer_t = lexer<BasicJsonType, InputAdapterType>;
     using token_type = typename lexer_t::token_type;
 
   public:
@@ -88,9 +88,10 @@ class parser
                     parser_callback_t<BasicJsonType> cb = nullptr,
                     const bool allow_exceptions_ = true,
                     const bool ignore_comments = false,
-                    const bool ignore_trailing_commas_ = false)
+                    const bool ignore_trailing_commas_ = false,
+                    const bool discard_number_values_ = false)
         : callback(std::move(cb))
-        , m_lexer(std::move(adapter), ignore_comments)
+        , m_lexer(std::move(adapter), ignore_comments, discard_number_values_)
         , allow_exceptions(allow_exceptions_)
         , ignore_trailing_commas(ignore_trailing_commas_)
     {

include/nlohmann/json.hpp

@@ -150,7 +150,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     // can be restored when json_pointer backwards compatibility is removed
     // friend ::nlohmann::json_pointer<StringType>;
 
-    template<typename BasicJsonType, typename InputType, bool DiscardValues>
+    template<typename BasicJsonType, typename InputType>
     friend class ::nlohmann::detail::parser;
     friend ::nlohmann::detail::serializer<basic_json>;
     template<typename BasicJsonType>
@@ -183,11 +183,12 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         detail::parser_callback_t<basic_json>cb = nullptr,
         const bool allow_exceptions = true,
         const bool ignore_comments = false,
-        const bool ignore_trailing_commas = false
+        const bool ignore_trailing_commas = false,
+        const bool discard_number_values = false
     )
     {
         return ::nlohmann::detail::parser<basic_json, InputAdapterType>(std::move(adapter),
-               std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas);
+               std::move(cb), allow_exceptions, ignore_comments, ignore_trailing_commas, discard_number_values);
     }
 
   private:
@@ -3116,9 +3117,7 @@ public:
                             const bool ignore_comments,
                             const bool ignore_trailing_commas)
     {
-        // the parser and lexer that discard the values of tokens (see lexer)
+        return parser(std::forward<InputAdapterType>(adapter), nullptr, false, ignore_comments, ignore_trailing_commas, true).accept(true);
-        return ::nlohmann::detail::parser<basic_json, typename std::decay<InputAdapterType>::type, true>(std::forward<InputAdapterType>(adapter),
-                nullptr, false, ignore_comments, ignore_trailing_commas).accept(true);
     }
 
     /// generate SAX events from the input behind an already created adapter

tests/src/unit-cbor.cpp

@@ -29,7 +29,9 @@ using nlohmann::json;
 #include "round_trip_corpus.hpp"
 #include "test_utils.hpp"
 #include "sax_countdown.hpp"
+#include "sax_event_loggers.hpp"
 using utils::SaxCountdown;
+using utils::SaxEventLogger;
 
 
 TEST_CASE("CBOR")
@@ -2446,6 +2448,78 @@ TEST_CASE("CBOR SAX parsing stops at every event")
     CHECK(count_events(std::vector<std::uint8_t>({0xBF, 0x61, 'a', 0x9F, 0x01, 0xFF, 0xFF})) == 6);
 }
 
+TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input")
+{
+    // contiguous input reads the integers, floats, booleans, and nulls an
+    // array begins with in one loop, other input value by value; both must
+    // give the same events, value, or error for the complete document and for
+    // every truncation of it, and stop at every event a SAX handler rejects
+    const std::vector<std::uint8_t> cbor =
+    {
+        0xA3,                                                       // map(3)
+        0x61, 'a',                                                  // "a"
+        0x98, 0x16,                                                 // array(22)
+        0x00, 0x17, 0x18, 0xFF, 0x19, 0x01, 0x00,                   // 0, 23, 255, 256
+        0x1A, 0x00, 0x01, 0x00, 0x00,                               // 65536
+        0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,       // 18446744073709551615
+        0x20, 0x37, 0x38, 0xFF, 0x39, 0x01, 0x00,                   // -1, -24, -256, -257
+        0x3A, 0x00, 0x01, 0x00, 0x00,                               // -65537
+        0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,       // -18446744073709551616 (a float)
+        0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,       // -9223372036854775808
+        0xF4, 0xF5, 0xF6,                                           // false, true, null
+        0xFA, 0x3F, 0xC0, 0x00, 0x00,                               // 1.5 (float32)
+        0xFB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A,       // -0.1
+        0xF9, 0x3C, 0x00,                                           // 1.0 (float16, ends the loop)
+        0x01, 0x61, 'a',                                            // 1, "a"
+        0xFB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       // 2.5
+        0x61, 'b',                                                  // "b"
+        0x83,                                                       // array(3)
+        0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
+        0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       //  -3.75]
+        0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
+        0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       //  -3.75]
+        0x80,                                                       // []
+        0x61, 'c',                                                  // "c"
+        0x84, 0xA0, 0x01, 0xC1, 0x02, 0x81, 0x03                    // [{}, 1, tag(1) 2, [3]]
+    };
+    const auto tags = json::cbor_tag_handler_t::ignore;
+
+    const json j = json::from_cbor(cbor, true, true, tags);
+    CHECK(j["a"].size() == 22);
+    CHECK(j["a"][5] == 18446744073709551615u);
+    CHECK(j["a"][11].is_number_float());
+    CHECK(j["a"][11] == -18446744073709551616.0);
+    CHECK(j["a"][12] == (std::numeric_limits<std::int64_t>::min)());
+    CHECK(j["a"][17] == -0.1);
+    CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]"));
+    CHECK(j["c"] == json::parse("[{}, 1, 2, [3]]"));
+
+    for (std::size_t length = 0; length <= cbor.size(); ++length)
+    {
+        CAPTURE(length)
+        const std::vector<std::uint8_t> contiguous(cbor.begin(), cbor.begin() + static_cast<std::ptrdiff_t>(length));
+        const std::list<std::uint8_t> bytewise(contiguous.begin(), contiguous.end());
+
+        CHECK(json::from_cbor(contiguous, true, false, tags).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false, tags).dump());
+
+        SaxEventLogger from_contiguous;
+        SaxEventLogger from_bytewise;
+        CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false, false, tags)
+              == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags));
+        CHECK(from_contiguous.events == from_bytewise.events);
+    }
+
+    for (int events = 0; events <= 60; ++events)
+    {
+        CAPTURE(events)
+        const std::list<std::uint8_t> bytewise(cbor.begin(), cbor.end());
+        SaxCountdown from_contiguous(events);
+        SaxCountdown from_bytewise(events);
+        CHECK(json::sax_parse(cbor, &from_contiguous, json::input_format_t::cbor, true, false, false, tags)
+              == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags));
+    }
+}
+
 TEST_CASE("CBOR indefinite-length strings do not recurse per chunk")
 {
     // Reading an indefinite-length string or byte array used to call itself

tests/src/unit-class_parser.cpp

@@ -2982,67 +2982,6 @@ TEST_CASE("accept() agrees with parse() on floats around the overflow limit")
     }
 }
 
-TEST_CASE("accept() agrees with parse() on strings, which it checks without copying them")
-{
-    // accept() scans strings and object keys with a separate instantiation of
-    // the string scanner that does not copy them into the token buffer; it
-    // must accept and reject exactly what parse() does
-    const std::vector<std::string> pieces =
-    {
-        "", "abc", R"(\"\\\/\b\f\n\r\t)", R"(\u0041\u00e4\u20AC)", "\\ud83d\\ude00", "\\uD83D\\uDE00",
-        "\\ud800", "\\udc00", "\\ud800\\u0041", "\\ud800x", "\\udbff\\udfff", "\\u12", "\\u12g4", "\\x", "\\",
-        "\xc3\xa4", "\xe2\x82\xac", "\xf0\x9f\x98\x80", "\xf4\x8f\xbf\xbf", "\xc0\xaf", "\xed\xa0\x80", "\xf4\x90\x80\x80",
-        "\xff", "\xe2\x82", "\xc3", "\x01", "\x1f", "\x7f", "\n", std::string(1, '\0')
-    };
-
-    for (std::size_t run = 0; run <= 70; ++run)
-    {
-        for (const std::string& piece : pieces)
-        {
-            // a run of ordinary characters puts the piece at every offset of the bulk scanners
-            const std::string str = std::string(run, 'a') + piece;
-            std::string opened = "\"";
-            opened += str;
-            std::string closed = opened;
-            closed += '"';
-            std::vector<std::string> texts;
-            texts.push_back(closed);
-            texts.emplace_back("[");
-            texts.back() += closed;
-            texts.back() += ']';
-            texts.emplace_back("{");
-            texts.back() += closed;
-            texts.back() += ":1}";
-            texts.emplace_back("{\"k\":");
-            texts.back() += closed;
-            texts.back() += ',';
-            texts.back() += closed;
-            texts.back() += ":[";
-            texts.back() += closed;
-            texts.back() += "]}";
-            texts.emplace_back("[");
-            texts.back() += opened;
-            for (const std::string& text : texts)
-            {
-                CAPTURE(text)
-                for (int options = 0; options < 4; ++options)
-                {
-                    const bool ignore_comments = (options & 1) != 0;
-                    const bool ignore_trailing_commas = (options & 2) != 0;
-                    const bool parses = !json::parse(text, nullptr, false, ignore_comments, ignore_trailing_commas).is_discarded();
-                    CHECK(json::accept(text, ignore_comments, ignore_trailing_commas) == parses);
-
-                    std::istringstream stream(text);
-                    CHECK(json::accept(stream, ignore_comments, ignore_trailing_commas) == parses);
-
-                    const std::list<char> chars(text.begin(), text.end());
-                    CHECK(json::accept(chars.begin(), chars.end(), ignore_comments, ignore_trailing_commas) == parses);
-                }
-            }
-        }
-    }
-}
-
 namespace
 {
 // a DOM parser that reports parse errors without throwing, as in the sax_no_exception example

tests/src/unit-msgpack.cpp

@@ -27,12 +27,15 @@ using nlohmann::json;
 #include <sstream>
 #include <iomanip>
 #include <limits>
+#include <list>
 #include <set>
 #include "make_test_data_available.hpp"
 #include "round_trip_corpus.hpp"
 #include "test_utils.hpp"
 #include "sax_countdown.hpp"
+#include "sax_event_loggers.hpp"
 using utils::SaxCountdown;
+using utils::SaxEventLogger;
 
 
 TEST_CASE("MessagePack")
@@ -2212,3 +2215,99 @@ TEST_CASE("MessagePack large strings and binaries (chunked reader)")
         }
     }
 }
+
+TEST_CASE("MessagePack arrays of scalars from contiguous and non-contiguous input")
+{
+    // contiguous input reads the integers, floats, booleans, and nils an
+    // array begins with in one loop, other input value by value; both must
+    // give the same events, value, or error for the complete document and for
+    // every truncation of it, and stop at every event a SAX handler rejects
+    const std::vector<std::uint8_t> msgpack =
+    {
+        0x83,                                                       // map(3)
+        0xA1, 'a',                                                  // "a"
+        0xDC, 0x00, 0x16,                                           // array 16 (22)
+        0x00, 0x7F, 0xCC, 0xFF, 0xCD, 0x01, 0x00,                   // 0, 127, 255, 256
+        0xCE, 0x00, 0x01, 0x00, 0x00,                               // 65536
+        0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,       // 18446744073709551615
+        0xFF, 0xE0, 0xD0, 0x80, 0xD1, 0xFF, 0x00,                   // -1, -32, -128, -256
+        0xD2, 0xFF, 0xFE, 0xFF, 0xFF,                               // -65537
+        0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       // -9223372036854775808
+        0xC2, 0xC3, 0xC0,                                           // false, true, nil
+        0xCA, 0x3F, 0xC0, 0x00, 0x00,                               // 1.5 (float 32)
+        0xCB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A,       // -0.1
+        0xA1, 'x',                                                  // "x" (ends the loop)
+        0x01,                                                       // 1
+        0xCB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       // 2.5
+        0xD9, 0x01, 'y',                                            // "y" (str 8)
+        0xC0,                                                       // nil
+        0xA1, 'b',                                                  // "b"
+        0x93,                                                       // fixarray(3)
+        0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
+        0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       //  -3.75]
+        0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25,
+        0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,       //  -3.75]
+        0x90,                                                       // []
+        0xA1, 'c',                                                  // "c"
+        0x94, 0x80, 0x01, 0xC4, 0x01, 0x05, 0x91, 0x03              // [{}, 1, bin 8, [3]]
+    };
+
+    const json j = json::from_msgpack(msgpack);
+    CHECK(j["a"].size() == 22);
+    CHECK(j["a"][5] == 18446744073709551615u);
+    CHECK(j["a"][7] == -32);
+    CHECK(j["a"][11] == (std::numeric_limits<std::int64_t>::min)());
+    CHECK(j["a"][15] == 1.5);
+    CHECK(j["a"][16] == -0.1);
+    CHECK(j["a"][17] == "x");
+    CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]"));
+    CHECK(j["c"][2].is_binary());
+
+    for (std::size_t length = 0; length <= msgpack.size(); ++length)
+    {
+        CAPTURE(length)
+        const std::vector<std::uint8_t> contiguous(msgpack.begin(), msgpack.begin() + static_cast<std::ptrdiff_t>(length));
+        const std::list<std::uint8_t> bytewise(contiguous.begin(), contiguous.end());
+
+        CHECK(json::from_msgpack(contiguous, true, false).dump() == json::from_msgpack(bytewise.begin(), bytewise.end(), true, false).dump());
+#if !defined(JSON_NOEXCEPTION)
+        std::string contiguous_error;
+        std::string bytewise_error;
+        try
+        {
+            json _;
+            _ = json::from_msgpack(contiguous);
+        }
+        catch (const json::parse_error& e)
+        {
+            contiguous_error = e.what();
+        }
+        try
+        {
+            json _;
+            _ = json::from_msgpack(bytewise.begin(), bytewise.end());
+        }
+        catch (const json::parse_error& e)
+        {
+            bytewise_error = e.what();
+        }
+        CHECK(contiguous_error == bytewise_error);
+#endif
+
+        SaxEventLogger from_contiguous;
+        SaxEventLogger from_bytewise;
+        CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::msgpack)
+              == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack));
+        CHECK(from_contiguous.events == from_bytewise.events);
+    }
+
+    for (int events = 0; events <= 60; ++events)
+    {
+        CAPTURE(events)
+        const std::list<std::uint8_t> bytewise(msgpack.begin(), msgpack.end());
+        SaxCountdown from_contiguous(events);
+        SaxCountdown from_bytewise(events);
+        CHECK(json::sax_parse(msgpack, &from_contiguous, json::input_format_t::msgpack)
+              == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack));
+    }
+}