From edf8d1760129d82162d0d1ca7e8fb2d0c08db638 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 1 Nov 2026 17:22:32 +0100 Subject: [PATCH] MessagePack reader: read runs of scalar array elements in one loop (#5980) * MessagePack reader: read runs of scalar array elements in one loop As for CBOR: when an array opens and the input is contiguous, read the integers, floats, booleans, and nils that follow in a tight loop instead of going through parse_msgpack_internal and parse_msgpack_value for every element. The loop stops at any other kind of value and when fewer than 9 bytes are left; those values are read by the general path as before. Non-contiguous inputs are not affected. Signed-off-by: Niels Lohmann * MessagePack reader: enter the scalar run only when a scalar follows As for CBOR: enter_msgpack_array tests the first byte and the remaining length inline and calls the out-of-line run only for a scalar, so arrays of maps or strings no longer pay for a call that reads nothing; the run's loop condition moves to the end (do-while). Signed-off-by: Niels Lohmann * MessagePack reader: mark the initial accepted in the scalar run as a false-positive dead store ci_infer reports DEAD_STORE for 'bool accepted = true;' in the MessagePack scalar run, as in the CBOR one, because every case assigns accepted before it is read. Signed-off-by: Niels Lohmann * MessagePack tests: assign the parsed value instead of casting it to void GCC (ci_test_gcc) reports -Werror=unused-result for static_cast(json::from_msgpack(...)): a cast to void does not silence warn_unused_result on GCC. Assign to a json, as the other tests in the file do. Signed-off-by: Niels Lohmann * MessagePack reader: drop the scalar pre-check before the run enter_msgpack_array now calls the run directly, and parse_msgpack_scalar_run tests the remaining count and length at the top of a while loop instead of relying on msgpack_scalar_follows(); its own switch already leaves on any value that is not a scalar. Instructions per repetition, new/old: SAX canada 0.946 (clang) and 0.973 (GCC 13), DOM read canada 0.991 / 0.992; no row more than 0.3 % higher. Signed-off-by: Niels Lohmann * MessagePack reader: say why the initial accepted in the scalar run stays As for CBOR: Infer's DEAD_STORE report is right, not a false positive. The initializer is kept for clang-tidy's cppcoreguidelines-init-variables. Signed-off-by: Niels Lohmann * MessagePack reader: give the scalar run the bulk tag instead of a wrapper As for CBOR: parse_msgpack_scalar_run takes the bulk-scan tag itself and is the function kept out of line, so parse_msgpack_array_start goes. Signed-off-by: Niels Lohmann * MessagePack reader: start the scalar run's accepted at false As for CBOR: Infer ignores dead stores of 0, so the suppression goes. Signed-off-by: Niels Lohmann * MessagePack reader: let the scalar run find its array itself As for CBOR: parse_msgpack_scalar_run reads container_stack.back() in its bulk version, so the caller no longer evaluates it for input that is not contiguous. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../nlohmann/detail/input/binary_reader.hpp | 137 +++++++++++++++++- single_include/nlohmann/json.hpp | 137 +++++++++++++++++- tests/src/unit-msgpack.cpp | 99 +++++++++++++ 3 files changed, 367 insertions(+), 6 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 686b265a6..0c345fea0 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -2130,7 +2130,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -2261,13 +2261,13 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(len) && enter_array(static_cast(len)); + return get_number(len) && enter_msgpack_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_array(conditional_static_cast(len)); + return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); } case 0xDE: // map 16 @@ -2609,6 +2609,137 @@ class binary_reader } } + /*! + @brief read the integers, floats, booleans, and nils that follow in a + MessagePack array, without the per-element bookkeeping of + @ref parse_msgpack_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_msgpack_value, which reads them the same way. + + Out of line, so that @ref parse_msgpack_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_msgpack_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0xC0: // nil + size = 1; + accepted = sax->null(); + break; + case 0xC2: // false + size = 1; + accepted = sax->boolean(false); + break; + case 0xC3: // true + size = 1; + accepted = sax->boolean(true); + break; + case 0xCA: // float 32 + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCB: // float 64 + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCC: // uint 8 + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0xCD: // uint 16 + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCE: // uint 32 + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCF: // uint 64 + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xD0: // int 8 + size = 2; + accepted = emit_signed(static_cast(data[1])); + break; + case 0xD1: // int 16 + size = 3; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD2: // int 32 + size = 5; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD3: // int 64 + size = 9; + accepted = emit_signed(load_big_endian(data + 1)); + break; + default: + if (head <= 0x7F) + { + // positive fixint + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0xE0) + { + // negative fixint + size = 1; + accepted = sax->number_integer(static_cast(head)); + } + else + { + // any other value is read by parse_msgpack_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the last byte + ia.bulk_skip(size); + chars_read += size; + current = data[size - 1]; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a MessagePack array and read the scalars it begins with (see + @ref parse_msgpack_scalar_run) + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_msgpack_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_msgpack_value + bool parse_msgpack_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a MessagePack value and everything nested inside it diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index fe5bf782b..d440efd8f 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -19935,7 +19935,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - return enter_array(conditional_static_cast(static_cast(current) & 0x0Fu)); + return enter_msgpack_array(conditional_static_cast(static_cast(current) & 0x0Fu)); // fixstr case 0xA0: @@ -20066,13 +20066,13 @@ class binary_reader case 0xDC: // array 16 { std::uint16_t len{}; - return get_number(len) && enter_array(static_cast(len)); + return get_number(len) && enter_msgpack_array(static_cast(len)); } case 0xDD: // array 32 { std::uint32_t len{}; - return get_number(len) && enter_array(conditional_static_cast(len)); + return get_number(len) && enter_msgpack_array(conditional_static_cast(len)); } case 0xDE: // map 16 @@ -20414,6 +20414,137 @@ class binary_reader } } + /*! + @brief read the integers, floats, booleans, and nils that follow in a + MessagePack array, without the per-element bookkeeping of + @ref parse_msgpack_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_msgpack_value, which reads them the same way. + + Out of line, so that @ref parse_msgpack_value stays small enough to be inlined. + + The array is the one on top of @ref container_stack; its count of + elements not yet read is updated. + + @return whether the SAX parser accepted the values + */ + JSON_HEDLEY_NEVER_INLINE + bool parse_msgpack_scalar_run(std::true_type /*bulk*/) + { + // nothing pushes onto the stack while the run reads scalars + std::size_t& remaining = container_stack.back().remaining; + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 0; + bool accepted = false; + switch (head) + { + case 0xC0: // nil + size = 1; + accepted = sax->null(); + break; + case 0xC2: // false + size = 1; + accepted = sax->boolean(false); + break; + case 0xC3: // true + size = 1; + accepted = sax->boolean(true); + break; + case 0xCA: // float 32 + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCB: // float 64 + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xCC: // uint 8 + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0xCD: // uint 16 + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCE: // uint 32 + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xCF: // uint 64 + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0xD0: // int 8 + size = 2; + accepted = emit_signed(static_cast(data[1])); + break; + case 0xD1: // int 16 + size = 3; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD2: // int 32 + size = 5; + accepted = emit_signed(load_big_endian(data + 1)); + break; + case 0xD3: // int 64 + size = 9; + accepted = emit_signed(load_big_endian(data + 1)); + break; + default: + if (head <= 0x7F) + { + // positive fixint + size = 1; + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0xE0) + { + // negative fixint + size = 1; + accepted = sax->number_integer(static_cast(head)); + } + else + { + // any other value is read by parse_msgpack_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the last byte + ia.bulk_skip(size); + chars_read += size; + current = data[size - 1]; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a MessagePack array and read the scalars it begins with (see + @ref parse_msgpack_scalar_run) + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_msgpack_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_msgpack_scalar_run(std::integral_constant {})); + } + + /// input that is not contiguous: every value is read by parse_msgpack_value + bool parse_msgpack_scalar_run(std::false_type /*bulk*/) const noexcept + { + return true; + } + /*! @brief read a MessagePack value and everything nested inside it diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 2890a240c..9b5891fdd 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -27,12 +27,15 @@ using nlohmann::json; #include #include #include +#include #include #include "make_test_data_available.hpp" #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("MessagePack") @@ -2212,3 +2215,99 @@ TEST_CASE("MessagePack large strings and binaries (chunked reader)") } } } + +TEST_CASE("MessagePack arrays of scalars from contiguous and non-contiguous input") +{ + // contiguous input reads the integers, floats, booleans, and nils an + // array begins with in one loop, other input value by value; both must + // give the same events, value, or error for the complete document and for + // every truncation of it, and stop at every event a SAX handler rejects + const std::vector msgpack = + { + 0x83, // map(3) + 0xA1, 'a', // "a" + 0xDC, 0x00, 0x16, // array 16 (22) + 0x00, 0x7F, 0xCC, 0xFF, 0xCD, 0x01, 0x00, // 0, 127, 255, 256 + 0xCE, 0x00, 0x01, 0x00, 0x00, // 65536 + 0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 18446744073709551615 + 0xFF, 0xE0, 0xD0, 0x80, 0xD1, 0xFF, 0x00, // -1, -32, -128, -256 + 0xD2, 0xFF, 0xFE, 0xFF, 0xFF, // -65537 + 0xD3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -9223372036854775808 + 0xC2, 0xC3, 0xC0, // false, true, nil + 0xCA, 0x3F, 0xC0, 0x00, 0x00, // 1.5 (float 32) + 0xCB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A, // -0.1 + 0xA1, 'x', // "x" (ends the loop) + 0x01, // 1 + 0xCB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2.5 + 0xD9, 0x01, 'y', // "y" (str 8) + 0xC0, // nil + 0xA1, 'b', // "b" + 0x93, // fixarray(3) + 0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x92, 0xCB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xCB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x90, // [] + 0xA1, 'c', // "c" + 0x94, 0x80, 0x01, 0xC4, 0x01, 0x05, 0x91, 0x03 // [{}, 1, bin 8, [3]] + }; + + const json j = json::from_msgpack(msgpack); + CHECK(j["a"].size() == 22); + CHECK(j["a"][5] == 18446744073709551615u); + CHECK(j["a"][7] == -32); + CHECK(j["a"][11] == (std::numeric_limits::min)()); + CHECK(j["a"][15] == 1.5); + CHECK(j["a"][16] == -0.1); + CHECK(j["a"][17] == "x"); + CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]")); + CHECK(j["c"][2].is_binary()); + + for (std::size_t length = 0; length <= msgpack.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(msgpack.begin(), msgpack.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_msgpack(contiguous, true, false).dump() == json::from_msgpack(bytewise.begin(), bytewise.end(), true, false).dump()); +#if !defined(JSON_NOEXCEPTION) + std::string contiguous_error; + std::string bytewise_error; + try + { + json _; + _ = json::from_msgpack(contiguous); + } + catch (const json::parse_error& e) + { + contiguous_error = e.what(); + } + try + { + json _; + _ = json::from_msgpack(bytewise.begin(), bytewise.end()); + } + catch (const json::parse_error& e) + { + bytewise_error = e.what(); + } + CHECK(contiguous_error == bytewise_error); +#endif + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::msgpack) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 60; ++events) + { + CAPTURE(events) + const std::list bytewise(msgpack.begin(), msgpack.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(msgpack, &from_contiguous, json::input_format_t::msgpack) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::msgpack)); + } +}