diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 9ed0e4d1b..5b105974e 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -741,11 +741,13 @@ class binary_reader bool get_cbor_negative_integer() { NumberType number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(number))) - { - return false; - } + return get_number(number) && emit_cbor_negative_integer(number); + } + /// pass the CBOR negative integer -1 - @a number to the SAX parser + template + bool emit_cbor_negative_integer(const NumberType number) + { // the value is -1 - number, which fits into number_integer_t // whenever number does; the outer cast undoes the integral promotion // for number_integer_t types narrower than int @@ -970,7 +972,7 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); + return enter_cbor_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) case 0x99: // array (two-byte uint16_t for n follow) @@ -979,7 +981,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size); + return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -1676,6 +1678,171 @@ class binary_reader return true; } + /*! + @brief read a big-endian number of type @a NumberType at @a data + */ + template + NumberType load_big_endian(const unsigned char* data) const noexcept + { + NumberType result{}; + std::memcpy(&result, data, sizeof(NumberType)); + if (is_little_endian) + { + byte_swap(result); + } + return result; + } + + /*! + @brief read the integers, floats, booleans, and nulls that follow in a + definite-length array, without the per-element bookkeeping of + @ref parse_cbor_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_cbor_value, which reads them the same way. + + @param[in,out] remaining the number of elements of the array not yet read + @return whether the SAX parser accepted the values + */ + bool parse_cbor_scalar_run(std::size_t& remaining) + { + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 1; + bool accepted = true; + switch (head) + { + case 0x18: + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0x19: + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1A: + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1B: + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x38: + size = 2; + accepted = emit_cbor_negative_integer(data[1]); + break; + case 0x39: + size = 3; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3A: + size = 5; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3B: + size = 9; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0xF4: + accepted = sax->boolean(false); + break; + case 0xF5: + accepted = sax->boolean(true); + break; + case 0xF6: + accepted = sax->null(); + break; + case 0xFA: + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xFB: + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + default: + if (head <= 0x17) + { + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0x20 && head <= 0x37) + { + accepted = sax->number_integer(static_cast(0x20 - 1 - head)); + } + else + { + // any other value is read by parse_cbor_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the head byte + ia.bulk_skip(size); + chars_read += size; + current = head; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a definite-length array and read the scalars it begins with + (see @ref parse_cbor_scalar_run) + + Only the start of an array is tried, which covers arrays of numbers + (e.g. coordinates) but costs arrays of other values only one test, not one + per element. + + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_cbor_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_cbor_array_start(std::integral_constant {})); + } + + /// read the scalars the array just opened begins with; out of line, so + /// that @ref parse_cbor_value stays small enough to be inlined into + /// @ref parse_cbor_internal + JSON_HEDLEY_NEVER_INLINE + bool parse_cbor_array_start(std::true_type /*bulk*/) + { + return !cbor_scalar_follows() || parse_cbor_scalar_run(container_stack.back().remaining); + } + + /// input that is not contiguous: every value is read by parse_cbor_value + bool parse_cbor_array_start(std::false_type /*bulk*/) const noexcept + { + return true; + } + + /*! + @brief whether @ref parse_cbor_scalar_run can read the next value: one of + the heads it handles, with at least 9 bytes of input left + */ + bool cbor_scalar_follows() const + { + if (ia.bulk_remaining() < 9) + { + return false; + } + const auto head = static_cast(*reinterpret_cast(ia.bulk_data())); + // major types 0 and 1 (unsigned and negative integers) up to an + // 8-byte argument, or false, true, null, float32, float64 (0xF4, + // 0xF5, 0xF6, 0xFA, 0xFB: bits 0, 1, 2, 6, 7 of 0xC7) + return ((head & 0xC0u) == 0 && (head & 0x1Fu) <= 0x1Bu) + || (head >= 0xF4u && ((0xC7u >> (head - 0xF4u)) & 1u) != 0); + } + + /*! @brief read a CBOR value and everything nested inside it @@ -1701,10 +1868,8 @@ class binary_reader { if (!container_stack.empty()) { - // a copy, not a reference: it must stay valid across the - // pop_back() below, which destroys the container_stack element - // it would otherwise alias const container_frame top = container_stack.back(); + const bool is_object = top.is_object; bool at_end = false; if (top.remaining != npos) @@ -1716,7 +1881,7 @@ class binary_reader { // claim the element about to be read --container_stack.back().remaining; - if (top.is_object) + if (is_object) { get(); } @@ -1729,7 +1894,7 @@ class binary_reader // Testing for it consumes a byte, which is the first byte // of the next element when it is not one. at_end = (get() == 0xFF); - fetch = top.is_object; + fetch = is_object; } if (at_end) @@ -1746,7 +1911,7 @@ class binary_reader continue; } - if (top.is_object) + if (is_object) { if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler))) { @@ -3691,13 +3856,13 @@ class binary_reader return sax->null(); case 0xFB: - return sax->number_float(static_cast(-1.0), ""); + return sax->number_float(static_cast(-1.0), no_source); case 0xFC: - return sax->number_float(static_cast(0.0), ""); + return sax->number_float(static_cast(0.0), no_source); case 0xFD: - return sax->number_float(static_cast(1.0), ""); + return sax->number_float(static_cast(1.0), no_source); case 0xFF: // empty string { @@ -4201,7 +4366,7 @@ class binary_reader { return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } - return sax->number_float(result, ""); + return sax->number_float(result, no_source); } /*! @@ -4261,7 +4426,7 @@ class binary_reader }(); return sax->number_float((half & 0x8000u) != 0 ? static_cast(-val) - : static_cast(val), ""); + : static_cast(val), no_source); } /*! @@ -4586,6 +4751,10 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the (empty) source text of every number_float event: binary formats + /// have none, and a string_t built for each float would cost more + const string_t no_source{}; + /// the containers that have been opened and not closed yet; see @ref container_frame std::vector container_stack{}; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 10b42dac1..b7ab5b7d9 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18517,11 +18517,13 @@ class binary_reader bool get_cbor_negative_integer() { NumberType number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(number))) - { - return false; - } + return get_number(number) && emit_cbor_negative_integer(number); + } + /// pass the CBOR negative integer -1 - @a number to the SAX parser + template + bool emit_cbor_negative_integer(const NumberType number) + { // the value is -1 - number, which fits into number_integer_t // whenever number does; the outer cast undoes the integral promotion // for number_integer_t types narrower than int @@ -18746,7 +18748,7 @@ class binary_reader case 0x95: case 0x96: case 0x97: - return enter_array(conditional_static_cast(static_cast(current) & 0x1Fu)); + return enter_cbor_array(conditional_static_cast(static_cast(current) & 0x1Fu)); case 0x98: // array (one-byte uint8_t for n follows) case 0x99: // array (two-byte uint16_t for n follow) @@ -18755,7 +18757,7 @@ class binary_reader { std::uint64_t len{}; std::size_t size{}; - return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_array(size); + return get_cbor_argument(len) && get_cbor_container_size(len, size, "array") && enter_cbor_array(size); } case 0x9F: // array (indefinite length) @@ -19452,6 +19454,171 @@ class binary_reader return true; } + /*! + @brief read a big-endian number of type @a NumberType at @a data + */ + template + NumberType load_big_endian(const unsigned char* data) const noexcept + { + NumberType result{}; + std::memcpy(&result, data, sizeof(NumberType)); + if (is_little_endian) + { + byte_swap(result); + } + return result; + } + + /*! + @brief read the integers, floats, booleans, and nulls that follow in a + definite-length array, without the per-element bookkeeping of + @ref parse_cbor_internal + + Stops at the end of the array, at any other kind of value, and when fewer + than 9 bytes (the longest of these values) are left in the input, leaving + those to @ref parse_cbor_value, which reads them the same way. + + @param[in,out] remaining the number of elements of the array not yet read + @return whether the SAX parser accepted the values + */ + bool parse_cbor_scalar_run(std::size_t& remaining) + { + while (remaining != 0 && ia.bulk_remaining() >= 9) + { + const auto* const data = reinterpret_cast(ia.bulk_data()); + const unsigned char head = data[0]; + std::size_t size = 1; + bool accepted = true; + switch (head) + { + case 0x18: + size = 2; + accepted = emit_unsigned(data[1]); + break; + case 0x19: + size = 3; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1A: + size = 5; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x1B: + size = 9; + accepted = emit_unsigned(load_big_endian(data + 1)); + break; + case 0x38: + size = 2; + accepted = emit_cbor_negative_integer(data[1]); + break; + case 0x39: + size = 3; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3A: + size = 5; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0x3B: + size = 9; + accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); + break; + case 0xF4: + accepted = sax->boolean(false); + break; + case 0xF5: + accepted = sax->boolean(true); + break; + case 0xF6: + accepted = sax->null(); + break; + case 0xFA: + size = 5; + accepted = emit_float(load_big_endian(data + 1)); + break; + case 0xFB: + size = 9; + accepted = emit_float(load_big_endian(data + 1)); + break; + default: + if (head <= 0x17) + { + accepted = sax->number_unsigned(static_cast(head)); + } + else if (head >= 0x20 && head <= 0x37) + { + accepted = sax->number_integer(static_cast(0x20 - 1 - head)); + } + else + { + // any other value is read by parse_cbor_value + return true; + } + break; + } + // as if read by get() and get_number(): current is the head byte + ia.bulk_skip(size); + chars_read += size; + current = head; + --remaining; + if (JSON_HEDLEY_UNLIKELY(!accepted)) + { + return false; + } + } + return true; + } + + /*! + @brief open a definite-length array and read the scalars it begins with + (see @ref parse_cbor_scalar_run) + + Only the start of an array is tried, which covers arrays of numbers + (e.g. coordinates) but costs arrays of other values only one test, not one + per element. + + @param[in] len the number of elements + @return whether the SAX parser accepted the array and the values read + */ + bool enter_cbor_array(const std::size_t len) + { + return enter_array(len) && (len == 0 || parse_cbor_array_start(std::integral_constant {})); + } + + /// read the scalars the array just opened begins with; out of line, so + /// that @ref parse_cbor_value stays small enough to be inlined into + /// @ref parse_cbor_internal + JSON_HEDLEY_NEVER_INLINE + bool parse_cbor_array_start(std::true_type /*bulk*/) + { + return !cbor_scalar_follows() || parse_cbor_scalar_run(container_stack.back().remaining); + } + + /// input that is not contiguous: every value is read by parse_cbor_value + bool parse_cbor_array_start(std::false_type /*bulk*/) const noexcept + { + return true; + } + + /*! + @brief whether @ref parse_cbor_scalar_run can read the next value: one of + the heads it handles, with at least 9 bytes of input left + */ + bool cbor_scalar_follows() const + { + if (ia.bulk_remaining() < 9) + { + return false; + } + const auto head = static_cast(*reinterpret_cast(ia.bulk_data())); + // major types 0 and 1 (unsigned and negative integers) up to an + // 8-byte argument, or false, true, null, float32, float64 (0xF4, + // 0xF5, 0xF6, 0xFA, 0xFB: bits 0, 1, 2, 6, 7 of 0xC7) + return ((head & 0xC0u) == 0 && (head & 0x1Fu) <= 0x1Bu) + || (head >= 0xF4u && ((0xC7u >> (head - 0xF4u)) & 1u) != 0); + } + + /*! @brief read a CBOR value and everything nested inside it @@ -19477,10 +19644,8 @@ class binary_reader { if (!container_stack.empty()) { - // a copy, not a reference: it must stay valid across the - // pop_back() below, which destroys the container_stack element - // it would otherwise alias const container_frame top = container_stack.back(); + const bool is_object = top.is_object; bool at_end = false; if (top.remaining != npos) @@ -19492,7 +19657,7 @@ class binary_reader { // claim the element about to be read --container_stack.back().remaining; - if (top.is_object) + if (is_object) { get(); } @@ -19505,7 +19670,7 @@ class binary_reader // Testing for it consumes a byte, which is the first byte // of the next element when it is not one. at_end = (get() == 0xFF); - fetch = top.is_object; + fetch = is_object; } if (at_end) @@ -19522,7 +19687,7 @@ class binary_reader continue; } - if (top.is_object) + if (is_object) { if (JSON_HEDLEY_UNLIKELY(!skip_cbor_key_tags(tag_handler))) { @@ -21467,13 +21632,13 @@ class binary_reader return sax->null(); case 0xFB: - return sax->number_float(static_cast(-1.0), ""); + return sax->number_float(static_cast(-1.0), no_source); case 0xFC: - return sax->number_float(static_cast(0.0), ""); + return sax->number_float(static_cast(0.0), no_source); case 0xFD: - return sax->number_float(static_cast(1.0), ""); + return sax->number_float(static_cast(1.0), no_source); case 0xFF: // empty string { @@ -21977,7 +22142,7 @@ class binary_reader { return out_of_range_error(exception_id::number_overflow, "number overflow", "value"); } - return sax->number_float(result, ""); + return sax->number_float(result, no_source); } /*! @@ -22037,7 +22202,7 @@ class binary_reader }(); return sax->number_float((half & 0x8000u) != 0 ? static_cast(-val) - : static_cast(val), ""); + : static_cast(val), no_source); } /*! @@ -22362,6 +22527,10 @@ class binary_reader /// the SAX parser json_sax_t* sax = nullptr; + /// the (empty) source text of every number_float event: binary formats + /// have none, and a string_t built for each float would cost more + const string_t no_source{}; + /// the containers that have been opened and not closed yet; see @ref container_frame std::vector container_stack{}; diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 44cfa7723..930cbe7f5 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -29,7 +29,9 @@ using nlohmann::json; #include "round_trip_corpus.hpp" #include "test_utils.hpp" #include "sax_countdown.hpp" +#include "sax_event_loggers.hpp" using utils::SaxCountdown; +using utils::SaxEventLogger; TEST_CASE("CBOR") @@ -2446,6 +2448,78 @@ TEST_CASE("CBOR SAX parsing stops at every event") CHECK(count_events(std::vector({0xBF, 0x61, 'a', 0x9F, 0x01, 0xFF, 0xFF})) == 6); } +TEST_CASE("CBOR arrays of scalars from contiguous and non-contiguous input") +{ + // contiguous input reads the integers, floats, booleans, and nulls an + // array begins with in one loop, other input value by value; both must + // give the same events, value, or error for the complete document and for + // every truncation of it, and stop at every event a SAX handler rejects + const std::vector cbor = + { + 0xA3, // map(3) + 0x61, 'a', // "a" + 0x98, 0x16, // array(22) + 0x00, 0x17, 0x18, 0xFF, 0x19, 0x01, 0x00, // 0, 23, 255, 256 + 0x1A, 0x00, 0x01, 0x00, 0x00, // 65536 + 0x1B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 18446744073709551615 + 0x20, 0x37, 0x38, 0xFF, 0x39, 0x01, 0x00, // -1, -24, -256, -257 + 0x3A, 0x00, 0x01, 0x00, 0x00, // -65537 + 0x3B, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -18446744073709551616 (a float) + 0x3B, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // -9223372036854775808 + 0xF4, 0xF5, 0xF6, // false, true, null + 0xFA, 0x3F, 0xC0, 0x00, 0x00, // 1.5 (float32) + 0xFB, 0xBF, 0xB9, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A, // -0.1 + 0xF9, 0x3C, 0x00, // 1.0 (float16, ends the loop) + 0x01, 0x61, 'a', // 1, "a" + 0xFB, 0x40, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 2.5 + 0x61, 'b', // "b" + 0x83, // array(3) + 0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x82, 0xFB, 0x3F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // [1.25, + 0xFB, 0xC0, 0x0E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // -3.75] + 0x80, // [] + 0x61, 'c', // "c" + 0x84, 0xA0, 0x01, 0xC1, 0x02, 0x81, 0x03 // [{}, 1, tag(1) 2, [3]] + }; + const auto tags = json::cbor_tag_handler_t::ignore; + + const json j = json::from_cbor(cbor, true, true, tags); + CHECK(j["a"].size() == 22); + CHECK(j["a"][5] == 18446744073709551615u); + CHECK(j["a"][11].is_number_float()); + CHECK(j["a"][11] == -18446744073709551616.0); + CHECK(j["a"][12] == (std::numeric_limits::min)()); + CHECK(j["a"][17] == -0.1); + CHECK(j["b"] == json::parse("[[1.25, -3.75], [1.25, -3.75], []]")); + CHECK(j["c"] == json::parse("[{}, 1, 2, [3]]")); + + for (std::size_t length = 0; length <= cbor.size(); ++length) + { + CAPTURE(length) + const std::vector contiguous(cbor.begin(), cbor.begin() + static_cast(length)); + const std::list bytewise(contiguous.begin(), contiguous.end()); + + CHECK(json::from_cbor(contiguous, true, false, tags).dump() == json::from_cbor(bytewise.begin(), bytewise.end(), true, false, tags).dump()); + + SaxEventLogger from_contiguous; + SaxEventLogger from_bytewise; + CHECK(json::sax_parse(contiguous, &from_contiguous, json::input_format_t::cbor, true, false, false, tags) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags)); + CHECK(from_contiguous.events == from_bytewise.events); + } + + for (int events = 0; events <= 60; ++events) + { + CAPTURE(events) + const std::list bytewise(cbor.begin(), cbor.end()); + SaxCountdown from_contiguous(events); + SaxCountdown from_bytewise(events); + CHECK(json::sax_parse(cbor, &from_contiguous, json::input_format_t::cbor, true, false, false, tags) + == json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, json::input_format_t::cbor, true, false, false, tags)); + } +} + TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") { // Reading an indefinite-length string or byte array used to call itself