diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index bbd2daceb5..c5be6465b1 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1927,6 +1927,21 @@ class binary_reader return result; } + /*! + @brief move past a value of @a size bytes, starting with @a head, that a + bulk run reads, as get() and get_number() would + + Called before the value is passed on, so that an error reported while + passing it on (see @ref emit_float) has the position and the last byte + that reading the value byte by byte gives. + */ + void bulk_advance(const std::size_t size, const unsigned char head) + { + ia.bulk_skip(size); + chars_read += size; + current = head; + } + /*! @brief read the integers, floats, booleans, and nulls that follow in a definite-length array, without the per-element bookkeeping of @@ -1952,71 +1967,70 @@ class binary_reader { const auto* const data = reinterpret_cast(ia.bulk_data()); const unsigned char head = data[0]; - std::size_t size = 0; bool accepted = false; switch (head) { case 0x18: - size = 2; + bulk_advance(2, head); accepted = emit_unsigned(data[1]); break; case 0x19: - size = 3; + bulk_advance(3, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x1A: - size = 5; + bulk_advance(5, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x1B: - size = 9; + bulk_advance(9, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x38: - size = 2; + bulk_advance(2, head); accepted = emit_cbor_negative_integer(data[1]); break; case 0x39: - size = 3; + bulk_advance(3, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0x3A: - size = 5; + bulk_advance(5, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0x3B: - size = 9; + bulk_advance(9, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0xF4: - size = 1; + bulk_advance(1, head); accepted = sax->boolean(false); break; case 0xF5: - size = 1; + bulk_advance(1, head); accepted = sax->boolean(true); break; case 0xF6: - size = 1; + bulk_advance(1, head); accepted = sax->null(); break; case 0xFA: - size = 5; + bulk_advance(5, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xFB: - size = 9; + bulk_advance(9, head); accepted = emit_float(load_big_endian(data + 1)); break; default: if (head <= 0x17) { - size = 1; + bulk_advance(1, head); accepted = sax->number_unsigned(static_cast(head)); } else if (head >= 0x20 && head <= 0x37) { - size = 1; + bulk_advance(1, head); accepted = sax->number_integer(static_cast(0x20 - 1 - head)); } else @@ -2026,10 +2040,6 @@ class binary_reader } break; } - // as if read by get() and get_number(): current is the head byte - ia.bulk_skip(size); - chars_read += size; - current = head; --remaining; if (JSON_HEDLEY_UNLIKELY(!accepted)) { @@ -2867,73 +2877,72 @@ class binary_reader { const auto* const data = reinterpret_cast(ia.bulk_data()); const unsigned char head = data[0]; - std::size_t size = 0; bool accepted = false; switch (head) { case 0xC0: // nil - size = 1; + bulk_advance(1, head); accepted = sax->null(); break; case 0xC2: // false - size = 1; + bulk_advance(1, head); accepted = sax->boolean(false); break; case 0xC3: // true - size = 1; + bulk_advance(1, head); accepted = sax->boolean(true); break; case 0xCA: // float 32 - size = 5; + bulk_advance(5, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xCB: // float 64 - size = 9; + bulk_advance(9, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xCC: // uint 8 - size = 2; + bulk_advance(2, head); accepted = emit_unsigned(data[1]); break; case 0xCD: // uint 16 - size = 3; + bulk_advance(3, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xCE: // uint 32 - size = 5; + bulk_advance(5, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xCF: // uint 64 - size = 9; + bulk_advance(9, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xD0: // int 8 - size = 2; + bulk_advance(2, head); accepted = emit_signed(static_cast(data[1])); break; case 0xD1: // int 16 - size = 3; + bulk_advance(3, head); accepted = emit_signed(load_big_endian(data + 1)); break; case 0xD2: // int 32 - size = 5; + bulk_advance(5, head); accepted = emit_signed(load_big_endian(data + 1)); break; case 0xD3: // int 64 - size = 9; + bulk_advance(9, head); accepted = emit_signed(load_big_endian(data + 1)); break; default: if (head <= 0x7F) { // positive fixint - size = 1; + bulk_advance(1, head); accepted = sax->number_unsigned(static_cast(head)); } else if (head >= 0xE0) { // negative fixint - size = 1; + bulk_advance(1, head); accepted = sax->number_integer(static_cast(head)); } else @@ -2943,10 +2952,6 @@ class binary_reader } break; } - // as if read by get() and get_number(): current is the last byte - ia.bulk_skip(size); - chars_read += size; - current = data[size - 1]; --remaining; if (JSON_HEDLEY_UNLIKELY(!accepted)) { @@ -3850,6 +3855,7 @@ class binary_reader const auto* const data = reinterpret_cast(ia.bulk_data()); // UBJSON stores numbers big-endian, BJData little-endian const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + const std::size_t start = chars_read; std::size_t read = 0; bool accepted = true; while (read < count && accepted) @@ -3861,16 +3867,39 @@ class binary_reader byte_swap(number); } ++read; + // where passing the value on can report an error (see + // emit_float), the position is that after the value, as if it + // was read by get_number() + set_chars_read(start + (read * size), std::integral_constant()> {}); accepted = emit_ubjson_number(number); } - // as if read by get_number(): current is the last byte read + // get_number() does not change current ia.bulk_skip(read * size); - chars_read += read * size; - current = data[(read * size) - 1]; + chars_read = start + (read * size); remaining -= read; return accepted; } + /*! + @brief whether a value of type @a NumberType can be finite and still + overflow number_float_t, which @ref emit_float reports as an error + + False for the default number types, so that the loop in @ref + parse_ubjson_typed_run need not track the position per value. + */ + template + static constexpr bool may_overflow_number_float() noexcept + { + return static_cast((std::numeric_limits::max)()) < static_cast((std::numeric_limits::max)()); + } + + void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept + { + chars_read = position; + } + + void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {} + template::value, int> = 0> bool emit_ubjson_number(const NumberType number) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 473bfa894e..73f164ccd8 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20605,6 +20605,21 @@ class binary_reader return result; } + /*! + @brief move past a value of @a size bytes, starting with @a head, that a + bulk run reads, as get() and get_number() would + + Called before the value is passed on, so that an error reported while + passing it on (see @ref emit_float) has the position and the last byte + that reading the value byte by byte gives. + */ + void bulk_advance(const std::size_t size, const unsigned char head) + { + ia.bulk_skip(size); + chars_read += size; + current = head; + } + /*! @brief read the integers, floats, booleans, and nulls that follow in a definite-length array, without the per-element bookkeeping of @@ -20630,71 +20645,70 @@ class binary_reader { const auto* const data = reinterpret_cast(ia.bulk_data()); const unsigned char head = data[0]; - std::size_t size = 0; bool accepted = false; switch (head) { case 0x18: - size = 2; + bulk_advance(2, head); accepted = emit_unsigned(data[1]); break; case 0x19: - size = 3; + bulk_advance(3, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x1A: - size = 5; + bulk_advance(5, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x1B: - size = 9; + bulk_advance(9, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0x38: - size = 2; + bulk_advance(2, head); accepted = emit_cbor_negative_integer(data[1]); break; case 0x39: - size = 3; + bulk_advance(3, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0x3A: - size = 5; + bulk_advance(5, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0x3B: - size = 9; + bulk_advance(9, head); accepted = emit_cbor_negative_integer(load_big_endian(data + 1)); break; case 0xF4: - size = 1; + bulk_advance(1, head); accepted = sax->boolean(false); break; case 0xF5: - size = 1; + bulk_advance(1, head); accepted = sax->boolean(true); break; case 0xF6: - size = 1; + bulk_advance(1, head); accepted = sax->null(); break; case 0xFA: - size = 5; + bulk_advance(5, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xFB: - size = 9; + bulk_advance(9, head); accepted = emit_float(load_big_endian(data + 1)); break; default: if (head <= 0x17) { - size = 1; + bulk_advance(1, head); accepted = sax->number_unsigned(static_cast(head)); } else if (head >= 0x20 && head <= 0x37) { - size = 1; + bulk_advance(1, head); accepted = sax->number_integer(static_cast(0x20 - 1 - head)); } else @@ -20704,10 +20718,6 @@ class binary_reader } break; } - // as if read by get() and get_number(): current is the head byte - ia.bulk_skip(size); - chars_read += size; - current = head; --remaining; if (JSON_HEDLEY_UNLIKELY(!accepted)) { @@ -21545,73 +21555,72 @@ class binary_reader { const auto* const data = reinterpret_cast(ia.bulk_data()); const unsigned char head = data[0]; - std::size_t size = 0; bool accepted = false; switch (head) { case 0xC0: // nil - size = 1; + bulk_advance(1, head); accepted = sax->null(); break; case 0xC2: // false - size = 1; + bulk_advance(1, head); accepted = sax->boolean(false); break; case 0xC3: // true - size = 1; + bulk_advance(1, head); accepted = sax->boolean(true); break; case 0xCA: // float 32 - size = 5; + bulk_advance(5, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xCB: // float 64 - size = 9; + bulk_advance(9, head); accepted = emit_float(load_big_endian(data + 1)); break; case 0xCC: // uint 8 - size = 2; + bulk_advance(2, head); accepted = emit_unsigned(data[1]); break; case 0xCD: // uint 16 - size = 3; + bulk_advance(3, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xCE: // uint 32 - size = 5; + bulk_advance(5, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xCF: // uint 64 - size = 9; + bulk_advance(9, head); accepted = emit_unsigned(load_big_endian(data + 1)); break; case 0xD0: // int 8 - size = 2; + bulk_advance(2, head); accepted = emit_signed(static_cast(data[1])); break; case 0xD1: // int 16 - size = 3; + bulk_advance(3, head); accepted = emit_signed(load_big_endian(data + 1)); break; case 0xD2: // int 32 - size = 5; + bulk_advance(5, head); accepted = emit_signed(load_big_endian(data + 1)); break; case 0xD3: // int 64 - size = 9; + bulk_advance(9, head); accepted = emit_signed(load_big_endian(data + 1)); break; default: if (head <= 0x7F) { // positive fixint - size = 1; + bulk_advance(1, head); accepted = sax->number_unsigned(static_cast(head)); } else if (head >= 0xE0) { // negative fixint - size = 1; + bulk_advance(1, head); accepted = sax->number_integer(static_cast(head)); } else @@ -21621,10 +21630,6 @@ class binary_reader } break; } - // as if read by get() and get_number(): current is the last byte - ia.bulk_skip(size); - chars_read += size; - current = data[size - 1]; --remaining; if (JSON_HEDLEY_UNLIKELY(!accepted)) { @@ -22528,6 +22533,7 @@ class binary_reader const auto* const data = reinterpret_cast(ia.bulk_data()); // UBJSON stores numbers big-endian, BJData little-endian const bool swap = is_little_endian != (input_format == input_format_t::bjdata); + const std::size_t start = chars_read; std::size_t read = 0; bool accepted = true; while (read < count && accepted) @@ -22539,16 +22545,39 @@ class binary_reader byte_swap(number); } ++read; + // where passing the value on can report an error (see + // emit_float), the position is that after the value, as if it + // was read by get_number() + set_chars_read(start + (read * size), std::integral_constant()> {}); accepted = emit_ubjson_number(number); } - // as if read by get_number(): current is the last byte read + // get_number() does not change current ia.bulk_skip(read * size); - chars_read += read * size; - current = data[(read * size) - 1]; + chars_read = start + (read * size); remaining -= read; return accepted; } + /*! + @brief whether a value of type @a NumberType can be finite and still + overflow number_float_t, which @ref emit_float reports as an error + + False for the default number types, so that the loop in @ref + parse_ubjson_typed_run need not track the position per value. + */ + template + static constexpr bool may_overflow_number_float() noexcept + { + return static_cast((std::numeric_limits::max)()) < static_cast((std::numeric_limits::max)()); + } + + void set_chars_read(const std::size_t position, std::true_type /*track*/) noexcept + { + chars_read = position; + } + + void set_chars_read(const std::size_t /*position*/, std::false_type /*track*/) const noexcept {} + template::value, int> = 0> bool emit_ubjson_number(const NumberType number) { diff --git a/tests/src/unit-binary_formats.cpp b/tests/src/unit-binary_formats.cpp index 333252190b..f9778d9e74 100644 --- a/tests/src/unit-binary_formats.cpp +++ b/tests/src/unit-binary_formats.cpp @@ -14,6 +14,7 @@ using nlohmann::json; #include #include #include +#include #include #include #include @@ -367,3 +368,125 @@ TEST_CASE("Binary formats with narrow number types") CHECK(format.decode(format.encode(json(1e300)), false).is_discarded()); } } + +namespace +{ +// records the SAX events of narrow_json, including the arguments of parse_error +struct narrow_event_recorder +{ + using number_integer_t = narrow_json::number_integer_t; + using number_unsigned_t = narrow_json::number_unsigned_t; + using number_float_t = narrow_json::number_float_t; + using string_t = narrow_json::string_t; + using binary_t = narrow_json::binary_t; + + bool null() + { + events.emplace_back("null"); + return true; + } + bool boolean(bool val) + { + events.emplace_back(val ? "true" : "false"); + return true; + } + bool number_integer(number_integer_t val) + { + events.push_back(std::to_string(val)); + return true; + } + bool number_unsigned(number_unsigned_t val) + { + events.push_back(std::to_string(val)); + return true; + } + bool number_float(number_float_t val, const string_t& /*unused*/) + { + events.push_back(std::to_string(val)); + return true; + } + bool string(string_t& val) + { + events.push_back(val); + return true; + } + bool binary(binary_t& /*unused*/) + { + events.emplace_back("binary"); + return true; + } + bool start_object(std::size_t /*unused*/) + { + events.emplace_back("{"); + return true; + } + bool key(string_t& val) + { + events.push_back(val); + return true; + } + bool end_object() + { + events.emplace_back("}"); + return true; + } + bool start_array(std::size_t /*unused*/) + { + events.emplace_back("["); + return true; + } + bool end_array() + { + events.emplace_back("]"); + return true; + } + bool parse_error(std::size_t position, const std::string& last_token, const nlohmann::detail::exception& ex) + { + events.push_back("parse_error(" + std::to_string(position) + ", " + last_token + ", " + ex.what() + ")"); + return false; + } + + std::vector events {}; // NOLINT(readability-redundant-member-init) +}; +} // namespace + +TEST_CASE("Binary formats with narrow number types from contiguous and non-contiguous input") +{ + // contiguous input reads runs of numbers in one loop; a number that + // overflows number_float_t must be reported with the same position and + // last byte as when the input is read byte by byte + const json values = {1, 70000, -70000, 1.5, 2.5, 1e300, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5}; + const json doubles = {1.5, 2.5, 3.5, 1e300, 4.5, 5.5, 6.5, 7.5, 8.5}; + + struct binary_input + { + const char* name; + json::input_format_t format; + bytes data; + }; + const std::vector inputs = + { + {"CBOR", json::input_format_t::cbor, json::to_cbor(values)}, + {"CBOR doubles", json::input_format_t::cbor, json::to_cbor(doubles)}, + {"MessagePack", json::input_format_t::msgpack, json::to_msgpack(values)}, + {"MessagePack doubles", json::input_format_t::msgpack, json::to_msgpack(doubles)}, + {"UBJSON", json::input_format_t::ubjson, json::to_ubjson(values, true)}, + {"UBJSON typed doubles", json::input_format_t::ubjson, json::to_ubjson(doubles, true, true)}, + {"BJData", json::input_format_t::bjdata, json::to_bjdata(values, true)}, + {"BJData typed doubles", json::input_format_t::bjdata, json::to_bjdata(doubles, true, true)}, + }; + + for (const auto& input : inputs) + { + const std::string name = input.name; + CAPTURE(name) + const std::list bytewise(input.data.begin(), input.data.end()); + narrow_event_recorder from_contiguous; + narrow_event_recorder from_bytewise; + CHECK_FALSE(narrow_json::sax_parse(input.data, &from_contiguous, input.format)); + CHECK_FALSE(narrow_json::sax_parse(bytewise.begin(), bytewise.end(), &from_bytewise, input.format)); + CHECK(from_contiguous.events == from_bytewise.events); + REQUIRE(!from_contiguous.events.empty()); + CHECK(from_contiguous.events.back().find("out_of_range.406") != std::string::npos); + } +}