diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 8732e52b4..399c567fa 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2088,7 +2088,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -2339,6 +2340,197 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and every + integer fits the 64 bits BSON has, so that writing an array of + numbers needs no range check + */ + using bson_single_pass_sink = std::integral_constant < bool, + is_detected::value + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits >; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref patch_bson_size + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + write_number(0, true); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + std::array bytes{}; + store_little_endian(bytes.data(), static_cast(size)); + oa.overwrite(pos, bytes.data(), bytes.size()); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const std::size_t limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template < typename SinkType = OutputSinkType, + typename std::enable_if < is_detected::value, int >::type = 0 > + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -2350,6 +2542,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e40d98548..5858c2667 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -227,6 +227,29 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace @a length characters written before, from position @a pos + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept + { + JSON_ASSERT(pos + length <= position()); + std::memcpy(v->data() + pos, s, length * sizeof(CharType)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 10b42dac1..457129749 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -26870,6 +26870,29 @@ class output_vector_sink void flush() noexcept {} + /// the number of characters written so far, the position of the next one + std::size_t written() const noexcept + { + return position(); + } + + /// @brief replace @a length characters written before, from position @a pos + /// + /// Lets a writer put a length in front of data it has written since, such + /// as the size of a BSON document, without computing it in advance. + void overwrite(const std::size_t pos, const CharType* s, const std::size_t length) noexcept + { + JSON_ASSERT(pos + length <= position()); + std::memcpy(v->data() + pos, s, length * sizeof(CharType)); + } + + /// drop the characters written from position @a pos on + void rewind(const std::size_t pos) noexcept + { + JSON_ASSERT(pos <= position()); + cursor = v->data() + pos; + } + private: /// number of characters of *v written so far std::size_t position() const noexcept @@ -29314,7 +29337,8 @@ class binary_writer /// arrays: the first index with one more digit; only maintained by /// @ref calc_bson_sizes std::uint64_t index_boundary = 10; - /// @ref calc_bson_sizes only: where its size goes in the table + /// @ref calc_bson_sizes: where its size goes in the table; @ref + /// write_bson_document_single_pass: the position of its size field std::size_t size_slot; /// @ref calc_bson_sizes only: the size of its entries seen so far std::size_t entries_size = 0; @@ -29565,6 +29589,197 @@ class binary_writer } } + template + using sink_written_t = decltype(std::declval().written()); + + /*! + @brief whether BSON documents can be written in a single pass: the sink + can overwrite what it has written (output_vector_sink), and every + integer fits the 64 bits BSON has, so that writing an array of + numbers needs no range check + */ + using bson_single_pass_sink = std::integral_constant < bool, + is_detected::value + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits + && std::numeric_limits::is_specialized + && std::numeric_limits::digits <= std::numeric_limits::digits >; + + /// write a placeholder for the size of an object or array whose entries + /// follow; @return its position, for @ref patch_bson_size + std::size_t write_bson_size_placeholder() + { + const std::size_t pos = oa.written(); + write_number(0, true); + return pos; + } + + /// end the object or array whose size is at @a pos and write its size + /// there: everything from that field to the terminating null byte + /// @return false (and the size is not written) if it does not fit + bool close_bson_container(const std::size_t pos) + { + oa.write_character(to_char_type(0x00)); + const std::size_t size = oa.written() - pos; + if (JSON_HEDLEY_UNLIKELY(!value_in_range_of(size))) + { + return false; + } + std::array bytes{}; + store_little_endian(bytes.data(), static_cast(size)); + oa.overwrite(pos, bytes.data(), bytes.size()); + return true; + } + + /*! + @brief check the value @a j of a BSON entry as @ref calc_bson_value_size + does, for @ref write_bson_document_single_pass + @return at least the size of the value + A number, boolean, or null needs no check there (its integers have at most + 64 bits) and has at most 8 characters; a discarded value is rejected with + the same exception when it is written. + */ + std::size_t check_bson_value(const BasicJsonType& j) + { + return (j.is_string() || j.is_binary()) ? calc_bson_value_size(j) : 8; + } + + /// whether an entry of @a entry_size characters keeps the document that + /// begins at @a start, closed by @a open_containers null bytes, within + /// the 32-bit size of BSON + bool bson_entry_fits(const std::size_t start, const std::size_t entry_size, const std::size_t open_containers) const noexcept + { + const std::size_t limit = static_cast((std::numeric_limits::max)()); + const std::size_t written = oa.written() - start; + return written <= limit && entry_size <= limit - written && open_containers <= limit - written - entry_size; + } + + /*! + @brief Serializes the JSON object @a document as a BSON document in a + single pass, writing the size of each object and array once it is + complete + + Checks every key and value as @ref calc_bson_sizes does, in the same + order, so the same exception is thrown; the output written up to then is + discarded with the vector it was written to. + + @return false as soon as the document or a string, binary value, object, + or array in it would be larger than the 32-bit BSON sizes allow; + nothing large has been written then, and the caller writes the + document in two passes, which reports that error + */ + template < typename SinkType = OutputSinkType, + typename std::enable_if < is_detected::value, int >::type = 0 > + bool write_bson_document_single_pass(const BasicJsonType& document) + { + const std::size_t start = oa.written(); + // the object or array whose entries are being written (with the + // position of its size), and the ones it is in + bson_frame current(&document, write_bson_size_placeholder()); + std::vector parents; + + while (true) + { + const BasicJsonType* nested = nullptr; + std::size_t nested_size_pos = 0; + if (current.value->is_object()) + { + const auto& object = *current.value->m_data.m_value.object; + while (nested == nullptr && current.member != object.cend()) + { + const auto& el = *current.member; + ++current.member; + const std::size_t header_size = calc_bson_entry_header_size(el.first, el.second); + const std::size_t value_size = el.second.is_structured() ? 0 : check_bson_value(el.second); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, header_size + value_size, parents.size() + 1))) + { + return false; + } + if (el.second.is_structured()) + { + write_bson_entry_header(el.first, el.second.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = &el.second; + } + else + { + write_bson_value(el.first, el.second); + } + } + } + else + { + const auto& array = *current.value->m_data.m_value.array; + while (nested == nullptr && current.index < array.size()) + { + if (is_bson_fixed_size(array[current.index])) + { + write_bson_array_fixed_entries(current); + if (current.index == array.size()) + { + break; + } + } + + // a string, binary value, object or array + const BasicJsonType& el = array[current.index]; + const string_t index_name(current.index_name.data(), current.index_name_size); + current.next_index(); + const std::size_t value_size = el.is_structured() ? 0 : check_bson_value(el); + if (JSON_HEDLEY_UNLIKELY(!bson_entry_fits(start, 2 + index_name.size() + value_size, parents.size() + 1))) + { + return false; + } + if (el.is_structured()) + { + write_bson_entry_header(index_name, el.is_object() ? 0x03 : 0x04); + nested_size_pos = write_bson_size_placeholder(); + nested = ⪙ + } + else + { + write_bson_value(index_name, el); + } + } + } + + if (nested != nullptr) + { + if (nested->is_array()) + { + // an array of booleans, numbers and null is written without a frame on the stack + bson_frame leaf(nested, nested_size_pos); + write_bson_array_fixed_entries(leaf); + if (leaf.index == nested->m_data.m_value.array->size()) + { + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(nested_size_pos))) + { + return false; + } + continue; + } + parents.push_back(std::move(current)); + current = std::move(leaf); + continue; + } + parents.push_back(std::move(current)); + current = bson_frame(nested, nested_size_pos); + continue; + } + + if (JSON_HEDLEY_UNLIKELY(!close_bson_container(current.size_slot))) + { + return false; + } + if (parents.empty()) + { + return true; + } + current = std::move(parents.back()); + parents.pop_back(); + } + } + /*! @brief Serializes the JSON object @a document as a BSON document @@ -29576,6 +29791,39 @@ class binary_writer @pre document.type() == value_t::object */ void write_bson_document(const BasicJsonType& document) + { + write_bson_document(document, bson_single_pass_sink()); + } + + /// the sink can overwrite: one pass, unless the error handler replaces or + /// drops ill-formed UTF-8, which changes the sizes the checks compute + void write_bson_document(const BasicJsonType& document, std::true_type /*single_pass*/) + { + const std::size_t start = oa.written(); + if ((error_handler == error_handler_t::keep || error_handler == error_handler_t::strict) + && write_bson_document_single_pass(document)) + { + return; + } + // a length does not fit: start over, so that it is reported as the + // two passes report it, before anything is written + oa.rewind(start); + write_bson_document_two_pass(document); + } + + void write_bson_document(const BasicJsonType& document, std::false_type /*single_pass*/) + { + write_bson_document_two_pass(document); + } + + /*! + @brief Serializes the JSON object @a document as a BSON document: the + sizes of all objects and arrays first, then the entries + + @param[in] document the JSON object to serialize + @pre document.type() == value_t::object + */ + void write_bson_document_two_pass(const BasicJsonType& document) { std::vector nested_sizes; const std::size_t document_size = calc_bson_sizes(document, nested_sizes); diff --git a/tests/src/unit-bson.cpp b/tests/src/unit-bson.cpp index 8e2d00cbf..3d1482d5e 100644 --- a/tests/src/unit-bson.cpp +++ b/tests/src/unit-bson.cpp @@ -2267,3 +2267,132 @@ TEST_CASE("BSON large strings and binaries (chunked reader)") } } } + +TEST_CASE("BSON written in one pass and in two passes") +{ + // to_bson() into a returned vector writes each size once the object or + // array is complete; through an output adapter, all sizes are computed + // first. Both must produce the same bytes and throw the same exceptions. + const auto two_pass = [](const json & j, const json::error_handler_t handler) + { + std::vector result; + json::to_bson(j, result, handler); + return result; + }; + + const auto check_same = [&](const json & j, const json::error_handler_t handler) + { + CAPTURE(j.dump(-1, ' ', true, json::error_handler_t::replace)) + CAPTURE(static_cast(handler)) +#if !defined(JSON_NOEXCEPTION) + std::string one_pass_error; + std::string two_pass_error; + std::vector one; + std::vector two; + try + { + one = json::to_bson(j, handler); + } + catch (const json::exception& e) + { + one_pass_error = e.what(); + } + try + { + two = two_pass(j, handler); + } + catch (const json::exception& e) + { + two_pass_error = e.what(); + } + CHECK(one_pass_error == two_pass_error); + CHECK(one == two); +#else + CHECK(json::to_bson(j, handler) == two_pass(j, handler)); +#endif + }; + + const json::error_handler_t handlers[] = {json::error_handler_t::keep, json::error_handler_t::strict, + json::error_handler_t::replace, json::error_handler_t::ignore + }; + + SECTION("corpus") + { + for (const auto& value : utils::round_trip_corpus::values()) + { + const json document = value.is_object() ? value : json{{"value", value}}; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + + SECTION("nesting and arrays of every length class") + { + json document = json::object(); + document["empty object"] = json::object(); + document["empty array"] = json::array(); + for (const std::size_t n : + { + std::size_t(1), std::size_t(9), std::size_t(10), std::size_t(11), std::size_t(99), std::size_t(100), std::size_t(101), std::size_t(1000) + }) + { + json numbers = json::array(); + json mixed = json::array(); + for (std::size_t i = 0; i < n; ++i) + { + numbers.push_back(i % 3 == 0 ? json(static_cast(i) / 3) : json(static_cast(i) * 1000003)); + mixed.push_back(i % 4 == 0 ? json("s" + std::to_string(i)) : i % 4 == 1 ? json::array({i, json::object({{"k", i}})}) : i % 4 == 2 ? json(nullptr) : json(i % 2 == 0)); + } + document["numbers " + std::to_string(n)] = numbers; + document["mixed " + std::to_string(n)] = mixed; + document["nested " + std::to_string(n)] = json::object({{"a", numbers}, {"b", json::array({mixed, json::object()})}}); + } + document["binary"] = json::binary({1, 2, 3}, 0x80); + document["binary old"] = json::binary({1, 2, 3}, 0x02); + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + + SECTION("errors at every position") + { + // an entry that cannot be written, at each place among valid ones + const std::string bad_utf8 = "\xC3\x28"; + const json bad_entries[] = + { + json::object({{std::string("a\0b", 3), 1}}), + json::object({{"s", bad_utf8}}), + json::object({{bad_utf8, 1}}), + json::array({1, 2, bad_utf8}), + json::object({{"b", json::binary({1}, 0x100)}}), + json::object({{"d", json(json::value_t::discarded)}}), + json::array({json(json::value_t::discarded)}), + }; + for (const auto& bad : bad_entries) + { + for (std::size_t pos = 0; pos < 4; ++pos) + { + json document = json::object(); + json list = json::array(); + for (std::size_t i = 0; i < 4; ++i) + { + if (i == pos) + { + document["x" + std::to_string(i)] = bad; + list.push_back(bad); + } + document["n" + std::to_string(i)] = json::array({i, "str", json::object({{"k", i}})}); + list.push_back(i); + } + document["list"] = list; + for (const auto handler : handlers) + { + check_same(document, handler); + } + } + } + } +}