diff --git a/docs/mkdocs/docs/api/basic_json/swap.md b/docs/mkdocs/docs/api/basic_json/swap.md index 3a3d288fb..aa5aa6c4c 100644 --- a/docs/mkdocs/docs/api/basic_json/swap.md +++ b/docs/mkdocs/docs/api/basic_json/swap.md @@ -34,10 +34,14 @@ void swap(typename binary_t::container_type& other); ``` 1. Exchanges the contents of the JSON value with those of `other`. Does not invoke any move, copy, or swap operations on - individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. + individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 2. Exchanges the contents of the JSON value from `left` with those of `right`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is - invalidated. Implemented as a friend function callable via ADL. + invalidated. Implemented as a friend function callable via ADL. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 3. Exchanges the contents of a JSON array with those of `other`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. 4. Exchanges the contents of a JSON object with those of `other`. Does not invoke any move, copy, or swap operations on diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index ec3e462c1..5c9f72fc3 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -394,7 +394,14 @@ plugins: - http://nlohmann.github.io/json/* - https://nlohmann.github.io/json/* - mailto:* - - privacy + - privacy: + # repology.org refuses requests from GitHub Actions runners, which made + # the privacy plugin abort the whole build when it could not download the + # package badges (the fetch fails, then reading the missing cache entry + # raises FileNotFoundError). Readers' browsers are served normally, so + # leave these badges as external references instead of self-hosting them. + assets_exclude: + - repology.org/* - llmstxt: markdown_description: > JSON for Modern C++ is a C++11 header-only library implementing a JSON diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index efa93070d..e0343fd0b 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1434,7 +1434,7 @@ class binary_reader // a copy, not a reference: it must stay valid across the // pop_back() below, which destroys the container_stack element // it would otherwise alias - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); bool at_end = false; if (top.remaining != npos) @@ -2211,7 +2211,7 @@ class binary_reader // would otherwise alias. for (;;) { - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); if (top.remaining != npos) { diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 1627d2326..37d0ab270 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,6 +8,7 @@ #pragma once +#include // min #include #include // string #include // enable_if_t @@ -17,6 +18,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -150,6 +152,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -305,6 +330,11 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } + return true; } @@ -683,6 +713,11 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } } return true; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 016a4dd27..741c08b53 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3576,6 +3576,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 150118a1c..2bb7a5b7b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7892,6 +7892,7 @@ NLOHMANN_JSON_NAMESPACE_END +#include // min #include #include // string #include // enable_if_t @@ -10729,6 +10730,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -10863,6 +10866,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -11018,6 +11044,11 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } + return true; } @@ -11396,6 +11427,11 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); + } } return true; @@ -13383,7 +13419,7 @@ class binary_reader // a copy, not a reference: it must stay valid across the // pop_back() below, which destroys the container_stack element // it would otherwise alias - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); bool at_end = false; if (top.remaining != npos) @@ -14160,7 +14196,7 @@ class binary_reader // would otherwise alias. for (;;) { - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); if (top.remaining != npos) { @@ -27309,6 +27345,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 9522df5a4..f949f3f12 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -252,4 +252,323 @@ static void BinaryToCbor(benchmark::State& state) } BENCHMARK(BinaryToCbor)->RangeMultiplier(2)->Range(8, 8 << 12); +////////////////////////////////////////////////////////////////////////////// +// parse binary formats +////////////////////////////////////////////////////////////////////////////// + +// Only MessagePack had a read benchmark (FromMsgpack above, left untouched so +// its numbers stay comparable across releases). The benchmarks below cover the +// other formats, and read from a contiguous buffer as well as from a FILE*: +// most callers pass a container, and the two adapters compile to different +// code. The test data repository ships JSON only, so the input for each is +// derived at setup time by serializing a parsed test file. + +/// binary format to benchmark; the _optimized variants add UBJSON/BJData size +/// and type annotations, which the readers handle in a separate code path +enum class binary_format +{ + cbor, + msgpack, + ubjson, + ubjson_optimized, + bjdata, + bjdata_optimized, + bson +}; + +static std::vector to_binary(const json& j, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::to_cbor(j); + case binary_format::msgpack: + return json::to_msgpack(j); + case binary_format::ubjson: + return json::to_ubjson(j); + case binary_format::ubjson_optimized: + return json::to_ubjson(j, true, true); + case binary_format::bjdata: + return json::to_bjdata(j); + case binary_format::bjdata_optimized: + return json::to_bjdata(j, true, true); + case binary_format::bson: + default: + return json::to_bson(j); + } +} + +static json from_binary(const std::vector& bytes, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::from_cbor(bytes); + case binary_format::msgpack: + return json::from_msgpack(bytes); + case binary_format::ubjson: + case binary_format::ubjson_optimized: + return json::from_ubjson(bytes); + case binary_format::bjdata: + case binary_format::bjdata_optimized: + return json::from_bjdata(bytes); + case binary_format::bson: + default: + return json::from_bson(bytes); + } +} + +static json from_binary(std::FILE* file, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::from_cbor(file); + case binary_format::msgpack: + return json::from_msgpack(file); + case binary_format::ubjson: + case binary_format::ubjson_optimized: + return json::from_ubjson(file); + case binary_format::bjdata: + case binary_format::bjdata_optimized: + return json::from_bjdata(file); + case binary_format::bson: + default: + return json::from_bson(file); + } +} + +/*! +@brief serialize a parsed test file to @a format + +Returns an empty vector and marks the benchmark as skipped if the file cannot +be represented in the format, rather than letting the exception escape: BSON +requires an object at the top level, and several test files are arrays. +*/ +static std::vector binary_input(benchmark::State& state, const char* filename, const binary_format format) +{ + std::ifstream f(filename); + std::string const str((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + const json j = json::parse(str); + + if (format == binary_format::bson && !j.is_object()) + { + state.SkipWithError("BSON requires an object at the top level"); + return {}; + } + + return to_binary(j, format); +} + +static void FromBinaryBuffer(benchmark::State& state, const char* filename, const binary_format format) +{ + const std::vector bytes = binary_input(state, filename, format); + if (bytes.empty()) + { + return; + } + + for (auto _ : state) + { + // the value is destroyed outside the timed section, because destroying + // a large DOM is not what this benchmark measures + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = from_binary(bytes, format); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / floats, TEST_DATA_DIRECTORY "/regression/floats.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson_optimized / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson_optimized / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata_optimized / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bjdata_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata_optimized / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata_optimized); +// BSON requires an object at the top level, so the array-rooted test files +// (jeopardy and the regression files) cannot be captured here +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bson); + +static void FromBinaryFile(benchmark::State& state, const char* filename, const binary_format format) +{ + const std::vector bytes = binary_input(state, filename, format); + if (bytes.empty()) + { + return; + } + + const char* tmp = "benchmark_input.bin"; + std::ofstream o(tmp, std::ios::binary); + o.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + o.flush(); + o.close(); + + for (auto _ : state) + { + state.PauseTiming(); + auto* j = new json(); + auto* file = std::fopen(tmp, "rb"); + state.ResumeTiming(); + + *j = from_binary(file, format); + + state.PauseTiming(); + std::fclose(file); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryFile, cbor / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryFile, cbor / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryFile, ubjson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryFile, ubjson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryFile, bjdata / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryFile, bson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bson); + +////////////////////////////////////////////////////////////////////////////// +// parse binary formats: value shapes +////////////////////////////////////////////////////////////////////////////// + +// The test files above are wide and shallow, but the readers' cost is per +// container, so these cover the shapes that stress the container handling +// itself. Every shape is wrapped in an object so that BSON, which requires an +// object at the top level, measures the same value as the other formats. + +/// deeply nested arrays: one container per level, no other work +static json make_nested() +{ + json nested = json::array(); + json* p = &nested; + for (std::size_t i = 1; i < 1000; ++i) + { + p->push_back(json::array()); + p = &p->operator[](0); + } + + json j = json::object(); + j["data"] = std::move(nested); + return j; +} + +/// many sibling containers: maximum container churn, minimum nesting +static json make_containers() +{ + json data = json::array(); + for (std::size_t i = 0; i < 100000; ++i) + { + data.push_back(json::array({1, 2})); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + +/// one flat array of numbers: the scalar decoding path, which must not move +static json make_scalars() +{ + json data = json::array(); + for (std::size_t i = 0; i < 1000000; ++i) + { + data.push_back(i); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + +static void FromBinaryShape(benchmark::State& state, json (*build)(), const binary_format format) +{ + const std::vector bytes = to_binary(build(), format); + + for (auto _ : state) + { + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = from_binary(bytes, format); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryShape, nested / cbor, make_nested, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, nested / msgpack, make_nested, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, nested / ubjson, make_nested, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, nested / bjdata, make_nested, binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryShape, nested / bson, make_nested, binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryShape, containers / cbor, make_containers, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, containers / msgpack, make_containers, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson, make_containers, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson_optimized, make_containers, binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryShape, containers / bjdata, make_containers, binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryShape, containers / bson, make_containers, binary_format::bson); +// BSON names every array element, so a large array measures key generation +// rather than scalar decoding and is left out here +BENCHMARK_CAPTURE(FromBinaryShape, scalars / cbor, make_scalars, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / msgpack, make_scalars, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / ubjson, make_scalars, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / bjdata, make_scalars, binary_format::bjdata); + +/*! +@brief parse an indefinite-length CBOR string + +The writer never emits this form, so the input is assembled by hand: 0x7F +opens the string, each chunk is a one-character string, and 0xFF closes it. +*/ +static void FromCborChunkedString(benchmark::State& state, const std::size_t chunks) +{ + std::vector bytes; + bytes.reserve(2 * chunks + 2); + bytes.push_back(0x7F); + for (std::size_t i = 0; i < chunks; ++i) + { + bytes.push_back(0x61); // string of length 1 + bytes.push_back(0x61); // 'a' + } + bytes.push_back(0xFF); + + for (auto _ : state) + { + json j = json::from_cbor(bytes); + benchmark::DoNotOptimize(j); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); + BENCHMARK_MAIN(); diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 3130bb720..9338e663e 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3551,6 +3551,111 @@ TEST_CASE("BJData") } } +TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no + // element data at all. max_size() for a std::vector is far larger + // than this count, so it does not reject the header outright; the + // (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_bjdata(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_bjdata(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_bjdata(j); + CHECK(json::from_bjdata(packed_plain) == j); + + const auto packed_optimized = json::to_bjdata(j, true, true); + CHECK(json::from_bjdata(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_bjdata(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 032e6641b..4c9107517 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2174,6 +2174,92 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") } } +TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's detail::unknown_size() sentinel (SIZE_MAX), so the + // format-level size check rejects it outright (out_of_range.408, + // "excessive ... size") before the SAX consumer's own max_size() + // check would even run; on a 64-bit platform it passes both of + // those checks and is only found short of data once the (capped) + // reservation looks for element bytes that were never provided + // (parse_error.110). Either is an acceptable, bounded rejection of + // the hostile header -- the property under test is that no path + // attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_cbor(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_cbor(j); + CHECK(json::from_cbor(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_cbor(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor)); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 34ca5db2b..af76e93cc 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2259,6 +2259,86 @@ TEST_CASE("parser class") #endif } +#if JSON_DIAGNOSTIC_POSITIONS + +TEST_CASE("diagnostic positions: value lifetime") +{ + SECTION("copy constructor copies positions, recursively") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + const json a = json::parse(s); + const json b = a; // NOLINT(performance-unnecessary-copy-initialization) + + CHECK(b.start_pos() == a.start_pos()); + CHECK(b.end_pos() == a.end_pos()); + CHECK(b["b"].start_pos() == a["b"].start_pos()); + CHECK(b["b"].end_pos() == a["b"].end_pos()); + } + + SECTION("move constructor resets the moved-from value to npos") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + json a = json::parse(s); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + + const json b(std::move(a)); + + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + } + + SECTION("swap() exchanges positions along with the values") + { + // basic_json::swap() (and the friend swap() that forwards to it) used + // to swap only m_data.m_type/m_data.m_value, leaving + // start_position/end_position untouched -- unlike copy-assignment's + // operator=(basic_json), which swaps positions as part of its + // copy-and-swap implementation. After swap(a, b), each value ended up + // with the *other* value's content but its *own* original position. + // This is now fixed so that swap() is consistent with copy-assignment. + json a = json::parse(R"({"a":1})"); + json b = json::parse(R"([1,2,3,4,5])"); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + const auto b_start = b.start_pos(); + const auto b_end = b.end_pos(); + // lengths (and thus end positions) differ, which is enough to tell + // after the swap whether positions actually moved with the values + CHECK(a_end != b_end); + + using std::swap; + swap(a, b); + + CHECK(a == json::parse(R"([1,2,3,4,5])")); + CHECK(b == json::parse(R"({"a":1})")); + + CHECK(a.start_pos() == b_start); + CHECK(a.end_pos() == b_end); + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + // member swap() behaves the same as the free function + json c = json::parse(R"({"a":1})"); + json d = json::parse(R"([1,2,3,4,5])"); + const auto c_start = c.start_pos(); + const auto c_end = c.end_pos(); + const auto d_start = d.start_pos(); + const auto d_end = d.end_pos(); + + c.swap(d); + + CHECK(c.start_pos() == d_start); + CHECK(c.end_pos() == d_end); + CHECK(d.start_pos() == c_start); + CHECK(d.end_pos() == c_end); + } +} +#endif + // this test relies on parse errors being thrown, so it is skipped when // exceptions are disabled (json::parse aborts instead of throwing there) #if !defined(JSON_NOEXCEPTION) diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 75c2ae464..74f7f4969 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1597,6 +1597,91 @@ TEST_CASE("MessagePack") } } +TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's SIZE_MAX, which some size-narrowing checks treat the + // same as detail::unknown_size(); it may then be rejected before + // the SAX consumer's own max_size() check (out_of_range.408) rather + // than being accepted and only found short of data once the + // (capped) reservation looks for element bytes that were never + // provided (parse_error.110). Either is an acceptable, bounded + // rejection of the hostile header -- the property under test is + // that no path attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_msgpack(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_msgpack(j); + CHECK(json::from_msgpack(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_msgpack(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack)); + } +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("MessagePack nesting does not consume the call stack") { diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index cb2ed59a6..11c6a7da8 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -27,6 +27,7 @@ using ordered_json = nlohmann::ordered_json; #endif #include +#include #include #include #include @@ -896,4 +897,49 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with } +TEST_CASE("regression test #5476 - array type without reserve()") +{ + // the capacity reserved for definite-length arrays must not require the + // array type to have a reserve() member function + using deque_json = nlohmann::basic_json; + + SECTION("std::deque") + { + const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); + CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); + + // the binary formats pass a definite length to start_array() + CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); + CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); + + // parse() instantiates the callback parser as well, which reserves too + const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept + { + return true; + }); + CHECK(with_callback == deque_json({1, 2, 3})); + } + + SECTION("std::vector still reserves") + { + json array = json::array(); + for (int i = 0; i < 100; ++i) + { + array.push_back(i); + } + + const auto j = json::from_cbor(json::to_cbor(array)); + CHECK(j == array); + CHECK(j.get_ref().capacity() >= 100); + } + + SECTION("the reservation stays capped") + { + // CBOR array announcing 2^32-1 elements, but truncated right after the + // header: the input must be rejected without reserving that capacity + const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_cbor(truncated, true, false).is_discarded()); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index eddf59f2c..511108c64 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -469,7 +469,7 @@ TEST_CASE("serialization of strings (bulk fast path)") SECTION("invalid UTF-8 handling is unaffected by the fast path") { const json j = std::string("valid\xff" "more"); - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\""); CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\""); CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\""); diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 2ebb55b90..aafbbf5a4 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2315,6 +2315,112 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } +TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all. + // max_size() for a std::vector is far larger than this count, so it + // does not reject the header outright; the (capped) reservation must + // not attempt to allocate space for billions of elements before the + // missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_ubjson(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_ubjson(j); + CHECK(json::from_ubjson(packed_plain) == j); + + const auto packed_optimized = json::to_ubjson(j, true, true); + CHECK(json::from_ubjson(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_ubjson(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson)); + } +} + + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value")