From d2514a46f7d94bd09ce2110df22127163b8967ee Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 13:15:09 +0200 Subject: [PATCH 1/6] Add benchmarks for the binary readers (#5510) The benchmark suite covered parsing JSON text, dumping, and serializing to CBOR, but only one binary read: FromMsgpack. Nothing measured from_cbor, from_ubjson, from_bjdata or from_bson, so a change to binary_reader.hpp had no baseline to be compared against. Add read benchmarks for every format, in the two shapes that matter: from a contiguous buffer, which is what most callers pass, and from a FILE*, which is what FromMsgpack already measures and which compiles to different code. FromMsgpack itself is left untouched so its numbers stay comparable across releases. The input is derived at setup time by serializing a parsed test file, because the test data repository ships JSON only. The test files are wide and shallow, but the readers' cost is per container, so add three value shapes they do not cover -- deeply nested containers, many sibling containers, and one flat array of numbers -- plus an indefinite-length CBOR string, a form the writer never emits and which therefore has to be assembled by hand. UBJSON and BJData are also captured in their size- and type-annotated form, which the readers handle in a separate code path. BSON requires an object at the top level, so it cannot reuse the array-rooted test files; it is captured on the object-rooted ones, and the shapes are wrapped in an object so every format measures the same value. The setup marks the benchmark as skipped rather than letting the exception escape if that requirement is ever violated. Signed-off-by: Niels Lohmann --- tests/benchmarks/src/benchmarks.cpp | 319 ++++++++++++++++++++++++++++ 1 file changed, 319 insertions(+) diff --git a/tests/benchmarks/src/benchmarks.cpp b/tests/benchmarks/src/benchmarks.cpp index 9522df5a4..f949f3f12 100644 --- a/tests/benchmarks/src/benchmarks.cpp +++ b/tests/benchmarks/src/benchmarks.cpp @@ -252,4 +252,323 @@ static void BinaryToCbor(benchmark::State& state) } BENCHMARK(BinaryToCbor)->RangeMultiplier(2)->Range(8, 8 << 12); +////////////////////////////////////////////////////////////////////////////// +// parse binary formats +////////////////////////////////////////////////////////////////////////////// + +// Only MessagePack had a read benchmark (FromMsgpack above, left untouched so +// its numbers stay comparable across releases). The benchmarks below cover the +// other formats, and read from a contiguous buffer as well as from a FILE*: +// most callers pass a container, and the two adapters compile to different +// code. The test data repository ships JSON only, so the input for each is +// derived at setup time by serializing a parsed test file. + +/// binary format to benchmark; the _optimized variants add UBJSON/BJData size +/// and type annotations, which the readers handle in a separate code path +enum class binary_format +{ + cbor, + msgpack, + ubjson, + ubjson_optimized, + bjdata, + bjdata_optimized, + bson +}; + +static std::vector to_binary(const json& j, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::to_cbor(j); + case binary_format::msgpack: + return json::to_msgpack(j); + case binary_format::ubjson: + return json::to_ubjson(j); + case binary_format::ubjson_optimized: + return json::to_ubjson(j, true, true); + case binary_format::bjdata: + return json::to_bjdata(j); + case binary_format::bjdata_optimized: + return json::to_bjdata(j, true, true); + case binary_format::bson: + default: + return json::to_bson(j); + } +} + +static json from_binary(const std::vector& bytes, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::from_cbor(bytes); + case binary_format::msgpack: + return json::from_msgpack(bytes); + case binary_format::ubjson: + case binary_format::ubjson_optimized: + return json::from_ubjson(bytes); + case binary_format::bjdata: + case binary_format::bjdata_optimized: + return json::from_bjdata(bytes); + case binary_format::bson: + default: + return json::from_bson(bytes); + } +} + +static json from_binary(std::FILE* file, const binary_format format) +{ + switch (format) + { + case binary_format::cbor: + return json::from_cbor(file); + case binary_format::msgpack: + return json::from_msgpack(file); + case binary_format::ubjson: + case binary_format::ubjson_optimized: + return json::from_ubjson(file); + case binary_format::bjdata: + case binary_format::bjdata_optimized: + return json::from_bjdata(file); + case binary_format::bson: + default: + return json::from_bson(file); + } +} + +/*! +@brief serialize a parsed test file to @a format + +Returns an empty vector and marks the benchmark as skipped if the file cannot +be represented in the format, rather than letting the exception escape: BSON +requires an object at the top level, and several test files are arrays. +*/ +static std::vector binary_input(benchmark::State& state, const char* filename, const binary_format format) +{ + std::ifstream f(filename); + std::string const str((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + const json j = json::parse(str); + + if (format == binary_format::bson && !j.is_object()) + { + state.SkipWithError("BSON requires an object at the top level"); + return {}; + } + + return to_binary(j, format); +} + +static void FromBinaryBuffer(benchmark::State& state, const char* filename, const binary_format format) +{ + const std::vector bytes = binary_input(state, filename, format); + if (bytes.empty()) + { + return; + } + + for (auto _ : state) + { + // the value is destroyed outside the timed section, because destroying + // a large DOM is not what this benchmark measures + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = from_binary(bytes, format); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / floats, TEST_DATA_DIRECTORY "/regression/floats.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, cbor / signed_ints, TEST_DATA_DIRECTORY "/regression/signed_ints.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, msgpack / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / jeopardy, TEST_DATA_DIRECTORY "/jeopardy/jeopardy.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson_optimized / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, ubjson_optimized / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata_optimized / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bjdata_optimized); +BENCHMARK_CAPTURE(FromBinaryBuffer, bjdata_optimized / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata_optimized); +// BSON requires an object at the top level, so the array-rooted test files +// (jeopardy and the regression files) cannot be captured here +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / citm_catalog, TEST_DATA_DIRECTORY "/nativejson-benchmark/citm_catalog.json", binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryBuffer, bson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bson); + +static void FromBinaryFile(benchmark::State& state, const char* filename, const binary_format format) +{ + const std::vector bytes = binary_input(state, filename, format); + if (bytes.empty()) + { + return; + } + + const char* tmp = "benchmark_input.bin"; + std::ofstream o(tmp, std::ios::binary); + o.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + o.flush(); + o.close(); + + for (auto _ : state) + { + state.PauseTiming(); + auto* j = new json(); + auto* file = std::fopen(tmp, "rb"); + state.ResumeTiming(); + + *j = from_binary(file, format); + + state.PauseTiming(); + std::fclose(file); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryFile, cbor / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryFile, cbor / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryFile, ubjson / canada, TEST_DATA_DIRECTORY "/nativejson-benchmark/canada.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryFile, ubjson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryFile, bjdata / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryFile, bson / twitter, TEST_DATA_DIRECTORY "/nativejson-benchmark/twitter.json", binary_format::bson); + +////////////////////////////////////////////////////////////////////////////// +// parse binary formats: value shapes +////////////////////////////////////////////////////////////////////////////// + +// The test files above are wide and shallow, but the readers' cost is per +// container, so these cover the shapes that stress the container handling +// itself. Every shape is wrapped in an object so that BSON, which requires an +// object at the top level, measures the same value as the other formats. + +/// deeply nested arrays: one container per level, no other work +static json make_nested() +{ + json nested = json::array(); + json* p = &nested; + for (std::size_t i = 1; i < 1000; ++i) + { + p->push_back(json::array()); + p = &p->operator[](0); + } + + json j = json::object(); + j["data"] = std::move(nested); + return j; +} + +/// many sibling containers: maximum container churn, minimum nesting +static json make_containers() +{ + json data = json::array(); + for (std::size_t i = 0; i < 100000; ++i) + { + data.push_back(json::array({1, 2})); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + +/// one flat array of numbers: the scalar decoding path, which must not move +static json make_scalars() +{ + json data = json::array(); + for (std::size_t i = 0; i < 1000000; ++i) + { + data.push_back(i); + } + + json j = json::object(); + j["data"] = std::move(data); + return j; +} + +static void FromBinaryShape(benchmark::State& state, json (*build)(), const binary_format format) +{ + const std::vector bytes = to_binary(build(), format); + + for (auto _ : state) + { + state.PauseTiming(); + auto* j = new json(); + state.ResumeTiming(); + + *j = from_binary(bytes, format); + + state.PauseTiming(); + delete j; + state.ResumeTiming(); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromBinaryShape, nested / cbor, make_nested, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, nested / msgpack, make_nested, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, nested / ubjson, make_nested, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, nested / bjdata, make_nested, binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryShape, nested / bson, make_nested, binary_format::bson); +BENCHMARK_CAPTURE(FromBinaryShape, containers / cbor, make_containers, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, containers / msgpack, make_containers, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson, make_containers, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, containers / ubjson_optimized, make_containers, binary_format::ubjson_optimized); +BENCHMARK_CAPTURE(FromBinaryShape, containers / bjdata, make_containers, binary_format::bjdata); +BENCHMARK_CAPTURE(FromBinaryShape, containers / bson, make_containers, binary_format::bson); +// BSON names every array element, so a large array measures key generation +// rather than scalar decoding and is left out here +BENCHMARK_CAPTURE(FromBinaryShape, scalars / cbor, make_scalars, binary_format::cbor); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / msgpack, make_scalars, binary_format::msgpack); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / ubjson, make_scalars, binary_format::ubjson); +BENCHMARK_CAPTURE(FromBinaryShape, scalars / bjdata, make_scalars, binary_format::bjdata); + +/*! +@brief parse an indefinite-length CBOR string + +The writer never emits this form, so the input is assembled by hand: 0x7F +opens the string, each chunk is a one-character string, and 0xFF closes it. +*/ +static void FromCborChunkedString(benchmark::State& state, const std::size_t chunks) +{ + std::vector bytes; + bytes.reserve(2 * chunks + 2); + bytes.push_back(0x7F); + for (std::size_t i = 0; i < chunks; ++i) + { + bytes.push_back(0x61); // string of length 1 + bytes.push_back(0x61); // 'a' + } + bytes.push_back(0xFF); + + for (auto _ : state) + { + json j = json::from_cbor(bytes); + benchmark::DoNotOptimize(j); + } + + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +BENCHMARK_CAPTURE(FromCborChunkedString, 10000 chunks, 10000); + BENCHMARK_MAIN(); From aa391dc0a56f8409e2e7aca6e7c9a9d766d44ce8 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 17:55:46 +0200 Subject: [PATCH 2/6] Fix two develop CI regressions: dump() nodiscard warning and binary-reader const-correctness (#5520) * Discard dump()'s [[nodiscard]] return value in an exception-only check CHECK_THROWS_WITH_AS(j.dump(), ...) called dump() only to trigger and catch the exception, but never used the return value. dump() is warn_unused_result, so GCC's pedantic build (-Werror --all-warnings) rejected it as -Werror=unused-result, breaking ci_test_gcc. Wrapped in utils::ignore_return_value(), matching every other such call in this file. Signed-off-by: Niels Lohmann * Mark container_frame top as const in CBOR/UBJSON readers clang-tidy's misc-const-correctness flagged these on PR #5520's CI: the BSON sibling copy was already const, but these two were left mutable even though only container_stack.back().remaining is ever written. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/binary_reader.hpp | 4 ++-- single_include/nlohmann/json.hpp | 4 ++-- tests/src/unit-serialization.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index efa93070d..e0343fd0b 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -1434,7 +1434,7 @@ class binary_reader // a copy, not a reference: it must stay valid across the // pop_back() below, which destroys the container_stack element // it would otherwise alias - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); bool at_end = false; if (top.remaining != npos) @@ -2211,7 +2211,7 @@ class binary_reader // would otherwise alias. for (;;) { - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); if (top.remaining != npos) { diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 150118a1c..3b2233a57 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -13383,7 +13383,7 @@ class binary_reader // a copy, not a reference: it must stay valid across the // pop_back() below, which destroys the container_stack element // it would otherwise alias - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); bool at_end = false; if (top.remaining != npos) @@ -14160,7 +14160,7 @@ class binary_reader // would otherwise alias. for (;;) { - container_frame top = container_stack.back(); + const container_frame top = container_stack.back(); if (top.remaining != npos) { diff --git a/tests/src/unit-serialization.cpp b/tests/src/unit-serialization.cpp index eddf59f2c..511108c64 100644 --- a/tests/src/unit-serialization.cpp +++ b/tests/src/unit-serialization.cpp @@ -469,7 +469,7 @@ TEST_CASE("serialization of strings (bulk fast path)") SECTION("invalid UTF-8 handling is unaffected by the fast path") { const json j = std::string("valid\xff" "more"); - CHECK_THROWS_WITH_AS(j.dump(), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); + CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 5: 0xFF", json::type_error&); CHECK(j.dump(-1, ' ', false, json::error_handler_t::replace) == "\"valid\xef\xbf\xbd" "more\""); CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"valid\\ufffdmore\""); CHECK(j.dump(-1, ' ', false, json::error_handler_t::ignore) == "\"validmore\""); From a50c2537ebf639570c6bbae0a7c9543e3c44f9da Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 12 Sep 2026 21:09:33 +0200 Subject: [PATCH 3/6] Reserve capped array capacity in json_sax_dom_parser::start_array() for definite-length binary arrays (#5476) * Reserve capped array capacity for definite-length binary arrays CBOR, MessagePack, and the optimized [$type#count UBJSON/BJData form all pass an exact element count to sax->start_array(len), but json_sax_dom_parser::start_array() (and the callback variant) only used len for an overflow check against max_size() and never reserved the underlying vector, so each element triggered a reallocation cascade via emplace_back(). Reserve upfront, but cap the reservation at 16384 elements: max_size() for a std::vector is far larger than any realistic input, so an unbounded reserve(len) would let a crafted/truncated header (e.g. CBOR 0x9A + a huge uint32 count with no data) trigger a multi-gigabyte allocation attempt instead of the normal graceful parse_error. With the cap, a hostile length still fails fast with the existing parse_error, while realistic arrays get a single up-front allocation. Signed-off-by: Niels Lohmann * Make the huge-claimed-length DoS regression tests portable across size_t widths On a platform where size_t is narrower than 64 bits (e.g. 32-bit mingw/msvc x86), the previously-hardcoded huge test lengths either collide with that platform's unknown_size() sentinel (CBOR/MessagePack, both using exactly SIZE_MAX) or exceed the platform's smaller vector::max_size() (UBJSON/BJData's 0x7FFFFFFF), so the header is now rejected outright (out_of_range.408) instead of being accepted and only found short of data (parse_error.110). Both are safe, bounded rejections of the hostile input; the property under test -- no attempt to allocate space for billions of elements -- holds either way. Accept both outcomes instead of pinning the 64-bit-only exact result. Also fixed an unrelated clang-tidy finding (google-readability-casting) on the functional-style std::size_t(...) casts in the neighboring "arrays of various sizes" section. Signed-off-by: Niels Lohmann * Fix remaining CI failures in the huge-claimed-length DoS regression tests - Apply the same google-readability-casting fix (std::size_t{N} instead of std::size_t(N)) to the "arrays of various sizes" section in unit-msgpack.cpp, unit-ubjson.cpp, and unit-bjdata.cpp; only unit-cbor.cpp had been fixed previously, since clang-tidy's build didn't get far enough to report the other three in the same pass. - json_sax_dom_parser::start_array()'s max_size() check calls JSON_THROW directly rather than going through sax->parse_error(), so unlike the scanner's own "not enough data" parse_error it is not gated by allow_exceptions=false. On a platform where a header's claimed count exceeds max_size() (e.g. 32-bit, for UBJSON/BJData's 0x7FFFFFFF test value), from_ubjson/from_bjdata(input, true, false) can therefore still throw instead of returning a discarded value. Make that assertion tolerant of either outcome, same as the main exception-catching check above it. - Guard all four "a huge claimed length..." SECTIONs with #if !defined(JSON_NOEXCEPTION), matching this test suite's existing convention for exception-dependent tests: under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ exception at all (it aborts the process), so a section that relies on try/catch to distinguish between two acceptable outcomes cannot be expressed under that build configuration regardless of platform. Signed-off-by: Niels Lohmann * Use (std::min)(len, reserve_cap) instead of a ternary in start_array() Addresses review feedback from @gregmarr on PR #5476. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- include/nlohmann/detail/input/json_sax.hpp | 21 ++++ single_include/nlohmann/json.hpp | 21 ++++ tests/src/unit-bjdata.cpp | 105 ++++++++++++++++++++ tests/src/unit-cbor.cpp | 86 +++++++++++++++++ tests/src/unit-msgpack.cpp | 85 +++++++++++++++++ tests/src/unit-ubjson.cpp | 106 +++++++++++++++++++++ 6 files changed, 424 insertions(+) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 1627d2326..60c468f30 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -8,6 +8,7 @@ #pragma once +#include // min #include #include // string #include // enable_if_t @@ -305,6 +306,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } + return true; } @@ -683,6 +694,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } } return true; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 3b2233a57..6eb006b68 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -7892,6 +7892,7 @@ NLOHMANN_JSON_NAMESPACE_END +#include // min #include #include // string #include // enable_if_t @@ -11018,6 +11019,16 @@ class json_sax_dom_parser JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } + return true; } @@ -11396,6 +11407,16 @@ class json_sax_dom_callback_parser { JSON_THROW(out_of_range::create(408, concat("excessive array size: ", std::to_string(len)), ref_stack.back())); } + + if (len != detail::unknown_size()) + { + // reserve upfront to avoid repeated reallocations while adding elements, + // but cap the reservation so a bogus/hostile length (which is not bounded + // by max_size(), unlike e.g. std::vector) cannot trigger an oversized + // allocation for a small or truncated input + constexpr std::size_t reserve_cap = 16384; + ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + } } return true; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index 3130bb720..9338e663e 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -3551,6 +3551,111 @@ TEST_CASE("BJData") } } +TEST_CASE("issue #5405 - array reserve for definite-length BJData arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // little-endian 'l' (int32) of 0x7FFFFFFF (2147483647), but no + // element data at all. max_size() for a std::vector is far larger + // than this count, so it does not reject the header outright; the + // (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_bjdata(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_bjdata(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_bjdata(j); + CHECK(json::from_bjdata(packed_plain) == j); + + const auto packed_optimized = json::to_bjdata(j, true, true); + CHECK(json::from_bjdata(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_bjdata(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::bjdata)); + } +} + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") diff --git a/tests/src/unit-cbor.cpp b/tests/src/unit-cbor.cpp index 032e6641b..4c9107517 100644 --- a/tests/src/unit-cbor.cpp +++ b/tests/src/unit-cbor.cpp @@ -2174,6 +2174,92 @@ TEST_CASE("CBOR indefinite-length strings do not recurse per chunk") } } +TEST_CASE("issue #5405 - array reserve for definite-length CBOR arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0x9A: array with a four-byte length; claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's detail::unknown_size() sentinel (SIZE_MAX), so the + // format-level size check rejects it outright (out_of_range.408, + // "excessive ... size") before the SAX consumer's own max_size() + // check would even run; on a 64-bit platform it passes both of + // those checks and is only found short of data once the (capped) + // reservation looks for element bytes that were never provided + // (parse_error.110). Either is an acceptable, bounded rejection of + // the hostile header -- the property under test is that no path + // attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_cbor(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing CBOR value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_cbor(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_cbor(j); + CHECK(json::from_cbor(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_cbor(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::cbor)); + } +} + TEST_CASE("CBOR roundtrips" * doctest::skip()) { SECTION("input from flynn") diff --git a/tests/src/unit-msgpack.cpp b/tests/src/unit-msgpack.cpp index 75c2ae464..74f7f4969 100644 --- a/tests/src/unit-msgpack.cpp +++ b/tests/src/unit-msgpack.cpp @@ -1597,6 +1597,91 @@ TEST_CASE("MessagePack") } } +TEST_CASE("issue #5405 - array reserve for definite-length MessagePack arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // 0xdd: array 32 (four-byte length); claims 0xFFFFFFFF (4294967295) + // elements but provides none. max_size() for a std::vector is far + // larger than this count, so it does not reject the header outright; + // the (capped) reservation must not attempt to allocate space for + // billions of elements before the missing data is detected. + json _; + const std::vector input = {0xdd, 0xFF, 0xFF, 0xFF, 0xFF}; + // On a platform where std::size_t is narrower than 64 bits (e.g. + // 32-bit), the claimed count 0xFFFFFFFF coincides with that + // platform's SIZE_MAX, which some size-narrowing checks treat the + // same as detail::unknown_size(); it may then be rejected before + // the SAX consumer's own max_size() check (out_of_range.408) rather + // than being accepted and only found short of data once the + // (capped) reservation looks for element bytes that were never + // provided (parse_error.110). Either is an acceptable, bounded + // rejection of the hostile header -- the property under test is + // that no path attempts to allocate space for billions of elements. + bool threw = false; + try + { + _ = json::from_msgpack(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing MessagePack value: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive") != std::string::npos); + } + CHECK(threw); + CHECK(json::from_msgpack(input, true, false).is_discarded()); + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + const auto packed = json::to_msgpack(j); + CHECK(json::from_msgpack(packed) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_msgpack(j); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::msgpack)); + } +} + // use this testcase outside [hide] to run it with Valgrind TEST_CASE("MessagePack nesting does not consume the call stack") { diff --git a/tests/src/unit-ubjson.cpp b/tests/src/unit-ubjson.cpp index 2ebb55b90..aafbbf5a4 100644 --- a/tests/src/unit-ubjson.cpp +++ b/tests/src/unit-ubjson.cpp @@ -2315,6 +2315,112 @@ TEST_CASE("UBJSON optimized arrays of a valueless type are bounded") } } +TEST_CASE("issue #5405 - array reserve for definite-length UBJSON arrays") +{ +#if !defined(JSON_NOEXCEPTION) + // this SECTION relies on catching a thrown exception to distinguish + // which of two acceptable, bounded rejections a hostile header took; + // under JSON_NOEXCEPTION, JSON_THROW never produces a catchable C++ + // exception (it aborts instead), so this cannot be tested that way here + SECTION("a huge claimed length with no element data must not over-allocate") + { + // optimized form [$type#count: type 'i' (int8), count as a four-byte + // 'l' (int32) of 0x7FFFFFFF (2147483647), but no element data at all. + // max_size() for a std::vector is far larger than this count, so it + // does not reject the header outright; the (capped) reservation must + // not attempt to allocate space for billions of elements before the + // missing data is detected. + json _; + const std::vector input = {'[', '$', 'i', '#', 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + // On a platform where std::vector::max_size() is smaller than + // the claimed count (e.g. 32-bit, where max_size() is bounded by a + // 32-bit SIZE_MAX divided by sizeof(json)), the SAX consumer's own + // check rejects the header outright (out_of_range.408, with the + // claimed count in the message) instead of accepting it and only + // finding it short of data once the (capped) reservation looks for + // element bytes that were never provided (parse_error.110). Either + // is an acceptable, bounded rejection of the hostile header -- the + // property under test is that no path attempts to allocate space + // for billions of elements. + bool threw = false; + try + { + _ = json::from_ubjson(input); + } + catch (const json::parse_error& e) + { + threw = true; + CHECK(e.id == 110); + CHECK(std::string(e.what()) == "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing UBJSON number: unexpected end of input"); + } + catch (const json::out_of_range& e) + { + threw = true; + CHECK(e.id == 408); + CHECK(std::string(e.what()).find("excessive array size") != std::string::npos); + } + CHECK(threw); + + // json_sax_dom_parser::start_array()'s max_size() check (unlike the + // scanner's own parse_error path) throws unconditionally via + // JSON_THROW rather than going through sax->parse_error(), so it is + // not gated by allow_exceptions=false on a platform where this + // header hits that check (e.g. 32-bit, see above) -- allow either + // a discarded result or the same out_of_range it throws with + // exceptions enabled. + try + { + CHECK(json::from_ubjson(input, true, false).is_discarded()); + } + catch (const json::out_of_range& e) + { + CHECK(e.id == 408); + } + } +#endif + + SECTION("arrays of various sizes decode to the same value as before the reserve optimization") + { + for (const auto size : + { + std::size_t{0}, std::size_t{1}, std::size_t{5}, // small + std::size_t{16384}, // exactly at the reserve cap + std::size_t{20000} // above the reserve cap + }) + { + CAPTURE(size) + json j = json::array(); + for (std::size_t i = 0; i < size; ++i) + { + j.push_back(static_cast(i % 1000)); + } + + // exercise both the plain and the optimized [$type#count encoding + const auto packed_plain = json::to_ubjson(j); + CHECK(json::from_ubjson(packed_plain) == j); + + const auto packed_optimized = json::to_ubjson(j, true, true); + CHECK(json::from_ubjson(packed_optimized) == j); + } + } + + SECTION("a user-defined SAX consumer is unaffected by the internal DOM reserve optimization") + { + // the reserve() call is local to json_sax_dom_parser / json_sax_dom_callback_parser; + // a custom SAX consumer that does not touch a DOM array sees identical events + json j = json::array(); + for (int i = 0; i < 100; ++i) + { + j.push_back(i); + } + const auto packed = json::to_ubjson(j, true, true); + + SaxCountdown scp(1000000); // large enough to never trigger an abort + CHECK(json::sax_parse(packed, &scp, json::input_format_t::ubjson)); + } +} + + TEST_CASE("Universal Binary JSON Specification Examples 1") { SECTION("Null Value") From c0b2878a44b485aa02ed3174e1222e815479f511 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 13 Sep 2026 17:43:59 +0200 Subject: [PATCH 4/6] Swap diagnostic positions in basic_json::swap() (#5493) basic_json::swap() (and the friend swap() that forwards to it) only exchanged m_data.m_type/m_data.m_value, leaving start_position/end_position untouched under JSON_DIAGNOSTIC_POSITIONS. This is inconsistent with copy-assignment's operator=(basic_json), which swaps positions as part of its copy-and-swap implementation, so after swap(a, b) each value ended up with the other value's content but its own original position. Signed-off-by: Niels Lohmann --- docs/mkdocs/docs/api/basic_json/swap.md | 8 ++- include/nlohmann/json.hpp | 5 ++ single_include/nlohmann/json.hpp | 5 ++ tests/src/unit-class_parser.cpp | 80 +++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs/docs/api/basic_json/swap.md b/docs/mkdocs/docs/api/basic_json/swap.md index 3a3d288fb..aa5aa6c4c 100644 --- a/docs/mkdocs/docs/api/basic_json/swap.md +++ b/docs/mkdocs/docs/api/basic_json/swap.md @@ -34,10 +34,14 @@ void swap(typename binary_t::container_type& other); ``` 1. Exchanges the contents of the JSON value with those of `other`. Does not invoke any move, copy, or swap operations on - individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. + individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 2. Exchanges the contents of the JSON value from `left` with those of `right`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is - invalidated. Implemented as a friend function callable via ADL. + invalidated. Implemented as a friend function callable via ADL. If macro + [`JSON_DIAGNOSTIC_POSITIONS`](../macros/json_diagnostic_positions.md) is defined to `#!cpp 1`, the + [`start_pos()`](start_pos.md)/[`end_pos()`](end_pos.md) diagnostic positions are exchanged along with the value. 3. Exchanges the contents of a JSON array with those of `other`. Does not invoke any move, copy, or swap operations on individual elements. All iterators and references remain valid. The past-the-end iterator is invalidated. 4. Exchanges the contents of a JSON object with those of `other`. Does not invoke any move, copy, or swap operations on diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 016a4dd27..741c08b53 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3576,6 +3576,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 6eb006b68..85ebca27b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -27330,6 +27330,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec std::swap(m_data.m_type, other.m_data.m_type); std::swap(m_data.m_value, other.m_data.m_value); +#if JSON_DIAGNOSTIC_POSITIONS + std::swap(start_position, other.start_position); + std::swap(end_position, other.end_position); +#endif + set_parents(); other.set_parents(); assert_invariant(); diff --git a/tests/src/unit-class_parser.cpp b/tests/src/unit-class_parser.cpp index 34ca5db2b..af76e93cc 100644 --- a/tests/src/unit-class_parser.cpp +++ b/tests/src/unit-class_parser.cpp @@ -2259,6 +2259,86 @@ TEST_CASE("parser class") #endif } +#if JSON_DIAGNOSTIC_POSITIONS + +TEST_CASE("diagnostic positions: value lifetime") +{ + SECTION("copy constructor copies positions, recursively") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + const json a = json::parse(s); + const json b = a; // NOLINT(performance-unnecessary-copy-initialization) + + CHECK(b.start_pos() == a.start_pos()); + CHECK(b.end_pos() == a.end_pos()); + CHECK(b["b"].start_pos() == a["b"].start_pos()); + CHECK(b["b"].end_pos() == a["b"].end_pos()); + } + + SECTION("move constructor resets the moved-from value to npos") + { + const std::string s = R"({"a":1,"b":[1,2,3]})"; + json a = json::parse(s); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + + const json b(std::move(a)); + + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + CHECK(a.start_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + CHECK(a.end_pos() == std::string::npos); // NOLINT(bugprone-use-after-move,clang-analyzer-cplusplus.Move) + } + + SECTION("swap() exchanges positions along with the values") + { + // basic_json::swap() (and the friend swap() that forwards to it) used + // to swap only m_data.m_type/m_data.m_value, leaving + // start_position/end_position untouched -- unlike copy-assignment's + // operator=(basic_json), which swaps positions as part of its + // copy-and-swap implementation. After swap(a, b), each value ended up + // with the *other* value's content but its *own* original position. + // This is now fixed so that swap() is consistent with copy-assignment. + json a = json::parse(R"({"a":1})"); + json b = json::parse(R"([1,2,3,4,5])"); + const auto a_start = a.start_pos(); + const auto a_end = a.end_pos(); + const auto b_start = b.start_pos(); + const auto b_end = b.end_pos(); + // lengths (and thus end positions) differ, which is enough to tell + // after the swap whether positions actually moved with the values + CHECK(a_end != b_end); + + using std::swap; + swap(a, b); + + CHECK(a == json::parse(R"([1,2,3,4,5])")); + CHECK(b == json::parse(R"({"a":1})")); + + CHECK(a.start_pos() == b_start); + CHECK(a.end_pos() == b_end); + CHECK(b.start_pos() == a_start); + CHECK(b.end_pos() == a_end); + + // member swap() behaves the same as the free function + json c = json::parse(R"({"a":1})"); + json d = json::parse(R"([1,2,3,4,5])"); + const auto c_start = c.start_pos(); + const auto c_end = c.end_pos(); + const auto d_start = d.start_pos(); + const auto d_end = d.end_pos(); + + c.swap(d); + + CHECK(c.start_pos() == d_start); + CHECK(c.end_pos() == d_end); + CHECK(d.start_pos() == c_start); + CHECK(d.end_pos() == c_end); + } +} +#endif + // this test relies on parse errors being thrown, so it is skipped when // exceptions are disabled (json::parse aborts instead of throwing there) #if !defined(JSON_NOEXCEPTION) From 1da2f68992a4b42d653f8c2f7c70580c1e65a98a Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 14 Sep 2026 06:44:01 +0200 Subject: [PATCH 5/6] Only reserve array capacity if the array type supports it (#5522) --- include/nlohmann/detail/input/json_sax.hpp | 38 ++++++++++++------ single_include/nlohmann/json.hpp | 39 ++++++++++++------ tests/src/unit-regression3.cpp | 46 ++++++++++++++++++++++ 3 files changed, 99 insertions(+), 24 deletions(-) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 60c468f30..37d0ab270 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -151,6 +152,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -308,12 +332,7 @@ class json_sax_dom_parser if (len != detail::unknown_size()) { - // reserve upfront to avoid repeated reallocations while adding elements, - // but cap the reservation so a bogus/hostile length (which is not bounded - // by max_size(), unlike e.g. std::vector) cannot trigger an oversized - // allocation for a small or truncated input - constexpr std::size_t reserve_cap = 16384; - ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); } return true; @@ -697,12 +716,7 @@ class json_sax_dom_callback_parser if (len != detail::unknown_size()) { - // reserve upfront to avoid repeated reallocations while adding elements, - // but cap the reservation so a bogus/hostile length (which is not bounded - // by max_size(), unlike e.g. std::vector) cannot trigger an oversized - // allocation for a small or truncated input - constexpr std::size_t reserve_cap = 16384; - ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); } } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 85ebca27b..2bb7a5b7b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -10730,6 +10730,8 @@ NLOHMANN_JSON_NAMESPACE_END // #include +// #include + // #include NLOHMANN_JSON_NAMESPACE_BEGIN @@ -10864,6 +10866,29 @@ constexpr std::size_t unknown_size() return (std::numeric_limits::max)(); } +/*! +@brief reserve capacity for @a len elements in array @a arr + +Reserving upfront avoids repeated reallocations while the elements are added, +but the reservation is capped so a bogus/hostile length (which is not bounded +by max_size(), unlike e.g. std::vector) cannot trigger an oversized allocation +for a small or truncated input. + +The overload below is selected for array types without reserve() (e.g., +std::deque), which are then left untouched. +*/ +template +auto reserve_array(ArrayType& arr, std::size_t len, priority_tag<1> /*unused*/) +-> decltype(arr.reserve(len), void()) +{ + constexpr std::size_t reserve_cap = 16384; + arr.reserve((std::min)(len, reserve_cap)); +} + +template +inline void reserve_array(ArrayType& /*arr*/, std::size_t /*len*/, priority_tag<0> /*unused*/) +{} + /*! @brief SAX implementation to create a JSON value from SAX events @@ -11021,12 +11046,7 @@ class json_sax_dom_parser if (len != detail::unknown_size()) { - // reserve upfront to avoid repeated reallocations while adding elements, - // but cap the reservation so a bogus/hostile length (which is not bounded - // by max_size(), unlike e.g. std::vector) cannot trigger an oversized - // allocation for a small or truncated input - constexpr std::size_t reserve_cap = 16384; - ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); } return true; @@ -11410,12 +11430,7 @@ class json_sax_dom_callback_parser if (len != detail::unknown_size()) { - // reserve upfront to avoid repeated reallocations while adding elements, - // but cap the reservation so a bogus/hostile length (which is not bounded - // by max_size(), unlike e.g. std::vector) cannot trigger an oversized - // allocation for a small or truncated input - constexpr std::size_t reserve_cap = 16384; - ref_stack.back()->m_data.m_value.array->reserve((std::min)(len, reserve_cap)); + reserve_array(*ref_stack.back()->m_data.m_value.array, len, priority_tag<1> {}); } } diff --git a/tests/src/unit-regression3.cpp b/tests/src/unit-regression3.cpp index cb2ed59a6..11c6a7da8 100644 --- a/tests/src/unit-regression3.cpp +++ b/tests/src/unit-regression3.cpp @@ -27,6 +27,7 @@ using ordered_json = nlohmann::ordered_json; #endif #include +#include #include #include #include @@ -896,4 +897,49 @@ TEST_CASE("issue #5402 - update(merge_objects=true) overwrites a primitive with } +TEST_CASE("regression test #5476 - array type without reserve()") +{ + // the capacity reserved for definite-length arrays must not require the + // array type to have a reserve() member function + using deque_json = nlohmann::basic_json; + + SECTION("std::deque") + { + const auto j = deque_json::parse(R"({"a":[1,[2,3]],"b":[]})"); + CHECK(j.dump() == R"({"a":[1,[2,3]],"b":[]})"); + + // the binary formats pass a definite length to start_array() + CHECK(deque_json::from_cbor(deque_json::to_cbor(j)) == j); + CHECK(deque_json::from_msgpack(deque_json::to_msgpack(j)) == j); + + // parse() instantiates the callback parser as well, which reserves too + const auto with_callback = deque_json::parse(R"([1,2,3])", [](int /*depth*/, deque_json::parse_event_t /*event*/, deque_json& /*parsed*/) noexcept + { + return true; + }); + CHECK(with_callback == deque_json({1, 2, 3})); + } + + SECTION("std::vector still reserves") + { + json array = json::array(); + for (int i = 0; i < 100; ++i) + { + array.push_back(i); + } + + const auto j = json::from_cbor(json::to_cbor(array)); + CHECK(j == array); + CHECK(j.get_ref().capacity() >= 100); + } + + SECTION("the reservation stays capped") + { + // CBOR array announcing 2^32-1 elements, but truncated right after the + // header: the input must be rejected without reserving that capacity + const std::vector truncated = {0x9A, 0xFF, 0xFF, 0xFF, 0xFF}; + CHECK(json::from_cbor(truncated, true, false).is_discarded()); + } +} + DOCTEST_CLANG_SUPPRESS_WARNING_POP From 3bfe2b6da7393af5cfd68c44f58a1058e60499e2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 14 Sep 2026 21:19:41 +0200 Subject: [PATCH 6/6] Stop the privacy plugin from downloading the repology.org badges (#5523) repology.org refuses requests coming from GitHub Actions runners. The Material privacy plugin logs the failed download as a warning, but then raises FileNotFoundError when it reads back the cache entry it never wrote, so ci_test_build_documentation aborts the whole documentation build. develop and every open pull request are currently red because of this. The badges are fine for readers -- repology only rejects non-browser clients -- so exclude them from the plugin's asset self-hosting and keep them as external references. All other external assets (fonts, shields, CDN files) are still downloaded and self-hosted as before. Signed-off-by: Niels Lohmann --- docs/mkdocs/mkdocs.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/mkdocs/mkdocs.yml b/docs/mkdocs/mkdocs.yml index ec3e462c1..5c9f72fc3 100644 --- a/docs/mkdocs/mkdocs.yml +++ b/docs/mkdocs/mkdocs.yml @@ -394,7 +394,14 @@ plugins: - http://nlohmann.github.io/json/* - https://nlohmann.github.io/json/* - mailto:* - - privacy + - privacy: + # repology.org refuses requests from GitHub Actions runners, which made + # the privacy plugin abort the whole build when it could not download the + # package badges (the fetch fails, then reading the missing cache entry + # raises FileNotFoundError). Readers' browsers are served normally, so + # leave these badges as external references instead of self-hosting them. + assets_exclude: + - repology.org/* - llmstxt: markdown_description: > JSON for Modern C++ is a C++11 header-only library implementing a JSON