From 9a091d2b8290a7e5f631dfb9b34c35b7ab777d22 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 5 Aug 2026 13:44:15 +0200 Subject: [PATCH] Do not write BJData ndarrays whose size overflows std::size_t (#5362) * Do not write BJData ndarrays whose size overflows std::size_t write_bjdata_ndarray() multiplied the _ArraySize_ dimensions into a std::size_t without checking for overflow. A product that wraps around to a value that happens to match the size of _ArrayData_ passed the length check, and the writer emitted an ndarray header announcing an element count that cannot be represented: {"_ArrayType_":"uint8","_ArraySize_":[9223372036854775808,2],"_ArrayData_":[]} was encoded as 5b 24 55 23 5b 4d 00 00 00 00 00 00 00 80 69 02 5d, an ndarray of 2^64 elements followed by no data. Reading that back throws out_of_range.408 ("excessive ndarray size caused overflow"), so to_bjdata produced output that from_bjdata rejects. This is reachable by parsing untrusted JSON and re-encoding it as BJData. Mirror the overflow check the binary reader already performs, and also reject a single dimension that does not fit into std::size_t, which the previous cast silently truncated where std::size_t is narrower than 64 bits. Such objects now fall back to a plain object encoding, which is what the surrounding type and length validation already does for annotations it cannot represent, and they round-trip unchanged. Signed-off-by: Niels Lohmann * Document when to_bjdata converts a JData annotation to an ND-array The BJData page described the 1-D vector case as the only situation in which an object carrying _ArrayType_/_ArraySize_/_ArrayData_ is not written as a compact ND-array. The writer has always had several other fallbacks -- an unknown _ArrayType_, a dimension that is not a non-negative integer, an _ArrayData_ whose length does not match the product of the dimensions, and elements that are not numbers of the annotated kind -- all of which cause the value to be serialized as a regular JSON object instead. Spell out the conditions, including the size-overflow check added in the preceding commit, so the documented behavior matches the implementation. Signed-off-by: Niels Lohmann --------- Signed-off-by: Niels Lohmann --- .../docs/features/binary_formats/bjdata.md | 16 +++++++++++--- .../nlohmann/detail/output/binary_writer.hpp | 18 +++++++++++++++- single_include/nlohmann/json.hpp | 18 +++++++++++++++- tests/src/unit-bjdata.cpp | 21 +++++++++++++++++++ 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/docs/mkdocs/docs/features/binary_formats/bjdata.md b/docs/mkdocs/docs/features/binary_formats/bjdata.md index b1b98dfe2..faff64f16 100644 --- a/docs/mkdocs/docs/features/binary_formats/bjdata.md +++ b/docs/mkdocs/docs/features/binary_formats/bjdata.md @@ -116,9 +116,19 @@ The library uses the following mapping from JSON values types to BJData types ac ``` Likewise, when a JSON object in the above form is serialized using - [`to_bjdata`](../../api/basic_json/to_bjdata.md), it is automatically converted into a compact BJData ND-array. The - only exception is, that when the 1-dimensional vector stored in `"_ArraySize_"` contains a single integer or two - integers with one being 1, a regular 1-D optimized array is generated. + [`to_bjdata`](../../api/basic_json/to_bjdata.md), it is automatically converted into a compact BJData ND-array. When + the 1-dimensional vector stored in `"_ArraySize_"` contains a single integer or two integers with one being 1, a + regular 1-D optimized array is generated instead. + + An object is only converted if the annotation actually describes a packed array; otherwise it is serialized as a + regular JSON object. This requires all of the following: + + - `"_ArrayType_"` is one of `uint8`, `int8`, `uint16`, `int16`, `uint32`, `int32`, `uint64`, `int64`, `single`, + `double`, `char`, or `byte`, + - every entry of `"_ArraySize_"` is a non-negative integer, and their product is representable as a `std::size_t`, + - `"_ArrayData_"` holds exactly that many elements, and + - every element of `"_ArrayData_"` is a number of the kind named by `"_ArrayType_"` (a floating-point number for + `single` and `double`, an integer otherwise). The current version of this library does not yet support automatic detection of and conversion from a nested JSON array input to a BJData ND-array. diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 9a8b53df7..35b5efa8a 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1670,7 +1670,23 @@ class binary_writer { return true; } - len *= static_cast(el.template get()); + + // a dimension that does not fit into std::size_t, or a product that + // overflows it, would wrap around and could match the size of + // _ArrayData_ by accident; the resulting header announces an + // element count that no reader can honor (the binary reader rejects + // it with out_of_range.408), so encode as a plain object instead + const auto dim = el.template get(); + if (!value_in_range_of(dim)) + { + return true; + } + const auto dim_size = static_cast(dim); + if (dim_size != 0 && len > (std::numeric_limits::max)() / dim_size) + { + return true; + } + len *= dim_size; } key = "_ArrayData_"; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a42a22d23..cca1b3666 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18577,7 +18577,23 @@ class binary_writer { return true; } - len *= static_cast(el.template get()); + + // a dimension that does not fit into std::size_t, or a product that + // overflows it, would wrap around and could match the size of + // _ArrayData_ by accident; the resulting header announces an + // element count that no reader can honor (the binary reader rejects + // it with out_of_range.408), so encode as a plain object instead + const auto dim = el.template get(); + if (!value_in_range_of(dim)) + { + return true; + } + const auto dim_size = static_cast(dim); + if (dim_size != 0 && len > (std::numeric_limits::max)() / dim_size) + { + return true; + } + len *= dim_size; } key = "_ArrayData_"; diff --git a/tests/src/unit-bjdata.cpp b/tests/src/unit-bjdata.cpp index d00e1fa67..d5f9c3acd 100644 --- a/tests/src/unit-bjdata.cpp +++ b/tests/src/unit-bjdata.cpp @@ -2730,6 +2730,27 @@ TEST_CASE("BJData") CHECK(json::from_bjdata(json::to_bjdata(j_type), true, true) == j_type); CHECK(json::from_bjdata(json::to_bjdata(j_size), true, true) == j_size); } + + SECTION("ndarray whose dimensions overflow stays as object") + { + // the product of the dimensions wraps around std::size_t to 0 + // and so matches the size of the empty _ArrayData_; writing this + // as an ndarray would announce an element count no reader can + // honor, so it has to stay a plain object + json j_overflow = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {9223372036854775808ull, 2}}, {"_ArrayType_", "uint8"}}); + CHECK(json::from_bjdata(json::to_bjdata(j_overflow), true, true) == j_overflow); + + // a single dimension that does not fit into std::size_t is + // rejected for the same reason (only observable where + // std::size_t is narrower than 64 bit) + json j_huge = json({{"_ArrayData_", json::array()}, {"_ArraySize_", {18446744073709551615ull}}, {"_ArrayType_", "uint8"}}); + CHECK(json::from_bjdata(json::to_bjdata(j_huge), true, true) == j_huge); + + // a well-formed ndarray is still encoded as one + json j_ok = json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint8"}}); + CHECK(json::to_bjdata(j_ok) == std::vector({'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 1, 2, 3, 4, 5, 6})); + CHECK(json::from_bjdata(json::to_bjdata(j_ok), true, true) == j_ok); + } } }