Compare commits

..
Author SHA1 Message Date
Niels Lohmann ab8aa8458b Fix 32-bit overflow in huge_string_t BSON length-overflow tests
huge_string_t doubles as basic_json's StringType, so it is used not only
for the JSON string value under test but also for object keys (e.g. "s",
"nested"). Making size() unconditionally lie about being huge therefore
inflated the keys' reported sizes as well, pushing the running totals
computed while walking the BSON document (calc_bson_object_size and
friends in binary_writer.hpp) past what a 32-bit std::size_t can hold.

On 64-bit platforms this happens to still produce a working (if
needlessly large) result, but on 32-bit platforms (e.g. the mingw x86 CI
job) the size_t arithmetic silently wraps around: for the "document" test
this merely surfaces the wrong number in the exception message, but for
the "string" test the wrapped total happens to fall back under
INT32_MAX, so the intended out_of_range.412 guard is skipped entirely and
the code goes on to actually write ~2 GiB worth of characters from the
key's real, tiny buffer - which is what raised the reported
"vector::_M_range_insert" exception instead of a controlled 412.

Make the fake-huge size opt-in via huge_string_t::as_huge() and only
apply it to the string value under test, leaving keys at their real
(small) size. This keeps every intermediate size well within 32-bit
size_t range on any platform, matching how huge_binary_t already avoids
the same trap (it is only ever used as the BSON value type, never as a
key). Expected out_of_range.412 messages are updated accordingly.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-06 11:32:21 +02:00
Niels Lohmann a33e015e39 Add test coverage for documented lenient BSON input handling
Issue #5333 documented three intentionally-lenient behaviors of the BSON
reader (any non-zero byte accepted as a boolean `true`, BSON array element
keys not validated against the required decimal sequence, and the payload
of binary subtype 0x02 "old binary" returned as-is including its inner
length prefix), but none of them was pinned by a test, so a future change
could silently regress the documented behavior.

Also add coverage for the out_of_range.412 length-overflow check
(shared by binary, string, and (sub-)document BSON length fields) for
the string and document cases; only the binary case was previously
tested.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-09-05 20:52:05 +02:00
3 changed files with 142 additions and 505 deletions
+139
View File
@@ -38,6 +38,54 @@ class huge_binary_t : public std::vector<std::uint8_t>
using huge_binary_json = nlohmann::basic_json < using huge_binary_json = nlohmann::basic_json <
std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t, std::map, std::vector, std::string, bool, std::int64_t, std::uint64_t,
double, std::allocator, nlohmann::adl_serializer, huge_binary_t, void >; double, std::allocator, nlohmann::adl_serializer, huge_binary_t, void >;
// a string type that can be made to report a size beyond INT32_MAX without
// allocating that much memory, so BSON length overflow can be tested for
// strings and (embedded) documents as well, following the same idea as
// huge_binary_t.
//
// Unlike huge_binary_t (which is only ever used as the BSON *value* type),
// this type doubles as basic_json's StringType and is therefore also used
// for *object keys* (e.g. "s" or "nested" below). Only the designated test
// value is meant to lie about its size - if every huge_string_t (including
// keys) reported a huge size, the running totals computed while walking the
// BSON document (see calc_bson_object_size & friends in binary_writer.hpp)
// would need more than 32 bits, and on platforms where std::size_t is only
// 32 bits wide that arithmetic would silently wrap around, producing wrong
// (or even unguarded) lengths. The fake size is therefore opt-in via
// as_huge(), and plain strings - in particular object keys - keep reporting
// their real, small size.
class huge_string_t : public std::string
{
public:
using std::string::string;
huge_string_t(const std::string& s) : std::string(s) {} // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
// returns a copy of @a s whose size() pretends to be huge
static huge_string_t as_huge(const std::string& s)
{
huge_string_t result(s);
result.pretend_huge = true;
return result;
}
size_type size() const noexcept
{
if (pretend_huge)
{
// one byte more than the BSON length field can represent
return static_cast<size_type>((std::numeric_limits<std::int32_t>::max)()) + 1;
}
return std::string::size();
}
private:
bool pretend_huge = false;
};
using huge_string_json = nlohmann::basic_json <
std::map, std::vector, huge_string_t, bool, std::int64_t, std::uint64_t,
double, std::allocator, nlohmann::adl_serializer, std::vector<std::uint8_t>, void >;
} // namespace } // namespace
TEST_CASE("BSON") TEST_CASE("BSON")
@@ -104,6 +152,11 @@ TEST_CASE("BSON")
} }
SECTION("lengths exceeding INT32_MAX cannot be serialized to BSON") SECTION("lengths exceeding INT32_MAX cannot be serialized to BSON")
{
// out_of_range.412 is thrown from a single shared helper
// (to_bson_length) that guards the BSON length fields of binary
// values, strings, and (embedded) documents alike
SECTION("binary")
{ {
huge_binary_json j; huge_binary_json j;
j["b"] = huge_binary_json::binary(huge_binary_t{}); j["b"] = huge_binary_json::binary(huge_binary_t{});
@@ -111,6 +164,27 @@ TEST_CASE("BSON")
CHECK_THROWS_WITH_AS(huge_binary_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_binary_json::out_of_range&); CHECK_THROWS_WITH_AS(huge_binary_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_binary_json::out_of_range&);
} }
SECTION("string")
{
huge_string_json j;
j["s"] = huge_string_t::as_huge("value");
CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483661 exceeds maximum of 2147483647", huge_string_json::out_of_range&);
}
SECTION("document")
{
// an oversized string nested one level deep makes the
// *embedded* document's own length exceed INT32_MAX as well
huge_string_json nested;
nested["s"] = huge_string_t::as_huge("value");
huge_string_json j;
j["nested"] = nested;
CHECK_THROWS_WITH_AS(huge_string_json::to_bson(j), "[json.exception.out_of_range.412] BSON length 2147483674 exceeds maximum of 2147483647", huge_string_json::out_of_range&);
}
}
SECTION("string length must be at least 1") SECTION("string length must be at least 1")
{ {
// from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11175 // from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11175
@@ -193,6 +267,23 @@ TEST_CASE("BSON")
CHECK(json::from_bson(result, true, false) == j); CHECK(json::from_bson(result, true, false) == j);
} }
SECTION("non-empty object with bool from a non-0/1 byte (lenient parsing)")
{
// documented lenient behavior (see gh-5333): any non-zero byte
// is accepted as `true`, not just 0x01
std::vector<std::uint8_t> const input =
{
0x0D, 0x00, 0x00, 0x00, // size (little endian)
0x08, // entry: boolean
'e', 'n', 't', 'r', 'y', '\x00',
0x02, // value = 0x02 (neither 0x00 nor 0x01)
0x00 // end marker
};
const json expected = { { "entry", true } };
CHECK(json::from_bson(input) == expected);
}
SECTION("non-empty object with double") SECTION("non-empty object with double")
{ {
json const j = json const j =
@@ -499,6 +590,29 @@ TEST_CASE("BSON")
CHECK(json::from_bson(result, true, false) == j); CHECK(json::from_bson(result, true, false) == j);
} }
SECTION("array elements with non-conforming keys (lenient parsing)")
{
// documented lenient behavior (see gh-5333): BSON array element
// keys are not checked against the required decimal sequence
// "0", "1", "2", ... - elements are taken in encoded order
std::vector<std::uint8_t> const input =
{
0x26, 0x00, 0x00, 0x00, // size (little endian)
0x04, 'e', 'n', 't', 'r', 'y', '\x00', // entry: embedded array
0x1A, 0x00, 0x00, 0x00, // size (little endian)
0x10, '5', 0x00, 0x0A, 0x00, 0x00, 0x00, // key "5" (bogus) -> 10
0x10, 'x', 0x00, 0x14, 0x00, 0x00, 0x00, // key "x" (non-numeric) -> 20
0x10, '1', 0x00, 0x1E, 0x00, 0x00, 0x00, // key "1" (out of order) -> 30
0x00, // end marker (embedded array)
0x00 // end marker
};
const json expected = { { "entry", json::array({10, 20, 30}) } };
CHECK(json::from_bson(input) == expected);
}
SECTION("non-empty object with binary member") SECTION("non-empty object with binary member")
{ {
const size_t N = 10; const size_t N = 10;
@@ -594,6 +708,31 @@ TEST_CASE("BSON")
CHECK(json::from_bson(result, true, false) == j); CHECK(json::from_bson(result, true, false) == j);
} }
SECTION("binary member with subtype 0x02 (old binary) keeps its inner length prefix (lenient parsing)")
{
// documented lenient behavior (see gh-5333): the payload for
// binary subtype 0x02 ("old binary") is returned as-is,
// including its own inner 4-byte length prefix; it is not
// stripped or reinterpreted
std::vector<std::uint8_t> const input =
{
0x17, 0x00, 0x00, 0x00, // size (little endian)
0x05, 'e', 'n', 't', 'r', 'y', '\x00', // entry: binary
0x06, 0x00, 0x00, 0x00, // size of binary (little endian)
0x02, // "old binary" subtype
0x02, 0x00, 0x00, 0x00, // inner length prefix (part of the old-binary payload)
0x68, 0x69, // payload ('h', 'i')
0x00 // end marker
};
// the inner length prefix is part of the (unmodified) payload
const std::vector<std::uint8_t> expected_payload = {0x02, 0x00, 0x00, 0x00, 0x68, 0x69};
const json expected = { { "entry", json::binary(expected_payload, 0x02) } };
CHECK(json::from_bson(input) == expected);
}
SECTION("Some more complex document") SECTION("Some more complex document")
{ {
json const j = json const j =
-489
View File
@@ -1,489 +0,0 @@
// __ _____ _____ _____
// __| | __| | | | JSON for Modern C++ (supporting code)
// | | |__ | | | | | | version 3.12.0
// |_____|_____|_____|_|___| https://github.com/nlohmann/json
//
// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann <https://nlohmann.me>
// SPDX-FileCopyrightText: 2018 Vitaliy Manushkin <agri@akamo.info>
// SPDX-License-Identifier: MIT
// This file closes a test-coverage gap described in GitHub issue #5421:
// nlohmann::ordered_json (and other non-default basic_json specializations,
// such as the alt_string-based one from unit-alt-string.cpp) were never
// exercised through the binary formats (CBOR/MessagePack/UBJSON/BSON/BJData)
// or through flatten()/unflatten()/diff()/patch()/merge_patch().
#include "doctest_compatibility.h"
#include <nlohmann/json.hpp>
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
using nlohmann::json;
using nlohmann::ordered_json;
/////////////////////////////////////////////////////////////////////////////
// alt_json: a second, independent copy of the custom-string_t basic_json
// specialization defined in unit-alt-string.cpp.
//
// It is duplicated here (rather than shared via a header) because every
// unit-*.cpp file in this test suite is compiled into its own standalone
// executable (see tests/CMakeLists.txt), so there is no ODR concern in
// having the same class name defined in multiple translation units.
//
// Two members had to be added relative to the original alt_string
// (a constructor from std::string, and a find(char, pos) overload) because
// the original type was never used with the binary writers/readers before
// this file: BSON's array/document writer converts std::to_string() results
// and checks for embedded NUL characters via find(char), and the UBJSON/BSON
// high-precision-number path constructs the SAX string_t argument from a
// std::string. Neither path is exercised anywhere else in the test suite for
// this type, which is presumably why the gap was never noticed.
/////////////////////////////////////////////////////////////////////////////
class alt_string;
bool operator<(const char* op1, const alt_string& op2) noexcept; // NOLINT(misc-use-internal-linkage)
void int_to_string(alt_string& target, std::size_t value); // NOLINT(misc-use-internal-linkage)
class alt_string
{
public:
using value_type = std::string::value_type;
static constexpr auto npos = (std::numeric_limits<std::size_t>::max)();
alt_string(const char* str): str_impl(str) {}
alt_string(const char* str, std::size_t count): str_impl(str, count) {}
alt_string(std::string str): str_impl(std::move(str)) {}
alt_string(size_t count, char chr): str_impl(count, chr) {}
alt_string() = default;
alt_string& append(char ch)
{
str_impl.push_back(ch);
return *this;
}
alt_string& append(const alt_string& str)
{
str_impl.append(str.str_impl);
return *this;
}
alt_string& append(const char* s, std::size_t length)
{
str_impl.append(s, length);
return *this;
}
void push_back(char c)
{
str_impl.push_back(c);
}
template <typename op_type>
bool operator==(const op_type& op) const
{
return str_impl == op;
}
bool operator==(const alt_string& op) const
{
return str_impl == op.str_impl;
}
template <typename op_type>
bool operator!=(const op_type& op) const
{
return str_impl != op;
}
bool operator!=(const alt_string& op) const
{
return str_impl != op.str_impl;
}
std::size_t size() const noexcept
{
return str_impl.size();
}
void resize(std::size_t n)
{
str_impl.resize(n);
}
void resize(std::size_t n, char c)
{
str_impl.resize(n, c);
}
template <typename op_type>
bool operator<(const op_type& op) const noexcept
{
return str_impl < op;
}
bool operator<(const alt_string& op) const noexcept
{
return str_impl < op.str_impl;
}
const char* c_str() const
{
return str_impl.c_str();
}
char& operator[](std::size_t index)
{
return str_impl[index];
}
const char& operator[](std::size_t index) const
{
return str_impl[index];
}
char& back()
{
return str_impl.back();
}
const char& back() const
{
return str_impl.back();
}
void clear()
{
str_impl.clear();
}
const value_type* data() const
{
return str_impl.data();
}
bool empty() const
{
return str_impl.empty();
}
std::size_t find(const alt_string& str, std::size_t pos = 0) const
{
return str_impl.find(str.str_impl, pos);
}
// needed by binary_writer's BSON support, which probes string keys for
// embedded NUL characters via find(char)
std::size_t find(char c, std::size_t pos = 0) const
{
return str_impl.find(c, pos);
}
std::size_t find_first_of(char c, std::size_t pos = 0) const
{
return str_impl.find_first_of(c, pos);
}
alt_string substr(std::size_t pos = 0, std::size_t count = npos) const
{
const std::string s = str_impl.substr(pos, count);
return {s.data(), s.size()};
}
alt_string& replace(std::size_t pos, std::size_t count, const alt_string& str)
{
str_impl.replace(pos, count, str.str_impl);
return *this;
}
void reserve(std::size_t new_cap = 0)
{
str_impl.reserve(new_cap);
}
private:
std::string str_impl {}; // NOLINT(readability-redundant-member-init)
friend bool operator<(const char* /*op1*/, const alt_string& /*op2*/) noexcept;
};
void int_to_string(alt_string& target, std::size_t value)
{
target = std::to_string(value).c_str();
}
using alt_json = nlohmann::basic_json <
std::map,
std::vector,
alt_string,
bool,
std::int64_t,
std::uint64_t,
double,
std::allocator,
nlohmann::adl_serializer >;
bool operator<(const char* op1, const alt_string& op2) noexcept
{
return op1 < op2.str_impl;
}
namespace
{
// collects the object keys of j, in iteration order
std::vector<std::string> collect_keys(const ordered_json& j)
{
std::vector<std::string> result;
for (auto it = j.cbegin(); it != j.cend(); ++it)
{
result.push_back(it.key());
}
return result;
}
// a nested object/array value with keys inserted in non-alphabetical order,
// used to check both round-trip equality and (for ordered_json) that
// insertion order survives a trip through a binary format
ordered_json make_rich_ordered_json()
{
ordered_json j;
j["zebra"] = 1;
j["apple"] = ordered_json::array({1, 2, 3});
j["mango"]["z_nested"] = true;
j["mango"]["a_nested"] = nullptr;
j["banana"] = "some text";
j["cherry"] = 3.14;
return j;
}
alt_json make_rich_alt_json()
{
alt_json j;
j["zebra"] = 1;
j["apple"] = alt_json::array({1, 2, 3});
j["mango"]["z_nested"] = true;
j["mango"]["a_nested"] = nullptr;
j["banana"] = "some text";
j["cherry"] = 3.14;
return j;
}
} // namespace
TEST_CASE("ordered_json across binary formats")
{
const ordered_json original = make_rich_ordered_json();
const std::vector<std::string> original_keys = collect_keys(original);
const std::vector<std::string> original_mango_keys = collect_keys(original["mango"]);
SECTION("CBOR")
{
const auto bytes = ordered_json::to_cbor(original);
const auto restored = ordered_json::from_cbor(bytes);
CHECK(restored == original);
CHECK(collect_keys(restored) == original_keys);
CHECK(collect_keys(restored["mango"]) == original_mango_keys);
}
SECTION("MessagePack")
{
const auto bytes = ordered_json::to_msgpack(original);
const auto restored = ordered_json::from_msgpack(bytes);
CHECK(restored == original);
CHECK(collect_keys(restored) == original_keys);
CHECK(collect_keys(restored["mango"]) == original_mango_keys);
}
SECTION("UBJSON")
{
const auto bytes = ordered_json::to_ubjson(original);
const auto restored = ordered_json::from_ubjson(bytes);
CHECK(restored == original);
CHECK(collect_keys(restored) == original_keys);
CHECK(collect_keys(restored["mango"]) == original_mango_keys);
}
SECTION("BSON")
{
const auto bytes = ordered_json::to_bson(original);
const auto restored = ordered_json::from_bson(bytes);
CHECK(restored == original);
CHECK(collect_keys(restored) == original_keys);
CHECK(collect_keys(restored["mango"]) == original_mango_keys);
}
SECTION("BJData")
{
const auto bytes = ordered_json::to_bjdata(original);
const auto restored = ordered_json::from_bjdata(bytes);
CHECK(restored == original);
CHECK(collect_keys(restored) == original_keys);
CHECK(collect_keys(restored["mango"]) == original_mango_keys);
}
}
TEST_CASE("alt_json (custom string_t) across binary formats")
{
const alt_json original = make_rich_alt_json();
SECTION("CBOR")
{
const auto bytes = alt_json::to_cbor(original);
const auto restored = alt_json::from_cbor(bytes);
CHECK(restored == original);
}
SECTION("MessagePack")
{
const auto bytes = alt_json::to_msgpack(original);
const auto restored = alt_json::from_msgpack(bytes);
CHECK(restored == original);
}
SECTION("UBJSON")
{
const auto bytes = alt_json::to_ubjson(original);
const auto restored = alt_json::from_ubjson(bytes);
CHECK(restored == original);
}
SECTION("BSON")
{
const auto bytes = alt_json::to_bson(original);
const auto restored = alt_json::from_bson(bytes);
CHECK(restored == original);
}
SECTION("BJData")
{
const auto bytes = alt_json::to_bjdata(original);
const auto restored = alt_json::from_bjdata(bytes);
CHECK(restored == original);
}
}
TEST_CASE("ordered_json operator== is sensitive to key order")
{
// Unlike nlohmann::json (whose object_t is a std::map, so equality never
// depends on insertion order), ordered_json's object_t (ordered_map) is a
// std::vector<std::pair<Key, T>> under the hood, and does not define its
// own operator==: it inherits std::vector's element-wise comparison. As a
// result, two ordered_json objects holding the very same key/value pairs
// in different insertion order compare *unequal*. This is the property
// that makes the round-trip `CHECK(restored == original)` checks above a
// meaningful order-preservation check by themselves (the explicit
// collect_keys() comparisons make that check explicit/readable, and
// guard against this operator== behavior ever changing).
ordered_json a;
a["x"] = 1;
a["y"] = 2;
ordered_json b;
b["y"] = 2;
b["x"] = 1;
CHECK(a.size() == b.size());
CHECK(a["x"] == b["x"]);
CHECK(a["y"] == b["y"]);
CHECK_FALSE(a == b);
}
TEST_CASE("duplicate keys in a binary-encoded object")
{
// CBOR encoding of a map with two entries under the same key "a": {"a": 1, "a": 2}
const std::vector<std::uint8_t> cbor_bytes
{
0xA2, 0x61, 'a', 0x01, 0x61, 'a', 0x02
};
// Both json (std::map, via operator[]) and ordered_json (ordered_map, via
// operator[]) build binary-decoded objects by looking up/creating the
// entry for each incoming key and then assigning the value into it. This
// means a repeated key does *not* produce two entries in either case;
// instead, the *first* occurrence's position is kept (relevant only for
// ordered_json) while the *last* occurrence's value wins (for both) --
// this matches operator[]'s "assign the referenced slot" semantics, and
// is worth noting because it differs from the initializer-list
// construction path (`ordered_json{{"a",1},{"a",2}}`), which builds
// through insert()/emplace() and therefore keeps the *first* value, not
// the last (see the "There are no dup keys..." case in
// unit-ordered_json.cpp).
const auto j = json::from_cbor(cbor_bytes);
const auto oj = ordered_json::from_cbor(cbor_bytes);
CHECK(j.size() == 1);
CHECK(oj.size() == 1);
CHECK(j["a"] == 2);
CHECK(oj["a"] == 2);
CHECK(j == json(oj));
}
TEST_CASE("ordered_json through flatten/unflatten")
{
const ordered_json original = make_rich_ordered_json();
const std::vector<std::string> original_keys = collect_keys(original);
const std::vector<std::string> original_mango_keys = collect_keys(original["mango"]);
const ordered_json flat = original.flatten();
const ordered_json unflattened = flat.unflatten();
CHECK(unflattened == original);
// flatten() walks the value depth-first in iteration order and
// unflatten() re-inserts each flattened key via operator[] in the flat
// object's iteration order, so for ordered_json the original key order
// (both top-level and nested) is preserved end-to-end.
CHECK(collect_keys(unflattened) == original_keys);
CHECK(collect_keys(unflattened["mango"]) == original_mango_keys);
}
TEST_CASE("ordered_json through diff/patch/patch_inplace")
{
ordered_json original;
original["one"] = 1;
original["two"] = 2;
original["three"] = 3;
ordered_json target = original;
target["one"] = 100; // replace
target.erase("two"); // remove
target["four"] = 4; // add
const ordered_json patch = ordered_json::diff(original, target);
SECTION("patch")
{
const ordered_json patched = original.patch(patch);
CHECK(patched == target);
}
SECTION("patch_inplace")
{
ordered_json copy = original;
copy.patch_inplace(patch);
CHECK(copy == target);
}
}
TEST_CASE("ordered_json through merge_patch")
{
ordered_json original;
original["a"] = 1;
original["b"] = 2;
const ordered_json patch = {{"b", nullptr}, {"c", 3}};
original.merge_patch(patch);
ordered_json expected;
expected["a"] = 1;
expected["c"] = 3;
CHECK(original == expected);
CHECK(collect_keys(original) == collect_keys(expected));
}
-13
View File
@@ -17,7 +17,6 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
using json = nlohmann::json; using json = nlohmann::json;
using ordered_json = nlohmann::ordered_json;
// JSON_HAS_CPP_20 (do not remove; see note at top of file) // JSON_HAS_CPP_20 (do not remove; see note at top of file)
#if JSON_HAS_STD_FORMAT #if JSON_HAS_STD_FORMAT
@@ -94,16 +93,4 @@ TEST_CASE("std::formatter<nlohmann::json>")
} }
} }
TEST_CASE("std::formatter<nlohmann::ordered_json>")
{
// spot-check a non-default basic_json instantiation, since the formatter
// is written against the generic NLOHMANN_BASIC_JSON_TPL_DECLARATION
// template and must actually instantiate (and behave correctly) for
// template arguments other than nlohmann::json
const ordered_json j = {{"foo", 1}, {"bar", {1, 2, 3}}};
CHECK(std::format("{}", j) == j.dump());
CHECK(std::format("{:#}", j) == j.dump(4));
CHECK(std::format("{:2}", j) == j.dump(2));
}
#endif #endif