Merge remote-tracking branch 'origin/develop' into claude/iterative-diff

This commit is contained in:
Niels Lohmann
2026-09-25 22:19:57 +02:00
52 changed files with 3988 additions and 421 deletions
+4
View File
@@ -40,6 +40,10 @@ TEST_CASE("default namespace")
expected += "_psp";
#endif
#if JSON_STRICT_NUL_HANDLING
expected += "_snul";
#endif
expected += "_v" STRINGIZE(NLOHMANN_JSON_VERSION_MAJOR);
expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_MINOR);
expected += "_" STRINGIZE(NLOHMANN_JSON_VERSION_PATCH) "::basic_json";
+4
View File
@@ -41,6 +41,10 @@ TEST_CASE("default namespace without version component")
expected += "_psp";
#endif
#if JSON_STRICT_NUL_HANDLING
expected += "_snul";
#endif
expected += "::basic_json";
// fallback for Clang
+21 -15
View File
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.11...3.14)
cmake_minimum_required(VERSION 3.14)
project(JSON_Benchmarks LANGUAGES CXX)
# set compiler flags
@@ -6,29 +6,35 @@ if((CMAKE_CXX_COMPILER_ID MATCHES GNU) OR (CMAKE_CXX_COMPILER_ID MATCHES Clang))
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto -DNDEBUG -O3")
endif()
# configure Google Benchmarks
# configure Google Benchmark; a fixed release, so that results stay comparable
set(JSON_GOOGLE_BENCHMARK_VERSION 1.9.5)
include(FetchContent)
FetchContent_Declare(
benchmark
GIT_REPOSITORY https://github.com/google/benchmark.git
GIT_TAG origin/main
GIT_SHALLOW TRUE
)
FetchContent_GetProperties(benchmark)
if(NOT benchmark_POPULATED)
FetchContent_Populate(benchmark)
set(BENCHMARK_ENABLE_TESTING OFF CACHE INTERNAL "" FORCE)
add_subdirectory(${benchmark_SOURCE_DIR} ${benchmark_BINARY_DIR})
endif()
# only the library is needed; -Werror would break the pinned release as soon as
# a newer compiler adds a warning
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE)
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE)
set(BENCHMARK_ENABLE_WERROR OFF CACHE BOOL "" FORCE)
FetchContent_Declare(benchmark
URL https://github.com/google/benchmark/archive/refs/tags/v${JSON_GOOGLE_BENCHMARK_VERSION}.tar.gz
URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340
DOWNLOAD_EXTRACT_TIMESTAMP TRUE
)
FetchContent_MakeAvailable(benchmark)
# download test data
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake ${CMAKE_MODULE_PATH})
include(download_test_data)
# the header to benchmark; point this at a directory holding another version's
# nlohmann/json.hpp to compare versions (see README.md)
set(JSON_BENCHMARK_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../single_include" CACHE PATH
"directory containing the nlohmann/json.hpp to benchmark")
# benchmark binary
add_executable(json_benchmarks src/benchmarks.cpp)
target_compile_features(json_benchmarks PRIVATE cxx_std_11)
target_link_libraries(json_benchmarks benchmark ${CMAKE_THREAD_LIBS_INIT})
add_dependencies(json_benchmarks download_test_data)
target_include_directories(json_benchmarks PRIVATE ${CMAKE_SOURCE_DIR}/../../single_include ${CMAKE_BINARY_DIR}/include)
target_include_directories(json_benchmarks PRIVATE ${JSON_BENCHMARK_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/include)
+130
View File
@@ -0,0 +1,130 @@
# Benchmarks
Micro-benchmarks for parsing, serialization and the binary formats, written with
[Google Benchmark](https://github.com/google/benchmark). They are not run by CI; see
[When to run them](#when-to-run-them).
## What is measured
| benchmark | what it does |
|---|---|
| `ParseFile`, `ParseString` | parse JSON from a file stream or a string |
| `ParseIndented` | parse the large files re-indented by 4 spaces, for the lexer's whitespace handling |
| `Dump` | serialize, compact (`-`) and indented (`4`) |
| `ToCbor`, `BinaryToCbor` | write CBOR; `BinaryToCbor` writes binary values of growing size |
| `FromMsgpack` | read MessagePack; unchanged over the years, so its numbers stay comparable across releases |
| `FromBinaryBuffer`, `FromBinaryFile` | read CBOR, MessagePack, UBJSON, BJData and BSON from a buffer or a `FILE*` |
| `FromBinaryShape` | read deeply nested, container-heavy and scalar-heavy documents in every binary format |
| `FromCborChunkedString` | read CBOR strings split into indefinite-length chunks |
The input files are those of [nativejson-benchmark](https://github.com/miloyip/nativejson-benchmark) (`canada`,
`citm_catalog`, `twitter`), a large `jeopardy` file, and number-heavy files (`floats`, `signed_ints`, ...).
`bytes_per_second` counts the bytes read or written: the JSON text when parsing, the output when serializing.
## Requirements
- CMake 3.14 or later, a C++11 compiler, and Ninja for the `make` target.
- Network access on the first configure: CMake downloads Google Benchmark and the
[test data](https://github.com/nlohmann/json_test_data) into the build directory. To reuse a download of the test
data, pass `-DJSON_TestDataDirectory=<build directory>/test_files`.
- Google Benchmark is pinned to a release (1.9.5), so that results from different days stay comparable. To update it,
change `JSON_GOOGLE_BENCHMARK_VERSION` and the archive's `URL_HASH` in `CMakeLists.txt` together.
- The benchmarks include `single_include/nlohmann/json.hpp`, so run `make amalgamate` after changing anything in
`include/`.
GCC and Clang builds use `-O3 -flto -DNDEBUG`.
## Running them
From the repository root, this builds everything from scratch in `cmake-build-benchmarks` and runs all benchmarks:
```sh
make run_benchmarks
```
To build once and run selectively:
```sh
cmake -S tests/benchmarks -B build-benchmarks -G Ninja -DCMAKE_BUILD_TYPE=Release
cmake --build build-benchmarks
build-benchmarks/json_benchmarks --benchmark_filter='ParseString|Dump'
```
Useful options of `json_benchmarks`:
| option | effect |
|---|---|
| `--benchmark_list_tests` | list the benchmarks instead of running them |
| `--benchmark_filter=<regex>` | run only the benchmarks whose names match |
| `--benchmark_repetitions=<n>` | run every benchmark `n` times and add mean, median, standard deviation and coefficient of variation |
| `--benchmark_enable_random_interleaving=true` | run the repetitions in random order, which spreads out drifts such as thermal throttling |
| `--benchmark_min_time=<seconds>s` | run each benchmark at least this long (e.g. `2s`) |
| `--benchmark_out=<file> --benchmark_out_format=json` | also write the results to a file, e.g. for `compare.py` |
## Reading the output
Each line shows the wall-clock `Time` and the `CPU` time per iteration, the number of `Iterations` Google Benchmark
chose, and the throughput in `bytes_per_second`. With repetitions, the lines ending in `_median` are the ones to
compare. A `_cv` (coefficient of variation) above a few percent means the machine was too noisy for small
differences to mean anything.
## Comparing two versions
To see what a change or a release did, build the same benchmarks twice: once against the header of the version to
compare with, and once against the current one. `JSON_BENCHMARK_INCLUDE_DIR` names the directory holding the
`nlohmann/json.hpp` to benchmark. For example, to compare the current checkout with 3.12.0:
```sh
# the header of the version to compare with
mkdir -p build-baseline-header/nlohmann
git show v3.12.0:single_include/nlohmann/json.hpp > build-baseline-header/nlohmann/json.hpp
# the same benchmarks, built against either header
cmake -S tests/benchmarks -B build-baseline -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DJSON_BENCHMARK_INCLUDE_DIR="$PWD/build-baseline-header"
cmake -S tests/benchmarks -B build-current -G Ninja -DCMAKE_BUILD_TYPE=Release
cmake --build build-baseline
cmake --build build-current
# run both, back to back
build-baseline/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \
--benchmark_out=build-baseline/results.json --benchmark_out_format=json
build-current/json_benchmarks --benchmark_repetitions=10 --benchmark_enable_random_interleaving=true \
--benchmark_out=build-current/results.json --benchmark_out_format=json
```
Google Benchmark ships a tool to compare the two result files. It needs NumPy and SciPy:
```sh
python3 -m venv build-venv
build-venv/bin/pip install numpy scipy
build-venv/bin/python build-current/_deps/benchmark-src/tools/compare.py -a benchmarks build-baseline/results.json build-current/results.json
```
The tool's own `tools/requirements.txt` pins NumPy and SciPy versions that need Python 3.11 or later; with an older
Python, unpinned versions work as well. In its output:
- the `Time` and `CPU` columns are relative changes: `-0.35` means 35% faster, `+0.10` means 10% slower;
- `_pvalue` lines report a Mann-Whitney U test of whether the two versions differ. It needs at least 9
repetitions, and a p-value below 0.05 means the difference is unlikely to be noise;
- `OVERALL_GEOMEAN` summarizes all benchmarks;
- `-a` shows only the aggregates, not every repetition.
The header you compare with must support everything the benchmarks use. The current benchmarks build against 3.12.0.
Only benchmarks present in both result files are compared, so for older releases, either filter the benchmarks or
build that release's own `tests/benchmarks` against its own header.
## Getting stable numbers
- Build and run both versions on the same machine, one right after the other.
- Keep the machine otherwise idle: no builds, no browser, and a laptop plugged in.
- On Linux, set the CPU frequency governor to `performance`, e.g. `sudo cpupower frequency-set --governor performance`.
Google Benchmark prints a warning when frequency scaling is enabled. Pinning the process to a core
(`taskset -c 2 ...`) helps as well.
- Use 10 or more repetitions with random interleaving, compare medians, and treat changes within the `_cv` as noise.
## When to run them
They are a manual step, not part of CI: shared CI runners vary more between runs than most of the effects measured.
Run the comparison above before a release, comparing the previous release tag with `develop`, and for pull requests
that claim to change performance.
+2 -1
View File
@@ -131,7 +131,8 @@ static void Dump(benchmark::State& state, const char* filename, int indent)
while (state.KeepRunning())
{
j.dump(indent);
std::string output = j.dump(indent);
benchmark::DoNotOptimize(output);
}
state.SetBytesProcessed(state.iterations() * j.dump(indent).size());
+76
View File
@@ -270,6 +270,82 @@ TEST_CASE("controlled bad_alloc")
}
}
namespace
{
// counts the allocations of pairs with a non-const first member: the object
// types store std::pair<const Key, T>, so only the scratch space of the
// iterative deep copy allocates std::pair<Key, T>
std::size_t scratch_pair_allocations = 0;
template<class T>
struct is_scratch_pair : std::false_type {};
template<class K, class V>
struct is_scratch_pair<std::pair<K, V>> : std::integral_constant < bool, !std::is_const<K>::value > {};
template<class T>
struct scratch_counting_allocator : std::allocator<T>
{
using std::allocator<T>::allocator;
T* allocate(std::size_t n)
{
if (is_scratch_pair<T>::value)
{
++scratch_pair_allocations;
}
return std::allocator<T>::allocate(n);
}
#ifdef __cpp_lib_allocate_at_least
// std::allocator<T>::allocate_at_least would bypass the counting, and
// libc++'s containers prefer it over allocate from C++23 on
auto allocate_at_least(std::size_t n)
{
if (is_scratch_pair<T>::value)
{
++scratch_pair_allocations;
}
return std::allocator<T>::allocate_at_least(n);
}
#endif
template <class U>
struct rebind
{
using other = scratch_counting_allocator<U>;
};
};
} // namespace
TEST_CASE("deep copy uses the provided allocator")
{
using counting_json = nlohmann::basic_json<std::map,
std::vector,
std::string,
bool,
std::int64_t,
std::uint64_t,
double,
scratch_counting_allocator>;
// deeper than the 128 levels the copy constructor descends into, so the
// innermost objects are copied by the iterative deep copy
counting_json j = 1;
for (std::size_t i = 0; i < 300; ++i)
{
counting_json wrapper = counting_json::object();
wrapper["a"] = std::move(j);
j = std::move(wrapper);
}
scratch_pair_allocations = 0;
// NOLINTNEXTLINE(performance-unnecessary-copy-initialization): the copy is what is tested
const counting_json copy(j);
CHECK(scratch_pair_allocations > 0);
CHECK(copy == j);
}
namespace
{
template<class T>
+138 -1
View File
@@ -49,7 +49,7 @@ using huge_binary_json = nlohmann::basic_json <
// for *object keys* (e.g. "s" or "nested" below). Only the designated test
// value is meant to lie about its size - if every huge_string_t (including
// keys) reported a huge size, the running totals computed while walking the
// BSON document (see calc_bson_object_size & friends in binary_writer.hpp)
// BSON document (see calc_bson_sizes in binary_writer.hpp)
// would need more than 32 bits, and on platforms where std::size_t is only
// 32 bits wide that arithmetic would silently wrap around, producing wrong
// (or even unguarded) lengths. The fake size is therefore opt-in via
@@ -1697,3 +1697,140 @@ TEST_CASE("BSON roundtrips" * doctest::skip())
}
}
}
TEST_CASE("BSON: deeply nested values")
{
SECTION("documents and arrays round-trip at every depth")
{
// nested documents and arrays, with siblings on every level, so
// every length prefix covers entries of both kinds
json value = "leaf";
for (std::size_t depth = 0; depth <= 300; ++depth)
{
CAPTURE(depth);
const json document = {{"value", value}, {"n", depth}};
CHECK(json::from_bson(json::to_bson(document)) == document);
value = depth % 2 == 0 ? json{{"a", std::move(value)}, {"b", {1, "x"}}} :
json::array({std::move(value), depth, json::object()});
}
}
SECTION("a key containing U+0000 is rejected before anything is written")
{
json value = json::object({{std::string("bad\0key", 7), 1}});
for (std::size_t depth = 0; depth < 200; ++depth)
{
value = json{{"a", {{"b", 1}}}, {"z", std::move(value)}};
}
std::vector<std::uint8_t> output;
CHECK_THROWS_AS(json::to_bson(value, output), json::out_of_range&);
CHECK(output.empty());
}
SECTION("values nested too deeply for the call stack (#5392)")
{
// serializing recursed once per nesting level, and computed every
// nested document's length by walking everything below it again.
// The values are only parsed, serialized and walked, never copied or
// compared, since those recurse too.
const std::size_t depth = 100000;
for (const bool objects :
{
false, true
})
{
CAPTURE(objects);
std::string text = "{\"a\":";
for (std::size_t i = 0; i < depth; ++i)
{
text += objects ? "{\"a\":" : "[";
}
text += "1";
text.append(depth, objects ? '}' : ']');
text += "}";
const auto bson = json::to_bson(json::parse(text));
const auto result = json::from_bson(bson);
const json* p = &result.at("a");
for (std::size_t i = 0; i < depth; ++i)
{
p = objects ? &p->at("a") : &p->at(0);
}
CHECK(*p == 1);
}
}
}
TEST_CASE("Invalid document size handling")
{
SECTION("document size must be at least 5")
{
std::vector<std::uint8_t> const v = {0x04, 0x00, 0x00, 0x00, 0x00};
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 4 does not match the number of bytes read (5)", json::parse_error&);
CHECK(json::from_bson(v, true, false).is_discarded());
}
SECTION("declared document size must match consumed bytes (extra trailing element)")
{
// Declares 5-byte empty document but appends an int32 element after the declared end.
std::vector<std::uint8_t> const v =
{
0x05, 0x00, 0x00, 0x00,
0x10, 'a', 'd', 'm', 'i', 'n', 0x00,
0x01, 0x00, 0x00, 0x00,
0x00
};
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 16: syntax error while parsing BSON document: document size 5 does not match the number of bytes read (16)", json::parse_error&);
CHECK(json::from_bson(v, true, false).is_discarded());
}
SECTION("declared document size must match consumed bytes (premature terminator)")
{
// Declares 32-byte document but only contains the size field followed by an immediate terminator.
std::vector<std::uint8_t> const v =
{
0x20, 0x00, 0x00, 0x00,
0x00
};
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 5: syntax error while parsing BSON document: document size 32 does not match the number of bytes read (5)", json::parse_error&);
CHECK(json::from_bson(v, true, false).is_discarded());
}
SECTION("array declared size must match consumed bytes")
{
// Outer object contains an array "a" that declares 5 bytes (empty) but
// actually contains an int32 element before its terminator.
std::vector<std::uint8_t> const v =
{
0x14, 0x00, 0x00, 0x00, // object size = 20
0x04, 'a', 0x00, // key "a", array type
0x05, 0x00, 0x00, 0x00, // array declared size = 5 (empty)
0x10, '0', 0x00, 0x01, 0x00, 0x00, 0x00, // extra int32 element "0" = 1
0x00, // array terminator
0x00 // object terminator
};
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 19: syntax error while parsing BSON document: document size 5 does not match the number of bytes read (12)", json::parse_error&);
CHECK(json::from_bson(v, true, false).is_discarded());
}
SECTION("BSON string must end with 0x00")
{
// Length-prefixed string whose terminator byte is 'X' (0x58), not 0x00.
std::vector<std::uint8_t> const v =
{
0x0F, 0x00, 0x00, 0x00,
0x02, 's', 0x00,
0x02, 0x00, 0x00, 0x00,
'A', 'X',
0x00
};
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(v), "[json.exception.parse_error.112] parse error at byte 13: syntax error while parsing BSON string: BSON string is not null-terminated", json::parse_error&);
CHECK(json::from_bson(v, true, false).is_discarded());
}
}
+53
View File
@@ -1833,6 +1833,59 @@ TEST_CASE("CBOR")
CHECK(json::from_cbor(std::vector<uint8_t>({0xa1, 0xff, 0x01}), true, false).is_discarded());
}
SECTION("invalid UTF-8 in string (see #5529)")
{
// a two-character text string (major type 3) whose bytes are not
// valid UTF-8 (0xC0 0xAE is an overlong encoding of '.') must be
// rejected at decode time, matching every other kind of
// malformed binary input, rather than only failing later when
// the resulting value is dumped
json _;
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x62, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
CHECK(json::from_cbor(std::vector<uint8_t>({0x62, 0xc0, 0xae}), true, false).is_discarded());
// a CBOR byte string (major type 2) with the very same bytes is
// NOT text and must still be accepted as-is
CHECK_NOTHROW(_ = json::from_cbor(std::vector<uint8_t>({0x42, 0xc0, 0xae})));
CHECK(_ == json::binary(std::vector<std::uint8_t>({0xc0, 0xae})));
// valid UTF-8 must still round-trip
const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語
CHECK(json::from_cbor(json::to_cbor(j)) == j);
}
SECTION("invalid UTF-8 in indefinite-length string")
{
json _;
// every chunk must be valid UTF-8 on its own (RFC 8949, Section
// 3.2.3), so a code point split across two chunks is rejected
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
CHECK(json::from_cbor(std::vector<uint8_t>({0x7f, 0x61, 0xc3, 0x61, 0xa9, 0xff}), true, false).is_discarded());
// an ill-formed later chunk is rejected after valid ones
CHECK_THROWS_WITH_AS(_ = json::from_cbor(std::vector<uint8_t>({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc0, 0xae, 0xff})), "[json.exception.parse_error.113] parse error at byte 7: syntax error while parsing CBOR string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
// valid multi-byte chunks are accepted
CHECK(json::from_cbor(std::vector<uint8_t>({0x7f, 0x62, 0xc3, 0xa9, 0x62, 0xc3, 0xb6, 0xff})) == "\xc3\xa9\xc3\xb6");
}
SECTION("many chunks in indefinite-length string")
{
// only the newly read chunk is validated, not the whole string
// collected so far; validating the latter made this input take
// quadratic time (about ten seconds for 100000 chunks)
constexpr std::size_t chunks = 100000;
std::vector<uint8_t> v{0x7f};
for (std::size_t i = 0; i < chunks; ++i)
{
v.push_back(0x61);
v.push_back('a');
}
v.push_back(0xff);
CHECK(json::from_cbor(v) == std::string(chunks, 'a'));
}
SECTION("strict mode")
{
std::vector<uint8_t> const vec = {0xf6, 0xf6};
+15 -1
View File
@@ -11,11 +11,15 @@
// capture whether JSON_STRICT_NUL_HANDLING was enabled on the command line
// (e.g. -DJSON_STRICT_NUL_HANDLING=1) *before* including json.hpp, since the
// library #undefs JSON_STRICT_NUL_HANDLING itself once the header has been
// fully processed (see include/nlohmann/detail/macro_unscope.hpp)
// fully processed unless JSON_TEST_KEEP_MACROS is defined (see
// include/nlohmann/detail/macro_unscope.hpp)
#if defined(JSON_STRICT_NUL_HANDLING) && (JSON_STRICT_NUL_HANDLING == 1)
#define JSON_TEST_STRICT_NUL_HANDLING_ENABLED 1
#endif
#define JSON_TEST_STRINGIZE_EX(x) #x
#define JSON_TEST_STRINGIZE(x) JSON_TEST_STRINGIZE_EX(x)
#define JSON_TESTS_PRIVATE
#include <nlohmann/json.hpp>
using nlohmann::json;
@@ -566,6 +570,16 @@ TEST_CASE("parser class")
// left at its default or forced to 1 (e.g. by the dedicated
// ci_test_strict_nul_handling CI target), so only the section
// matching the actual, compiled-in behavior can pass.
SECTION("the macro is part of the ABI tag")
{
const std::string ns = JSON_TEST_STRINGIZE(NLOHMANN_JSON_NAMESPACE);
#if defined(JSON_TEST_STRICT_NUL_HANDLING_ENABLED)
CHECK(ns.find("_snul") != std::string::npos);
#else
CHECK(ns.find("_snul") == std::string::npos);
#endif
}
#if !defined(JSON_TEST_STRICT_NUL_HANDLING_ENABLED)
SECTION("default behavior (macro not enabled)")
{
+21
View File
@@ -1554,6 +1554,27 @@ TEST_CASE("MessagePack")
CHECK(json::from_msgpack(std::vector<uint8_t>({0x81, 0xff, 0x01}), true, false).is_discarded());
}
SECTION("invalid UTF-8 in string (see #5529)")
{
// a fixstr of length 2 (0xA0 | 2) whose bytes are not valid UTF-8
// (0xC0 0xAE is an overlong encoding of '.') must be rejected at
// decode time, matching every other kind of malformed binary
// input, rather than only failing later when the resulting
// value is dumped
json _;
CHECK_THROWS_WITH_AS(_ = json::from_msgpack(std::vector<uint8_t>({0xa2, 0xc0, 0xae})), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing MessagePack string: invalid string: ill-formed UTF-8 byte", json::parse_error&);
CHECK(json::from_msgpack(std::vector<uint8_t>({0xa2, 0xc0, 0xae}), true, false).is_discarded());
// a MessagePack bin8 blob with the very same bytes is NOT text
// and must still be accepted as-is
CHECK_NOTHROW(_ = json::from_msgpack(std::vector<uint8_t>({0xc4, 0x02, 0xc0, 0xae})));
CHECK(_ == json::binary(std::vector<std::uint8_t>({0xc0, 0xae})));
// valid UTF-8 must still round-trip
const json j = "h\xc3\xa9llo, w\xc3\xb6rld! \xe6\x97\xa5\xe6\x9c\xac\xe8\xaa\x9e"; // héllo, wörld! 日本語
CHECK(json::from_msgpack(json::to_msgpack(j)) == j);
}
SECTION("strict mode")
{
std::vector<uint8_t> const vec = {0xc0, 0xc0};
+10
View File
@@ -94,6 +94,16 @@ TEST_CASE("serialization")
CHECK(j.dump(-1, ' ', true, json::error_handler_t::replace) == "\"\\u00e4\\ufffd\\u00fc\"");
}
SECTION("invalid character (regression guard for shared UTF-8 decoder, see #5529)")
{
// dump_escaped_impl() now calls the UTF-8 decoder shared with the
// binary readers (detail::decode() in string_utils.hpp) instead
// of a private copy; the exact type_error.316 message/behavior
// must stay byte-for-byte the same as before that extraction
const json j = "ä\xA9ü";
CHECK_THROWS_WITH_AS(utils::ignore_return_value(j.dump()), "[json.exception.type_error.316] invalid UTF-8 byte at index 2: 0xA9", json::type_error&);
}
SECTION("ending with incomplete character")
{
const json j = "123\xC2";
+367
View File
@@ -778,6 +778,193 @@ class derived_person_only_serialize_private_3 : person_without_default_construct
NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE_WITH_NAMES(derived_person_only_serialize_private_3, person_without_default_constructor_3, "json_hair_color", hair_color)
};
// Zero-member types for issue #4041: NLOHMANN_DEFINE_TYPE_* and
// NLOHMANN_DEFINE_DERIVED_TYPE_* must compile and produce a valid (empty)
// JSON object when no member arguments are given.
class empty_intrusive
{
public:
bool operator==(const empty_intrusive& /*rhs*/) const
{
return true;
}
NLOHMANN_DEFINE_TYPE_INTRUSIVE(empty_intrusive)
};
class empty_intrusive_with_default
{
public:
bool operator==(const empty_intrusive_with_default& /*rhs*/) const
{
return true;
}
NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(empty_intrusive_with_default)
};
class empty_intrusive_only_serialize
{
public:
NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE(empty_intrusive_only_serialize)
};
class empty_non_intrusive
{
public:
bool operator==(const empty_non_intrusive& /*rhs*/) const
{
return true;
}
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(empty_non_intrusive)
class empty_non_intrusive_with_default
{
public:
bool operator==(const empty_non_intrusive_with_default& /*rhs*/) const
{
return true;
}
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(empty_non_intrusive_with_default)
class empty_non_intrusive_only_serialize {};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(empty_non_intrusive_only_serialize)
class empty_derived_intrusive : public person_with_private_data
{
public:
empty_derived_intrusive() = default;
empty_derived_intrusive(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE(empty_derived_intrusive, person_with_private_data)
};
class empty_derived_intrusive_with_default : public person_with_private_data
{
public:
empty_derived_intrusive_with_default() = default;
empty_derived_intrusive_with_default(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT(empty_derived_intrusive_with_default, person_with_private_data)
};
class empty_derived_intrusive_only_serialize : public person_with_private_data
{
public:
empty_derived_intrusive_only_serialize() = default;
empty_derived_intrusive_only_serialize(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE(empty_derived_intrusive_only_serialize, person_with_private_data)
};
class empty_derived_non_intrusive : public person_with_private_data
{
public:
empty_derived_non_intrusive() = default;
empty_derived_non_intrusive(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(empty_derived_non_intrusive, person_with_private_data)
class empty_derived_non_intrusive_with_default : public person_with_private_data
{
public:
empty_derived_non_intrusive_with_default() = default;
empty_derived_non_intrusive_with_default(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT(empty_derived_non_intrusive_with_default, person_with_private_data)
class empty_derived_non_intrusive_only_serialize : public person_with_private_data
{
public:
empty_derived_non_intrusive_only_serialize() = default;
empty_derived_non_intrusive_only_serialize(std::string name_, int age_, json metadata_)
: person_with_private_data(std::move(name_), age_, std::move(metadata_))
{}
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE(empty_derived_non_intrusive_only_serialize, person_with_private_data)
// Types at the documented maximum member count (63) for issue #4041's
// argument-count dispatch. The derived-type macros carry a two-token
// Type,BaseType prefix, so they reach two slots further into
// NLOHMANN_JSON_GET_MACRO than the non-derived ones and are the first to break
// if the tag dispatch runs out of positional slots.
class max_members
{
public:
int m1{}, m2{}, m3{}, m4{}, m5{}, m6{}, m7{}, m8{}, m9{}, m10{}, m11{}, m12{}, m13{}, m14{}, m15{}, m16{}, m17{}, m18{}, m19{}, m20{}, m21{}, m22{}, m23{}, m24{}, m25{}, m26{}, m27{}, m28{}, m29{}, m30{}, m31{}, m32{}, m33{}, m34{}, m35{}, m36{}, m37{}, m38{}, m39{}, m40{}, m41{}, m42{}, m43{}, m44{}, m45{}, m46{}, m47{}, m48{}, m49{}, m50{}, m51{}, m52{}, m53{}, m54{}, m55{}, m56{}, m57{}, m58{}, m59{}, m60{}, m61{}, m62{}, m63{};
NLOHMANN_DEFINE_TYPE_INTRUSIVE(max_members, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15, m16, m17, m18, m19, m20, m21, m22, m23, m24, m25, m26, m27, m28, m29, m30, m31, m32, m33, m34, m35, m36, m37, m38, m39, m40, m41, m42, m43, m44, m45, m46, m47, m48, m49, m50, m51, m52, m53, m54, m55, m56, m57, m58, m59, m60, m61, m62, m63)
};
class max_members_base
{
public:
int base_value = 0;
NLOHMANN_DEFINE_TYPE_INTRUSIVE(max_members_base, base_value)
};
class max_members_derived : public max_members_base
{
public:
int m1{}, m2{}, m3{}, m4{}, m5{}, m6{}, m7{}, m8{}, m9{}, m10{}, m11{}, m12{}, m13{}, m14{}, m15{}, m16{}, m17{}, m18{}, m19{}, m20{}, m21{}, m22{}, m23{}, m24{}, m25{}, m26{}, m27{}, m28{}, m29{}, m30{}, m31{}, m32{}, m33{}, m34{}, m35{}, m36{}, m37{}, m38{}, m39{}, m40{}, m41{}, m42{}, m43{}, m44{}, m45{}, m46{}, m47{}, m48{}, m49{}, m50{}, m51{}, m52{}, m53{}, m54{}, m55{}, m56{}, m57{}, m58{}, m59{}, m60{}, m61{}, m62{}, m63{};
NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE(max_members_derived, max_members_base, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15, m16, m17, m18, m19, m20, m21, m22, m23, m24, m25, m26, m27, m28, m29, m30, m31, m32, m33, m34, m35, m36, m37, m38, m39, m40, m41, m42, m43, m44, m45, m46, m47, m48, m49, m50, m51, m52, m53, m54, m55, m56, m57, m58, m59, m60, m61, m62, m63)
};
// User macros named like the dispatch suffixes (EMPTY is a common empty-macro
// idiom) must not leak into the NLOHMANN_DEFINE_TYPE_* dispatch.
#define EMPTY
#define MEMBERS clobbered_by_user_macro
class dispatch_with_user_macros_empty
{
public:
NLOHMANN_DEFINE_TYPE_INTRUSIVE(dispatch_with_user_macros_empty)
};
class dispatch_with_user_macros_members
{
public:
int value = 0;
NLOHMANN_DEFINE_TYPE_INTRUSIVE(dispatch_with_user_macros_members, value)
};
class dispatch_with_user_macros_derived_empty : public dispatch_with_user_macros_members
{
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(dispatch_with_user_macros_derived_empty, dispatch_with_user_macros_members)
class dispatch_with_user_macros_derived_members : public dispatch_with_user_macros_members
{
public:
int own = 0;
};
// NOLINTNEXTLINE(misc-use-internal-linkage)
NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE(dispatch_with_user_macros_derived_members, dispatch_with_user_macros_members, own)
// testing for the macros also keeps -Wunused-macros from rejecting them
#if !defined(EMPTY) || !defined(MEMBERS)
#error "EMPTY and MEMBERS must stay defined for the tests above"
#endif
#undef EMPTY
#undef MEMBERS
} // namespace persons
TEST_CASE_TEMPLATE("Serialization/deserialization via NLOHMANN_DEFINE_TYPE_INTRUSIVE and NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE", Pair, // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization)
@@ -1191,3 +1378,183 @@ TEST_CASE_TEMPLATE("Serialization of non-default-constructible classes via NLOHM
}
}
}
// Regression tests for issue #4041: NLOHMANN_DEFINE_TYPE_* and
// NLOHMANN_DEFINE_DERIVED_TYPE_* macros must compile and produce valid
// (empty, or base-only for the derived case) JSON objects when no member
// arguments are given, on every supported C++ standard.
TEST_CASE_TEMPLATE("Serialization/deserialization of zero-member types via NLOHMANN_DEFINE_TYPE_* (issue #4041)", Json, // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization)
nlohmann::json, nlohmann::ordered_json)
{
constexpr bool is_ordered = std::is_same<Json, nlohmann::ordered_json>::value;
const char* const derived_dump = is_ordered
? R"({"age":1,"name":"Erik","metadata":null})"
: R"({"age":1,"metadata":null,"name":"Erik"})";
SECTION("NLOHMANN_DEFINE_TYPE_INTRUSIVE with zero members")
{
persons::empty_intrusive obj{};
Json j = obj;
CHECK(j.dump() == "{}");
CHECK(j.template get<persons::empty_intrusive>() == obj);
}
SECTION("NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT with zero members")
{
persons::empty_intrusive_with_default obj{};
Json j = obj;
CHECK(j.dump() == "{}");
CHECK(j.template get<persons::empty_intrusive_with_default>() == obj);
}
SECTION("NLOHMANN_DEFINE_TYPE_INTRUSIVE_ONLY_SERIALIZE with zero members")
{
const persons::empty_intrusive_only_serialize obj{};
Json j = obj;
CHECK(j.dump() == "{}");
}
SECTION("NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE with zero members")
{
persons::empty_non_intrusive obj{};
Json j = obj;
CHECK(j.dump() == "{}");
CHECK(j.template get<persons::empty_non_intrusive>() == obj);
}
SECTION("NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT with zero members")
{
persons::empty_non_intrusive_with_default obj{};
Json j = obj;
CHECK(j.dump() == "{}");
CHECK(j.template get<persons::empty_non_intrusive_with_default>() == obj);
}
SECTION("NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE with zero members")
{
const persons::empty_non_intrusive_only_serialize obj{};
Json j = obj;
CHECK(j.dump() == "{}");
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE with zero own members")
{
persons::empty_derived_intrusive obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
CHECK(j.template get<persons::empty_derived_intrusive>() == obj);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_WITH_DEFAULT with zero own members")
{
persons::empty_derived_intrusive_with_default obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
CHECK(j.template get<persons::empty_derived_intrusive_with_default>() == obj);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE_ONLY_SERIALIZE with zero own members")
{
const persons::empty_derived_intrusive_only_serialize obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE with zero own members")
{
persons::empty_derived_non_intrusive obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
CHECK(j.template get<persons::empty_derived_non_intrusive>() == obj);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_WITH_DEFAULT with zero own members")
{
persons::empty_derived_non_intrusive_with_default obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
CHECK(j.template get<persons::empty_derived_non_intrusive_with_default>() == obj);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_NON_INTRUSIVE_ONLY_SERIALIZE with zero own members")
{
const persons::empty_derived_non_intrusive_only_serialize obj{"Erik", 1, nullptr};
Json j = obj;
CHECK(j.dump() == derived_dump);
}
}
// Regression test for the argument-count dispatch added for issue #4041: the
// documented maximum of 63 members must keep working, including for the
// derived-type macros whose Type,BaseType prefix consumes two dispatch slots.
TEST_CASE_TEMPLATE("Serialization/deserialization of maximum-member-count types via NLOHMANN_DEFINE_TYPE_*", Json, // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization)
nlohmann::json, nlohmann::ordered_json)
{
SECTION("NLOHMANN_DEFINE_TYPE_INTRUSIVE with 63 members")
{
persons::max_members obj{};
obj.m1 = 1;
obj.m63 = 63;
Json j = obj;
CHECK(j.size() == 63);
const auto obj2 = j.template get<persons::max_members>();
CHECK(obj2.m1 == 1);
CHECK(obj2.m63 == 63);
}
SECTION("NLOHMANN_DEFINE_DERIVED_TYPE_INTRUSIVE with 63 own members")
{
persons::max_members_derived obj{};
obj.base_value = 7;
obj.m1 = 1;
obj.m63 = 63;
Json j = obj;
CHECK(j.size() == 64);
const auto obj2 = j.template get<persons::max_members_derived>();
CHECK(obj2.base_value == 7);
CHECK(obj2.m1 == 1);
CHECK(obj2.m63 == 63);
}
}
TEST_CASE_TEMPLATE("NLOHMANN_DEFINE_TYPE_* dispatch is unaffected by user macros named EMPTY or MEMBERS", Json, // NOLINT(readability-math-missing-parentheses, bugprone-throwing-static-initialization)
nlohmann::json, nlohmann::ordered_json)
{
SECTION("zero members")
{
const persons::dispatch_with_user_macros_empty obj{};
const Json j = obj;
CHECK(j == Json::object());
CHECK_NOTHROW(j.template get<persons::dispatch_with_user_macros_empty>());
}
SECTION("one member")
{
persons::dispatch_with_user_macros_members obj{};
obj.value = 42;
const Json j = obj;
CHECK(j == Json({{"value", 42}}));
CHECK(j.template get<persons::dispatch_with_user_macros_members>().value == 42);
}
SECTION("derived with zero own members")
{
persons::dispatch_with_user_macros_derived_empty obj{};
obj.value = 42;
const Json j = obj;
CHECK(j == Json({{"value", 42}}));
CHECK(j.template get<persons::dispatch_with_user_macros_derived_empty>().value == 42);
}
SECTION("derived with own members")
{
persons::dispatch_with_user_macros_derived_members obj{};
obj.value = 42;
obj.own = 7;
const Json j = obj;
CHECK(j == Json({{"value", 42}, {"own", 7}}));
const auto obj2 = j.template get<persons::dispatch_with_user_macros_derived_members>();
CHECK(obj2.value == 42);
CHECK(obj2.own == 7);
}
}